boltons-16.4.1/0000755000175000017500000000000012727357214011377 5ustar hlehleboltons-16.4.1/setup.py0000644000175000017500000000255612727357214013121 0ustar hlehle"""Functionality that should be in the standard library. Like builtins, but Boltons. Otherwise known as, "everyone's util.py," but cleaned up and tested. Contains over 160 BSD-licensed utility types and functions that can be used as a package or independently. `Extensively documented on Read the Docs `_. """ from setuptools import setup __author__ = 'Mahmoud Hashemi' __version__ = '16.4.1' __contact__ = 'mahmoudrhashemi@gmail.com' __url__ = 'https://github.com/mahmoud/boltons' __license__ = 'BSD' setup(name='boltons', version=__version__, description="When they're not builtins, they're boltons.", long_description=__doc__, author=__author__, author_email=__contact__, url=__url__, packages=['boltons'], include_package_data=True, zip_safe=False, license=__license__, platforms='any', classifiers=[ 'Topic :: Utilities', 'Intended Audience :: Developers', 'Topic :: Software Development :: Libraries', 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: Implementation :: PyPy', ] ) boltons-16.4.1/requirements-rtd.txt0000644000175000017500000000013012727357214015444 0ustar hlehlesphinxcontrib-napoleon # sphinx_rtd_theme # you'll need sphinx_rtd_theme for local dev boltons-16.4.1/tests/0000755000175000017500000000000012727357214012541 5ustar hlehleboltons-16.4.1/tests/test_funcutils_fb_py3.py0000644000175000017500000000376612727357214017444 0ustar hlehlefrom boltons.funcutils import wraps, FunctionBuilder import pytest def pita_wrap(flag=False): def cedar_dec(func): @wraps(func) def cedar_wrapper(*a, **kw): return (flag, func.__name__, func(*a, **kw)) return cedar_wrapper return cedar_dec def test_wraps_py3(): @pita_wrap(flag=True) def annotations(a: int, b: float=1) -> "tuple": return a, b annotations(0) == (True, "annotations", (0, 1)) annotations.__annotations__ == {'a': int, 'b': float, 'return': 'tuple'} @pita_wrap(flag=False) def kwonly_arg(a, *, b, c=2): return a, b, c with pytest.raises(TypeError): kwonly_arg(0) kwonly_arg(0, b=1) == (False, "kwonly_arg", (0, 1, 2)) kwonly_arg(0, b=1, c=3) == (False, "kwonly_arg", (0, 1, 3)) @pita_wrap(flag=True) def kwonly_non_roundtrippable_repr(*, x=lambda y: y + 1): return x(1) assert kwonly_non_roundtrippable_repr() == ( True, 'kwonly_non_roundtrippable_repr', 2) @pytest.mark.parametrize('signature,should_match', [('a, *, b', True), ('a,*,b', True), ('a, * , b', True), ('a, *,\nb', True), ('a, *\n,b', True), ('a, b', False), ('a, *args', False), ('a, *args, **kwargs', False), ('*args', False), ('*args, **kwargs', False)]) def test_FunctionBuilder_KWONLY_MARKER(signature, should_match): """ _KWONLY_MARKER matches the keyword-only argument separator, regardless of whitespace. Note: it assumes the signature is valid Python. """ matched = bool(FunctionBuilder._KWONLY_MARKER.search(signature)) message = "{!r}: should_match was {}, but result was {}".format( signature, should_match, matched) assert bool(matched) == should_match, message boltons-16.4.1/tests/test_formatutils.py0000644000175000017500000000343212727357214016525 0ustar hlehle# -*- coding: utf-8 -*- import re from collections import namedtuple from boltons.formatutils import (get_format_args, split_format_str, tokenize_format_str, infer_positional_format_args) PFAT = namedtuple("PositionalFormatArgTest", "fstr arg_vals res") _PFATS = [PFAT('{} {} {}', ('hi', 'hello', 'bye'), "hi hello bye"), PFAT('{:d} {}', (1, 2), "1 2"), PFAT('{!s} {!r}', ('str', 'repr'), "str 'repr'"), PFAT('{[hi]}, {.__name__!r}', ({'hi': 'hi'}, re), "hi, 're'"), PFAT('{{joek}} ({} {})', ('so', 'funny'), "{joek} (so funny)")] def test_pos_infer(): for i, (tmpl, args, res) in enumerate(_PFATS): converted = infer_positional_format_args(tmpl) assert converted.format(*args) == res _TEST_TMPLS = ["example 1: {hello}", "example 2: {hello:*10}", "example 3: {hello:*{width}}", "example 4: {hello!r:{fchar}{width}}, {width}, yes", "example 5: {0}, {1:d}, {2:f}, {1}"] try: from collections import OrderedDict except ImportError: pass # skip the non-2.6 compatible tests on 2.6 else: _TEST_TMPLS.append("example 6: {}, {}, {}, {1}") del OrderedDict def test_get_fstr_args(): results = [] for t in _TEST_TMPLS: inferred_t = infer_positional_format_args(t) res = get_format_args(inferred_t) results.append(res) return results def test_split_fstr(): results = [] for t in _TEST_TMPLS: res = split_format_str(t) results.append(res) return results def test_tokenize_format_str(): results = [] for t in _TEST_TMPLS: res = tokenize_format_str(t) results.append(res) return results boltons-16.4.1/tests/test_cacheutils.py0000644000175000017500000001432412727357214016302 0ustar hlehle# -*- coding: utf-8 -*- import string from boltons.cacheutils import LRU, LRI, cached, cachedmethod, cachedproperty class CountingCallable(object): def __init__(self): self.call_count = 0 def __call__(self, *a, **kw): self.call_count += 1 return self.call_count def test_lru_add(): cache = LRU(max_size=3) for i in range(4): cache[i] = i assert len(cache) == 3 assert 0 not in cache def test_lri(): bc = LRI(10, on_miss=lambda k: k.upper()) for char in string.ascii_letters: x = bc[char] assert x == char.upper() assert len(bc) == 10 def test_lru_basic(): lru = LRU(max_size=1) repr(lru) # sanity lru['hi'] = 0 lru['bye'] = 1 assert len(lru) == 1 lru['bye'] assert lru.get('hi') is None del lru['bye'] assert 'bye' not in lru assert len(lru) == 0 assert not lru try: lru.pop('bye') except KeyError: pass else: assert False default = object() assert lru.pop('bye', default) is default try: lru.popitem() except KeyError: pass else: assert False lru['another'] = 1 assert lru.popitem() == ('another', 1) lru['yet_another'] = 2 assert lru.pop('yet_another') == 2 lru['yet_another'] = 3 assert lru.pop('yet_another', default) == 3 lru['yet_another'] = 4 lru.clear() assert not lru lru['yet_another'] = 5 second_lru = LRU(max_size=1) assert lru.copy() == lru second_lru['yet_another'] = 5 assert second_lru == lru assert lru == second_lru lru.update(LRU(max_size=2, values=[('a', 1), ('b', 2)])) assert len(lru) == 1 assert 'yet_another' not in lru lru.setdefault('x', 2) assert dict(lru) == {'x': 2} lru.setdefault('x', 3) assert dict(lru) == {'x': 2} assert lru != second_lru assert second_lru != lru def test_lru_with_dupes(): SIZE = 2 lru = LRU(max_size=SIZE) for i in [0, 0, 1, 1, 2, 2]: lru[i] = i assert _test_linkage(lru._anchor, SIZE + 1), 'linked list invalid' def test_lru_with_dupes_2(): "From Issue #55, h/t github.com/mt" SIZE = 3 lru = LRU(max_size=SIZE) keys = ['A', 'A', 'B', 'A', 'C', 'B', 'D', 'E'] for i, k in enumerate(keys): lru[k] = 'HIT' assert _test_linkage(lru._anchor, SIZE + 1), 'linked list invalid' return def _test_linkage(dll, max_count=10000, prev_idx=0, next_idx=1): """A function to test basic invariants of doubly-linked lists (with links made of Python lists). 1. Test that the list is not longer than a certain length 2. That the forward links (indicated by `next_idx`) correspond to the backward links (indicated by `prev_idx`). The `dll` parameter is the root/anchor link of the list. """ start = cur = dll i = 0 prev = None while 1: if i > max_count: raise Exception("did not return to anchor link after %r rounds" % max_count) if prev is not None and cur is start: break prev = cur cur = cur[next_idx] if cur[prev_idx] is not prev: raise Exception('prev_idx does not point to prev at i = %r' % i) i += 1 return True def test_cached_dec(): lru = LRU() inner_func = CountingCallable() func = cached(lru)(inner_func) assert inner_func.call_count == 0 func() assert inner_func.call_count == 1 func() assert inner_func.call_count == 1 func('man door hand hook car door') assert inner_func.call_count == 2 return def test_callable_cached_dec(): lru = LRU() get_lru = lambda: lru inner_func = CountingCallable() func = cached(get_lru)(inner_func) assert inner_func.call_count == 0 func() assert inner_func.call_count == 1 func() assert inner_func.call_count == 1 lru.clear() func() assert inner_func.call_count == 2 func() assert inner_func.call_count == 2 print(repr(func)) return def test_cachedmethod(): class Car(object): def __init__(self, cache=None): self.h_cache = LRI() if cache is None else cache self.door_count = 0 self.hook_count = 0 self.hand_count = 0 @cachedmethod('h_cache') def hand(self, *a, **kw): self.hand_count += 1 @cachedmethod(lambda obj: obj.h_cache) def hook(self, *a, **kw): self.hook_count += 1 @cachedmethod('h_cache', selfish=False) def door(self, *a, **kw): self.door_count += 1 car = Car() # attribute name-style assert car.hand_count == 0 car.hand('h', a='nd') assert car.hand_count == 1 car.hand('h', a='nd') assert car.hand_count == 1 # callable-style assert car.hook_count == 0 car.hook() assert car.hook_count == 1 car.hook() assert car.hook_count == 1 # Ensure that non-selfish caches share the cache nicely lru = LRU() car_one = Car(cache=lru) assert car_one.door_count == 0 car_one.door('bob') assert car_one.door_count == 1 car_one.door('bob') assert car_one.door_count == 1 car_two = Car(cache=lru) assert car_two.door_count == 0 car_two.door('bob') assert car.door_count == 0 # try unbound for kicks Car.door(Car(), 'bob') # always check the repr print(repr(car_two.door)) print(repr(Car.door)) return def test_cachedproperty(): class Proper(object): def __init__(self): self.expensive_func = CountingCallable() @cachedproperty def useful_attr(self): return self.expensive_func() prop = Proper() assert prop.expensive_func.call_count == 0 assert prop.useful_attr == 1 assert prop.expensive_func.call_count == 1 assert prop.useful_attr == 1 assert prop.expensive_func.call_count == 1 prop.useful_attr += 1 # would not be possible with normal properties assert prop.useful_attr == 2 delattr(prop, 'useful_attr') assert prop.expensive_func.call_count == 1 assert prop.useful_attr assert prop.expensive_func.call_count == 2 repr(Proper.useful_attr) boltons-16.4.1/tests/test_socketutils.py0000644000175000017500000002376412727357214016537 0ustar hlehle# -*- coding: utf-8 -*- import sys import time import errno import socket import threading from boltons.socketutils import (BufferedSocket, NetstringSocket, ConnectionClosed, NetstringMessageTooLong, MessageTooLong, Timeout) import pytest # skip if there's no socketpair pytestmark = pytest.mark.skipif(getattr(socket, 'socketpair', None) is None, reason='no socketpair (likely Py2 on Windows)') def test_short_lines(): for ms in (2, 4, 6, 1024, None): x, y = socket.socketpair() bs = BufferedSocket(x) y.sendall(b'1\n2\n3\n') assert bs.recv_until(b'\n', maxsize=ms) == b'1' assert bs.recv_until(b'\n', maxsize=ms) == b'2' y.close() assert bs.recv_close(maxsize=ms) == b'3\n' try: bs.recv_size(1) except ConnectionClosed: pass else: assert False, 'expected ConnectionClosed' bs.close() return def test_multibyte_delim(): """Primarily tests recv_until with various maxsizes and True/False for with_delimiter. """ delim = b'\r\n' for with_delim in (True, False): if with_delim: cond_delim = b'\r\n' else: cond_delim = b'' empty = b'' small_one = b'1' big_two = b'2' * 2048 for ms in (3, 5, 1024, None): x, y = socket.socketpair() bs = BufferedSocket(x) y.sendall(empty + delim) y.sendall(small_one + delim) y.sendall(big_two + delim) kwargs = {'maxsize': ms, 'with_delimiter': with_delim} assert bs.recv_until(delim, **kwargs) == empty + cond_delim assert bs.recv_until(delim, **kwargs) == small_one + cond_delim try: assert bs.recv_until(delim, **kwargs) == big_two + cond_delim except MessageTooLong: if ms is None: assert False, 'unexpected MessageTooLong' else: if ms is not None: assert False, 'expected MessageTooLong' return def test_props(): x, y = socket.socketpair() bs = BufferedSocket(x) assert bs.type == x.type assert bs.proto == x.proto assert bs.family == x.family return def test_buffers(): x, y = socket.socketpair() bx, by = BufferedSocket(x), BufferedSocket(y) assert by.getrecvbuffer() == b'' assert by.getsendbuffer() == b'' assert bx.getrecvbuffer() == b'' by.buffer(b'12') by.sendall(b'3') assert bx.recv_size(1) == b'1' assert bx.getrecvbuffer() == b'23' return def test_client_disconnecting(): def get_bs_pair(): x, y = socket.socketpair() bx, by = BufferedSocket(x), BufferedSocket(y) # sanity check by.sendall(b'123') bx.recv_size(3) == b'123' return bx, by bx, by = get_bs_pair() assert bx.fileno() > 0 bx.close() assert bx.getrecvbuffer() == b'' try: bx.recv(1) except socket.error: pass else: assert False, 'expected socket.error on closed recv' assert bx.fileno() == -1 by.buffer(b'123') assert by.getsendbuffer() try: by.flush() except socket.error: assert by.getsendbuffer() == b'123' else: if sys.platform != 'win32': # Windows socketpairs are kind of bad assert False, 'expected socket.error broken pipe' try: by.shutdown(socket.SHUT_RDWR) except socket.error: # Mac sockets are already shut down at this point. See #71. if sys.platform != 'darwin': raise by.close() assert not by.getsendbuffer() try: by.send(b'123') except socket.error: pass else: assert False, 'expected socket.error on closed send' return def test_split_delim(): delim = b'\r\n' first = b'1234\r' second = b'\n5' x, y = socket.socketpair() bs = BufferedSocket(x) y.sendall(first) try: bs.recv_until(delim, timeout=0.0001) except Timeout: pass y.sendall(second) assert bs.recv_until(delim, with_delimiter=True) == b'1234\r\n' assert bs.recv_size(1) == b'5' return def test_basic_nonblocking(): delim = b'\n' # test with per-call timeout x, y = socket.socketpair() bs = BufferedSocket(x) try: bs.recv_until(delim, timeout=0) except socket.error as se: assert se.errno == errno.EWOULDBLOCK y.sendall(delim) # sending an empty message, effectively assert bs.recv_until(delim) == b'' # test with instance-level default timeout x, y = socket.socketpair() bs = BufferedSocket(x, timeout=0) try: bs.recv_until(delim) except socket.error as se: assert se.errno == errno.EWOULDBLOCK y.sendall(delim) assert bs.recv_until(delim) == b'' # test with setblocking(0) on the underlying socket x, y = socket.socketpair() x.setblocking(0) bs = BufferedSocket(x) try: bs.recv_until(delim) except socket.error as se: assert se.errno == errno.EWOULDBLOCK y.sendall(delim) assert bs.recv_until(delim) == b'' return def test_simple_buffered_socket_passthroughs(): x, y = socket.socketpair() bs = BufferedSocket(x) assert bs.getsockname() == x.getsockname() assert bs.getpeername() == x.getpeername() def test_timeout_setters_getters(): x, y = socket.socketpair() bs = BufferedSocket(x) assert bs.settimeout(1.0) is None assert bs.gettimeout() == 1.0 assert bs.setblocking(False) is None assert bs.gettimeout() == 0.0 assert bs.setblocking(True) is None assert bs.gettimeout() is None def netstring_server(server_socket): "A basic netstring server loop, supporting a few operations" running = True try: while running: clientsock, addr = server_socket.accept() client = NetstringSocket(clientsock) while 1: request = client.read_ns() if request == b'close': clientsock.close() break elif request == b'shutdown': running = False break elif request == b'reply4k': client.write_ns(b'a' * 4096) elif request == b'ping': client.write_ns(b'pong') elif request == b'reply128k': client.setmaxsize(128 * 1024) client.write_ns(b'huge' * 32 * 1024) # 128kb client.setmaxsize(32768) # back to default except Exception as e: print(u'netstring_server exiting with error: %r' % e) raise return def test_socketutils_netstring(): """A holistic feature test of BufferedSocket via the NetstringSocket wrapper. Runs """ print("running self tests") # Set up server server_socket = socket.socket() server_socket.bind(('127.0.0.1', 0)) # localhost with ephemeral port server_socket.listen(100) ip, port = server_socket.getsockname() start_server = lambda: netstring_server(server_socket) threading.Thread(target=start_server).start() # set up client def client_connect(): clientsock = socket.create_connection((ip, port)) client = NetstringSocket(clientsock) return client # connect, ping-pong client = client_connect() client.write_ns(b'ping') assert client.read_ns() == b'pong' s = time.time() for i in range(1000): client.write_ns(b'ping') assert client.read_ns() == b'pong' dur = time.time() - s print("netstring ping-pong latency", dur, "ms") s = time.time() for i in range(1000): client.write_ns(b'ping') resps = [] for i in range(1000): resps.append(client.read_ns()) e = time.time() assert all([r == b'pong' for r in resps]) assert client.bsock.getrecvbuffer() == b'' dur = e - s print("netstring pipelined ping-pong latency", dur, "ms") # tell the server to close the socket and then try a failure case client.write_ns(b'close') try: client.read_ns() raise Exception('read from closed socket') except ConnectionClosed: print("raised ConnectionClosed correctly") # test big messages client = client_connect() client.setmaxsize(128 * 1024) client.write_ns(b'reply128k') res = client.read_ns() assert len(res) == (128 * 1024) client.write_ns(b'close') # test that read timeouts work client = client_connect() client.settimeout(0.1) try: client.read_ns() raise Exception('did not timeout') except Timeout: print("read_ns raised timeout correctly") client.write_ns(b'close') # test that netstring max sizes work client = client_connect() client.setmaxsize(2048) client.write_ns(b'reply4k') try: client.read_ns() raise Exception('read more than maxsize') except NetstringMessageTooLong: print("raised MessageTooLong correctly") try: client.bsock.recv_until(b'b', maxsize=4096) raise Exception('recv_until did not raise MessageTooLong') except MessageTooLong: print("raised MessageTooLong correctly") assert client.bsock.recv_size(4097) == b'a' * 4096 + b',' print('correctly maintained buffer after exception raised') # test BufferedSocket read timeouts with recv_until and recv_size client.bsock.settimeout(0.01) try: client.bsock.recv_until(b'a') raise Exception('recv_until did not raise Timeout') except Timeout: print('recv_until correctly raised Timeout') try: client.bsock.recv_size(1) raise Exception('recv_size did not raise Timeout') except Timeout: print('recv_size correctly raised Timeout') client.write_ns(b'shutdown') print("all passed") boltons-16.4.1/tests/test_tbutils_parsed_exc.py0000644000175000017500000000367112727357214020044 0ustar hlehle from boltons.tbutils import ParsedException def test_parsed_exc_basic(): _tb_str = u"""\ Traceback (most recent call last): File "example.py", line 2, in plarp NameError: name 'plarp' is not defined""" parsed_tb = ParsedException.from_string(_tb_str) print(parsed_tb) assert parsed_tb.exc_type == 'NameError' assert parsed_tb.exc_msg == "name 'plarp' is not defined" assert parsed_tb.frames == [{'source_line': u'plarp', 'filepath': u'example.py', 'lineno': u'2', 'funcname': u''}] assert parsed_tb.to_string() == _tb_str def test_parsed_exc_nosrcline(): """just making sure that everything can be parsed even if there is a line without source and also if the exception has no message""" _tb_str = u"""\ Traceback (most recent call last): File "/home/mahmoud/virtualenvs/chert/bin/chert", line 9, in load_entry_point('chert==0.2.1.dev0', 'console_scripts', 'chert')() File "/home/mahmoud/projects/chert/chert/core.py", line 1281, in main ch.process() File "/home/mahmoud/projects/chert/chert/core.py", line 741, in process self.load() File "", line 2, in load File "/home/mahmoud/projects/lithoxyl/lithoxyl/logger.py", line 291, in logged_func return func_to_log(*a, **kw) File "/home/mahmoud/projects/chert/chert/core.py", line 775, in load raise RuntimeError RuntimeError""" parsed_tb = ParsedException.from_string(_tb_str) assert parsed_tb.exc_type == 'RuntimeError' assert parsed_tb.exc_msg == '' assert len(parsed_tb.frames) == 6 assert parsed_tb.frames[3] == {'source_line': u'', 'filepath': u'', 'lineno': u'2', 'funcname': u'load'} assert parsed_tb.to_string() == _tb_str boltons-16.4.1/tests/test_funcutils_fb.py0000644000175000017500000000560712727357214016645 0ustar hlehleimport pytest from boltons.funcutils import wraps, FunctionBuilder def pita_wrap(flag=False): def cedar_dec(func): @wraps(func) def cedar_wrapper(*a, **kw): return (flag, func.__name__, func(*a, **kw)) return cedar_wrapper return cedar_dec def wrappable_func(a, b): return a, b def test_wraps_basic(): @pita_wrap(flag=True) def simple_func(): "my doc string" return 'hello' assert simple_func() == (True, 'simple_func', 'hello') assert simple_func.__doc__ == "my doc string" @pita_wrap(flag=False) def less_simple_func(arg='hello'): return arg assert less_simple_func() == (False, 'less_simple_func', 'hello') assert less_simple_func(arg='bye') == (False, 'less_simple_func', 'bye') with pytest.raises(TypeError): simple_func(no_such_arg='nope') @pita_wrap(flag=False) def default_non_roundtrippable_repr(x=lambda y: y + 1): return x(1) assert default_non_roundtrippable_repr() == ( False, 'default_non_roundtrippable_repr', 2) def test_wraps_injected(): def inject_string(func): @wraps(func, injected="a") def wrapped(*args, **kwargs): return func(1, *args, **kwargs) return wrapped assert inject_string(wrappable_func)(2) == (1, 2) def inject_list(func): @wraps(func, injected=["b"]) def wrapped(a, *args, **kwargs): return func(a, 2, *args, **kwargs) return wrapped assert inject_list(wrappable_func)(1) == (1, 2) def inject_nonexistent_arg(func): @wraps(func, injected=["X"]) def wrapped(*args, **kwargs): return func(*args, **kwargs) return wrapped with pytest.raises(ValueError): inject_nonexistent_arg(wrappable_func) def test_wraps_update_dict(): def updated_dict(func): @wraps(func, update_dict=True) def wrapped(*args, **kwargs): return func(*args, **kwargs) return wrapped def f(a, b): return a, b f.something = True assert getattr(updated_dict(f), 'something') def test_wraps_unknown_args(): def fails(func): @wraps(func, foo="bar") def wrapped(*args, **kwargs): return func(*args, **kwargs) return wrapped with pytest.raises(TypeError): fails(wrappable_func) def test_FunctionBuilder_invalid_args(): with pytest.raises(TypeError): FunctionBuilder(name="fails", foo="bar") def test_FunctionBuilder_invalid_body(): with pytest.raises(SyntaxError): FunctionBuilder(name="fails", body="*").get_func() def test_FunctionBuilder_modify(): fb = FunctionBuilder('return_five', doc='returns the integer 5', body='return 5') f = fb.get_func() assert f() == 5 fb.varkw = 'kw' f_kw = fb.get_func() assert f_kw(ignored_arg='ignored_val') == 5 boltons-16.4.1/tests/test_setutils.py0000644000175000017500000000412212727357214016025 0ustar hlehle# -*- coding: utf-8 -*- from boltons.setutils import IndexedSet, _MISSING def test_indexed_set_basic(): zero2nine = IndexedSet(range(10)) five2nine = zero2nine & IndexedSet(range(5, 15)) x = IndexedSet(five2nine) x |= set([10]) assert list(zero2nine) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] assert set(zero2nine) == set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) assert list(five2nine) == [5, 6, 7, 8, 9] assert x == IndexedSet([5, 6, 7, 8, 9, 10]) assert x[-1] == 10 assert zero2nine ^ five2nine == IndexedSet([0, 1, 2, 3, 4]) assert x[:3] == IndexedSet([5, 6, 7]) assert x[2:4:-1] == IndexedSet([8, 7]) def test_indexed_set_mutate(): thou = IndexedSet(range(1000)) assert (thou.pop(), thou.pop()) == (999, 998) assert (thou.pop(499), thou.pop(499)) == (499, 500) ref = [495, 496, 497, 498, 501, 502, 503, 504, 505, 506] assert [thou[i] for i in range(495, 505)] == ref assert len(thou) == 996 while len(thou) > 600: dead_idx_len = len(thou.dead_indices) dead_idx_count = thou._dead_index_count thou.pop(0) new_dead_idx_len = len(thou.dead_indices) if new_dead_idx_len < dead_idx_len: assert dead_idx_count > 0 # 124, 109, 95 assert len(thou) == 600 assert thou._dead_index_count == 67 assert not any([thou[i] is _MISSING for i in range(len(thou))]) thou &= IndexedSet(range(500, 503)) assert thou == IndexedSet([501, 502]) return def big_popper(): # more of a benchmark than a test from os import urandom import time big_set = IndexedSet(range(100000)) rands = [ord(r) for r in urandom(len(big_set))] start_time, start_size = time.time(), len(big_set) while len(big_set) > 10000: if len(big_set) % 10000 == 0: print(len(big_set) / 10000) rand = rands.pop() big_set.pop(rand) big_set.pop(-rand) end_time, end_size = time.time(), len(big_set) print() print('popped %s items in %s seconds' % (start_size - end_size, end_time - start_time)) boltons-16.4.1/tests/test_ecoutils.py0000644000175000017500000000041712727357214016003 0ustar hlehle import sys from boltons import ecoutils def test_basic(): # basic sanity test prof = ecoutils.get_profile() assert prof['python']['bin'] == sys.executable def test_scrub(): prof = ecoutils.get_profile(scrub=True) assert prof['username'] == '-' boltons-16.4.1/tests/jsonl_test_data.txt0000644000175000017500000000004712727357214016460 0ustar hlehle{} {"1": 1} {"2": 2} {"3": 3} {"4": 4} boltons-16.4.1/tests/test_namedutils.py0000644000175000017500000000115712727357214016323 0ustar hlehle# -*- coding: utf-8 -*- try: from cPickle import loads, dumps except: from pickle import loads, dumps from boltons.namedutils import namedlist, namedtuple Point = namedtuple('Point', 'x, y', rename=True) MutablePoint = namedlist('MutablePoint', 'x, y', rename=True) def test_namedlist(): p = MutablePoint(x=10, y=20) assert p == [10, 20] p[0] = 11 assert p == [11, 20] p.x = 12 assert p == [12, 20] def test_namedlist_pickle(): p = MutablePoint(x=10, y=20) assert p == loads(dumps(p)) def test_namedtuple_pickle(): p = Point(x=10, y=20) assert p == loads(dumps(p)) boltons-16.4.1/tests/test_tbutils.py0000644000175000017500000000541112727357214015641 0ustar hlehle# -*- coding: utf-8 -*- import sys try: from cStringIO import StringIO except: from io import StringIO from boltons.tbutils import (TracebackInfo, ExceptionInfo, print_exception, fix_print_exception, ContextualCallpoint, ContextualExceptionInfo) def test_exception_info(): # test ExceptionInfo and TracebackInfo and hooks, via StringIOs builtin_exc_hook = sys.excepthook fix_print_exception() tbi_str = '' def test(): raise ValueError('yay fun') fake_stderr1 = StringIO() fake_stderr2 = StringIO() sys.stderr = fake_stderr1 try: test() except: _, _, exc_traceback = sys.exc_info() tbi = TracebackInfo.from_traceback(exc_traceback) exc_info = ExceptionInfo.from_exc_info(*sys.exc_info()) exc_info2 = ExceptionInfo.from_current() tbi_str = str(tbi) print_exception(*sys.exc_info(), file=fake_stderr2) new_exc_hook_res = fake_stderr2.getvalue() builtin_exc_hook(*sys.exc_info()) builtin_exc_hook_res = fake_stderr1.getvalue() finally: sys.stderr = sys.__stderr__ # Single frame single_frame_str = tbi.frames[-1].tb_frame_str() assert 'in test' in single_frame_str assert 'yay fun' in single_frame_str # Traceback info assert len(tbi_str.splitlines()) == 5 assert 'yay fun' in tbi_str # Full except hook output assert 'ValueError: yay fun' in new_exc_hook_res assert "ValueError('yay fun')" in new_exc_hook_res assert len(new_exc_hook_res) > len(tbi_str) assert new_exc_hook_res == builtin_exc_hook_res def test_contextual(): def func1(): return func2() def func2(): x = 5 return func3() def func3(): return ContextualCallpoint.from_current(level=2) callpoint = func1() assert callpoint.func_name == 'func2' line = str(callpoint.line) assert line.startswith(' ') assert line.strip() == 'return func3()' assert 'func2' in repr(callpoint) def func_a(): a = 1 raise Exception('func_a exception') def func_b(): b = 2 return func_a() def func_c(): c = 3 return func_b() try: func_c() except Exception as e: ctx_ei = ContextualExceptionInfo.from_current() ctx_ei_str = ctx_ei.get_formatted() ctx_ei_lines = ctx_ei_str.splitlines() assert ctx_ei_lines[-1] == 'Exception: func_a exception' assert ctx_ei_lines[0] == 'Traceback (most recent call last):' assert len(ctx_ei_lines) == 10 assert "Exception('func_a exception')" in ctx_ei_str assert ctx_ei.tb_info.frames[2].local_reprs['b'] == '2' boltons-16.4.1/tests/conftest.py0000644000175000017500000000065212727357214014743 0ustar hlehleimport sys import re _VERSION_MARKER = re.compile('_py(?P\d)') def pytest_ignore_collect(path, config): """ Ignore tests that end with _pyX, where X does not equal this interpreter's major version. """ filename = path.basename modulename = filename.split('.', 1)[0] match = _VERSION_MARKER.search(modulename) return match and not int(match.group('version')) == sys.version_info[0] boltons-16.4.1/tests/test_funcutils.py0000644000175000017500000000203712727357214016170 0ustar hlehle# -*- coding: utf-8 -*- from boltons.funcutils import (copy_function, InstancePartial, CachedInstancePartial) class Greeter(object): def __init__(self, greeting): self.greeting = greeting def greet(self, excitement='.'): return self.greeting.capitalize() + excitement partial_greet = InstancePartial(greet, excitement='!') cached_partial_greet = CachedInstancePartial(greet, excitement='...') def native_greet(self): return self.greet(';') class SubGreeter(Greeter): pass def test_partials(): g = SubGreeter('hello') assert g.greet() == 'Hello.' assert g.native_greet() == 'Hello;' assert g.partial_greet() == 'Hello!' assert g.cached_partial_greet() == 'Hello...' assert CachedInstancePartial(g.greet, excitement='s')() == 'Hellos' def test_copy_function(): def callee(): return 1 callee_copy = copy_function(callee) assert callee is not callee_copy assert callee() == callee_copy() boltons-16.4.1/tests/test_dictutils.py0000644000175000017500000001317012727357214016160 0ustar hlehle# -*- coding: utf-8 -*- from boltons.dictutils import OMD _ITEMSETS = [[], [('a', 1), ('b', 2), ('c', 3)], [('A', 'One'), ('A', 'One'), ('A', 'One')], [('Z', -1), ('Y', -2), ('Y', -2)], [('a', 1), ('b', 2), ('a', 3), ('c', 4)]] def test_dict_init(): d = dict(_ITEMSETS[1]) omd = OMD(d) assert omd['a'] == 1 assert omd['b'] == 2 assert omd['c'] == 3 assert len(omd) == 3 assert omd.getlist('a') == [1] assert omd == d def test_todict(): omd = OMD(_ITEMSETS[2]) assert len(omd) == 1 assert omd['A'] == 'One' d = dict(omd) assert len(d) == 1 assert d['A'] == ['One', 'One', 'One'] flat = omd.todict() assert flat['A'] == 'One' for itemset in _ITEMSETS: omd = OMD(itemset) d = dict(itemset) flat = omd.todict() assert flat == d def test_eq(): omd = OMD(_ITEMSETS[3]) assert omd == omd assert not (omd != omd) omd2 = OMD(_ITEMSETS[3]) assert omd == omd2 assert omd2 == omd assert not (omd != omd2) d = dict(_ITEMSETS[3]) assert d == omd omd3 = OMD(d) assert omd != omd3 def test_copy(): for itemset in _ITEMSETS: omd = OMD(itemset) omd_c = omd.copy() assert omd == omd_c if omd_c: omd_c.pop(itemset[0][0]) assert omd != omd_c return def test_clear(): for itemset in _ITEMSETS: omd = OMD(itemset) omd.clear() assert len(omd) == 0 assert not omd omd.clear() assert not omd omd['a'] = 22 assert omd omd.clear() assert not omd def test_types(): import collections omd = OMD() assert isinstance(omd, dict) assert isinstance(omd, collections.MutableMapping) def test_multi_correctness(): size = 100 redun = 5 _rng = range(size) _rng_redun = list(range(size//redun)) * redun _pairs = zip(_rng_redun, _rng) omd = OMD(_pairs) for multi in (True, False): vals = [x[1] for x in omd.iteritems(multi=multi)] strictly_ascending = all([x < y for x, y in zip(vals, vals[1:])]) assert strictly_ascending return def test_kv_consistency(): for itemset in _ITEMSETS: omd = OMD(itemset) for multi in (True, False): items = omd.items(multi=multi) keys = omd.keys(multi=multi) values = omd.values(multi=multi) assert keys == [x[0] for x in items] assert values == [x[1] for x in items] return def test_update_basic(): omd = OMD(_ITEMSETS[1]) omd2 = OMD({'a': 10}) omd.update(omd2) assert omd['a'] == 10 assert omd.getlist('a') == [10] omd2_c = omd2.copy() omd2_c.pop('a') assert omd2 != omd2_c def test_update(): for first, second in zip(_ITEMSETS, _ITEMSETS[1:]): omd1 = OMD(first) omd2 = OMD(second) ref1 = dict(first) ref2 = dict(second) omd1.update(omd2) ref1.update(ref2) assert omd1.todict() == ref1 omd1_repr = repr(omd1) omd1.update(omd1) assert omd1_repr == repr(omd1) def test_update_extend(): for first, second in zip(_ITEMSETS, _ITEMSETS[1:] + [[]]): omd1 = OMD(first) omd2 = OMD(second) ref = dict(first) orig_keys = set(omd1) ref.update(second) omd1.update_extend(omd2) for k in omd2: assert len(omd1.getlist(k)) >= len(omd2.getlist(k)) assert omd1.todict() == ref assert orig_keys <= set(omd1) def test_invert(): for items in _ITEMSETS: omd = OMD(items) iomd = omd.inverted() # first, test all items made the jump assert len(omd.items(multi=True)) == len(iomd.items(multi=True)) for val in omd.values(): assert val in iomd # all values present as keys def test_poplast(): for items in _ITEMSETS[1:]: omd = OMD(items) assert omd.poplast() == items[-1][-1] def test_pop(): omd = OMD() omd.add('even', 0) omd.add('odd', 1) omd.add('even', 2) assert omd.pop('odd') == 1 assert omd.pop('odd', 99) == 99 try: omd.pop('odd') import pdb;pdb.set_trace() assert False except KeyError: pass assert len(omd) == 1 assert len(omd.items(multi=True)) == 2 def test_pop_all(): omd = OMD() omd.add('even', 0) omd.add('odd', 1) omd.add('even', 2) assert omd.popall('odd') == [1] assert len(omd) == 1 try: omd.popall('odd') assert False except KeyError: pass assert omd.popall('odd', None) is None assert omd.popall('even') == [0, 2] assert len(omd) == 0 assert omd.popall('nope', None) is None def test_reversed(): try: from collections import OrderedDict except: # skip on python 2.6 return for items in _ITEMSETS: omd = OMD(items) od = OrderedDict(items) for ik, ok in zip(reversed(od), reversed(omd)): assert ik == ok r100 = range(100) omd = OMD(zip(r100, r100)) for i in r100: omd.add(i, i) r100 = list(reversed(r100)) assert list(reversed(omd)) == r100 omd = OMD() assert list(reversed(omd)) == list(reversed(omd.keys())) for i in range(20): for j in range(i): omd.add(i, i) assert list(reversed(omd)) == list(reversed(omd.keys())) def test_setdefault(): omd = OMD() empty_list = [] x = omd.setdefault('1', empty_list) assert x is empty_list y = omd.setdefault('2') assert y is None assert omd.setdefault('1', None) is empty_list boltons-16.4.1/tests/test_iterutils.py0000644000175000017500000003213012727357214016175 0ustar hlehle import pytest from boltons.dictutils import OMD from boltons.iterutils import (first, remap, default_enter, default_exit, get_path) from boltons.namedutils import namedtuple isbool = lambda x: isinstance(x, bool) isint = lambda x: isinstance(x, int) odd = lambda x: isint(x) and x % 2 != 0 even = lambda x: isint(x) and x % 2 == 0 is_meaning_of_life = lambda x: x == 42 class TestFirst(object): def test_empty_iterables(self): """ Empty iterables return None. """ s = set() l = [] assert first(s) is None assert first(l) is None def test_default_value(self): """ Empty iterables + a default value return the default value. """ s = set() l = [] assert first(s, default=42) == 42 assert first(l, default=3.14) == 3.14 l = [0, False, []] assert first(l, default=3.14) == 3.14 def test_selection(self): """ Success cases with and without a key function. """ l = [(), 0, False, 3, []] assert first(l, default=42) == 3 assert first(l, key=isint) == 0 assert first(l, key=isbool) is False assert first(l, key=odd) == 3 assert first(l, key=even) == 0 assert first(l, key=is_meaning_of_life) is None class TestRemap(object): # TODO: test namedtuples and other immutable containers def test_basic_clone(self): orig = {"a": "b", "c": [1, 2]} assert orig == remap(orig) orig2 = [{1: 2}, {"a": "b", "c": [1, 2, {"cat": "dog"}]}] assert orig2 == remap(orig2) def test_empty(self): assert [] == remap([]) assert {} == remap({}) assert set() == remap(set()) def test_unremappable(self): obj = object() with pytest.raises(TypeError): remap(obj) def test_basic_upper(self): orig = {'a': 1, 'b': object(), 'c': {'d': set()}} remapped = remap(orig, lambda p, k, v: (k.upper(), v)) assert orig['a'] == remapped['A'] assert orig['b'] == remapped['B'] assert orig['c']['d'] == remapped['C']['D'] def test_item_drop(self): orig = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] even_items = remap(orig, lambda p, k, v: not (v % 2)) assert even_items == [0, 2, 4, 6, 8] def test_noncallables(self): with pytest.raises(TypeError): remap([], visit='test') with pytest.raises(TypeError): remap([], enter='test') with pytest.raises(TypeError): remap([], exit='test') def test_sub_selfref(self): coll = [0, 1, 2, 3] sub = [] sub.append(sub) coll.append(sub) with pytest.raises(RuntimeError): # if equal, should recurse infinitely assert coll == remap(coll) def test_root_selfref(self): selfref = [0, 1, 2, 3] selfref.append(selfref) with pytest.raises(RuntimeError): assert selfref == remap(selfref) selfref2 = {} selfref2['self'] = selfref2 with pytest.raises(RuntimeError): assert selfref2 == remap(selfref2) def test_duperef(self): val = ['hello'] duperef = [val, val] remapped = remap(duperef) assert remapped[0] is remapped[1] assert remapped[0] is not duperef[0] def test_namedtuple(self): """TODO: this fails right now because namedtuples' __new__ is overridden to accept arguments. remap's default_enter tries to create an empty namedtuple and gets a TypeError. Could make it so that immutable types actually don't create a blank new parent and instead use the old_parent as a placeholder, creating a new one at exit-time from the value's __class__ (how default_exit works now). But even then it would have to *args in the values, as namedtuple constructors don't take an iterable. """ Point = namedtuple('Point', 'x y') point_map = {'origin': [Point(0, 0)]} with pytest.raises(TypeError): remapped = remap(point_map) assert isinstance(remapped['origin'][0], Point) def test_path(self): path_map = {} # test visit's path target_str = 'test' orig = [[[target_str]]] ref_path = (0, 0, 0) def visit(path, key, value): if value is target_str: path_map['target_str'] = path + (key,) return key, value remapped = remap(orig, visit=visit) assert remapped == orig assert path_map['target_str'] == ref_path # test enter's path target_obj = object() orig = {'a': {'b': {'c': {'d': ['e', target_obj, 'f']}}}} ref_path = ('a', 'b', 'c', 'd', 1) def enter(path, key, value): if value is target_obj: path_map['target_obj'] = path + (key,) return default_enter(path, key, value) remapped = remap(orig, enter=enter) assert remapped == orig assert path_map['target_obj'] == ref_path # test exit's path target_set = frozenset([1, 7, 3, 8]) orig = [0, 1, 2, [3, 4, [5, target_set]]] ref_path = (3, 2, 1) def exit(path, key, old_parent, new_parent, new_items): if old_parent is target_set: path_map['target_set'] = path + (key,) return default_exit(path, key, old_parent, new_parent, new_items) remapped = remap(orig, exit=exit) assert remapped == orig assert path_map['target_set'] == ref_path def test_reraise_visit(self): root = {'A': 'b', 1: 2} key_to_lower = lambda p, k, v: (k.lower(), v) with pytest.raises(AttributeError): remap(root, key_to_lower) remapped = remap(root, key_to_lower, reraise_visit=False) assert remapped['a'] == 'b' assert remapped[1] == 2 def test_drop_nones(self): orig = {'a': 1, 'b': None, 'c': [3, None, 4, None]} ref = {'a': 1, 'c': [3, 4]} drop_none = lambda p, k, v: v is not None remapped = remap(orig, visit=drop_none) assert remapped == ref orig = [None] * 100 remapped = remap(orig, drop_none) assert not remapped def test_dict_to_omd(self): def enter(path, key, value): if isinstance(value, dict): return OMD(), sorted(value.items()) return default_enter(path, key, value) orig = [{'title': 'Wild Palms', 'ratings': {1: 1, 2: 3, 3: 5, 4: 6, 5: 3}}, {'title': 'Twin Peaks', 'ratings': {1: 3, 2: 2, 3: 8, 4: 12, 5: 15}}] remapped = remap(orig, enter=enter) assert remapped == orig assert isinstance(remapped[0], OMD) assert isinstance(remapped[0]['ratings'], OMD) assert isinstance(remapped[1], OMD) assert isinstance(remapped[1]['ratings'], OMD) def test_sort_all_lists(self): def exit(path, key, old_parent, new_parent, new_items): # NB: in this case, I'd normally use *a, **kw ret = default_exit(path, key, old_parent, new_parent, new_items) if isinstance(ret, list): ret.sort() return ret # NB: Airplane model numbers (Boeing and Airbus) orig = [[[7, 0, 7], [7, 2, 7], [7, 7, 7], [7, 3, 7]], [[3, 8, 0], [3, 2, 0], [3, 1, 9], [3, 5, 0]]] ref = [[[0, 2, 3], [0, 3, 5], [0, 3, 8], [1, 3, 9]], [[0, 7, 7], [2, 7, 7], [3, 7, 7], [7, 7, 7]]] remapped = remap(orig, exit=exit) assert remapped == ref def test_collector_pattern(self): all_interests = set() def enter(path, key, value): try: all_interests.update(value['interests']) except: pass return default_enter(path, key, value) orig = [{'name': 'Kate', 'interests': ['theater', 'manga'], 'dads': [{'name': 'Chris', 'interests': ['biking', 'python']}]}, {'name': 'Avery', 'interests': ['museums', 'pears'], 'dads': [{'name': 'Kurt', 'interests': ['python', 'recursion']}]}] ref = set(['python', 'recursion', 'biking', 'museums', 'pears', 'theater', 'manga']) remap(orig, enter=enter) assert all_interests == ref def test_add_length(self): def exit(path, key, old_parent, new_parent, new_items): ret = default_exit(path, key, old_parent, new_parent, new_items) try: ret['review_length'] = len(ret['review']) except: pass return ret orig = {'Star Trek': {'TNG': {'stars': 10, 'review': "Episodic AND deep. <3 Data."}, 'DS9': {'stars': 8.5, 'review': "Like TNG, but with a story and no Data."}, 'ENT': {'stars': None, 'review': "Can't review what you can't watch."}}, 'Babylon 5': {'stars': 6, 'review': "Sophomoric, like a bitter laugh."}, 'Dr. Who': {'stars': None, 'review': "800 episodes is too many to review."}} remapped = remap(orig, exit=exit) assert (remapped['Star Trek']['TNG']['review_length'] < remapped['Star Trek']['DS9']['review_length']) def test_prepop(self): """Demonstrating normalization and ID addition through prepopulating the objects wth an enter callback. """ base_obj = {'name': None, 'rank': None, 'id': 1} def enter(path, key, value): new_parent, new_items = default_enter(path, key, value) try: new_parent.update(base_obj) base_obj['id'] += 1 except: pass return new_parent, new_items orig = [{'name': 'Firefox', 'rank': 1}, {'name': 'Chrome', 'rank': 2}, {'name': 'IE'}] ref = [{'name': 'Firefox', 'rank': 1, 'id': 1}, {'name': 'Chrome', 'rank': 2, 'id': 2}, {'name': 'IE', 'rank': None, 'id': 3}] remapped = remap(orig, enter=enter) assert remapped == ref class TestGetPath(object): def test_depth_one(self): root = ['test'] assert get_path(root, (0,)) == 'test' assert get_path(root, '0') == 'test' root = {'key': 'value'} assert get_path(root, ('key',)) == 'value' assert get_path(root, 'key') == 'value' def test_depth_two(self): root = {'key': ['test']} assert get_path(root, ('key', 0)) == 'test' assert get_path(root, 'key.0') == 'test' def test_backoff_basic(): from boltons.iterutils import backoff assert backoff(1, 16) == [1.0, 2.0, 4.0, 8.0, 16.0] assert backoff(1, 1) == [1.0] assert backoff(2, 15) == [2.0, 4.0, 8.0, 15.0] def test_backoff_repeat(): from boltons.iterutils import backoff_iter fives = [] for val in backoff_iter(5, 5, count='repeat'): fives.append(val) if len(fives) >= 1000: break assert fives == [5] * 1000 def test_backoff_zero_start(): from boltons.iterutils import backoff assert backoff(0, 16) == [0.0, 1.0, 2.0, 4.0, 8.0, 16.0] assert backoff(0, 15) == [0.0, 1.0, 2.0, 4.0, 8.0, 15.0] slow_backoff = [round(x, 2) for x in backoff(0, 2.9, factor=1.2)] assert slow_backoff == [0.0, 1.0, 1.2, 1.44, 1.73, 2.07, 2.49, 2.9] def test_backoff_validation(): from boltons.iterutils import backoff with pytest.raises(ValueError): backoff(8, 2) with pytest.raises(ValueError): backoff(1, 0) with pytest.raises(ValueError): backoff(-1, 10) with pytest.raises(ValueError): backoff(2, 8, factor=0) with pytest.raises(ValueError): backoff(2, 8, jitter=20) def test_backoff_jitter(): from boltons.iterutils import backoff start, stop = 1, 256 unjittered = backoff(start, stop) jittered = backoff(start, stop, jitter=True) assert len(unjittered) == len(jittered) assert [u >= j for u, j in zip(unjittered, jittered)] neg_jittered = backoff(start, stop, jitter=-0.01) assert len(unjittered) == len(neg_jittered) assert [u <= j for u, j in zip(unjittered, neg_jittered)] o_jittered = backoff(start, stop, jitter=-0.0) assert len(unjittered) == len(o_jittered) assert [u == j for u, j in zip(unjittered, o_jittered)] nonconst_jittered = backoff(stop, stop, count=5, jitter=True) assert len(nonconst_jittered) == 5 # no two should be equal realistically assert len(set(nonconst_jittered)) == 5 boltons-16.4.1/tests/test_queueutils.py0000644000175000017500000000113712727357214016361 0ustar hlehle# -*- coding: utf-8 -*- from boltons.queueutils import SortedPriorityQueue, HeapPriorityQueue def _test_priority_queue(queue_type): pq = queue_type() func = lambda x: x pq.add(func) pq.remove(func) pq.add(func) pq.add(func) assert len(pq) == 1 assert func == pq.pop() assert len(pq) == 0 try: pq.pop() except IndexError: pass else: assert False, 'priority queue should be empty' return def test_heap_queue(): _test_priority_queue(HeapPriorityQueue) def test_sorted_queue(): _test_priority_queue(SortedPriorityQueue) boltons-16.4.1/tests/test_statsutils_histogram.py0000644000175000017500000000730712727357214020455 0ustar hlehle from boltons.statsutils import Stats # [round(random.normalvariate(10, 3), 3) for i in range(100)] NORM_DATA = [12.975, 8.341, 12.27, 12.799, 15.443, 6.739, 10.572, 14.863, 3.723, 9.825, 7.716, 12.218, 11.641, 9.02, 13.037, 11.175, 13.156, 11.706, 8.184, 13.306, 9.845, 11.665, 14.298, 12.021, 8.419, 10.209, 10.698, 6.559, 10.346, 9.895, 11.742, 13.391, 10.587, 6.639, 10.23, 8.841, 10.511, 6.033, 5.767, 8.482, 9.517, 9.039, 11.111, 13.845, 4.331, 5.323, 14.486, 14.875, 10.005, 6.367, 12.18, 11.69, 13.97, 4.14, 7.979, 11.114, 4.126, 10.028, 9.295, 10.078, 14.615, 7.055, 7.641, 9.037, 9.933, 10.077, 14.174, 14.645, 10.398, 10.238, 9.067, 4.841, 13.159, 15.829, 8.464, 7.47, 11.858, 9.885, 11.978, 5.418, 12.19, 8.206, 10.755, 6.455, 10.019, 11.594, 9.082, 10.245, 12.321, 8.508, 9.711, 5.5, 15.001, 9.922, 7.864, 7.794, 10.546, 9.203, 8.798, 9.853] SIMPLE_RANGE_DATA = range(110) LAYER_RANGE_DATA = (list(range(100)) + list(range(20, 80)) + list(range(40, 60))) EMPTY_DATA = [] ALL_DATASETS = [EMPTY_DATA, LAYER_RANGE_DATA, SIMPLE_RANGE_DATA, NORM_DATA] def test_check_sum(): for data in ALL_DATASETS: for bin_size in [0, 1, 10, 99]: # bin_size=0 tests freedman stats = Stats(data) hist_counts = stats.get_histogram_counts() hist_counts_sum = sum([c for _, c in hist_counts]) assert len(data) == hist_counts_sum if not data: continue assert min(data) >= hist_counts[0][0] assert max(data) >= hist_counts[-1][0] return def test_norm_regression(): stats = Stats(NORM_DATA) assert stats.format_histogram(width=80) == NORM_DATA_FREEDMAN_OUTPUT assert stats.format_histogram(10, width=80) == NORM_DATA_TEN_BIN_OUTPUT subpar_bin_out = stats.format_histogram([12.0], width=80) assert subpar_bin_out == NORM_DATA_SINGLE_SUBPAR_BIN_OUTPUT format_bin_out = stats.format_histogram(5, width=80, format_bin=lambda b: '%sms' % b) assert format_bin_out == NORM_DATA_FORMAT_BIN_OUTPUT NORM_DATA_FREEDMAN_OUTPUT = """\ 3.7: 5 ################ 5.2: 10 ################################ 6.8: 9 ############################# 8.3: 21 ################################################################### 9.9: 22 ###################################################################### 11.4: 16 ################################################### 13.0: 10 ################################ 14.5: 7 ######################""" NORM_DATA_TEN_BIN_OUTPUT = """\ 3.7: 5 ############## 4.9: 5 ############## 6.1: 6 ################# 7.3: 12 ################################## 8.5: 11 ############################### 9.7: 25 ###################################################################### 10.9: 12 ################################## 12.1: 12 ################################## 13.4: 5 ############## 14.6: 7 ####################""" # make sure the minimum gets added NORM_DATA_SINGLE_SUBPAR_BIN_OUTPUT = """\ 3.7: 75 ###################################################################### 12.0: 25 #######################""" NORM_DATA_FORMAT_BIN_OUTPUT = """\ 3.7ms: 10 ################### 6.1ms: 18 ################################## 8.5ms: 36 #################################################################### 10.9ms: 24 ############################################# 13.4ms: 12 #######################""" def main(): print(Stats(NORM_DATA).format_histogram(10)) if __name__ == '__main__': main() boltons-16.4.1/tests/newlines_test_data.txt0000644000175000017500000000004412727357214017154 0ustar hlehlea b c d e f g hijklmnop qrstuv wxyz boltons-16.4.1/tests/test_timeutils.py0000644000175000017500000000363612727357214016201 0ustar hlehle from datetime import timedelta, date from boltons.timeutils import total_seconds, daterange def test_float_total_seconds(): """Check for floating point precision loss per http://bugs.python.org/issue8644 and tests in the corresponding diff, spurred by https://github.com/mahmoud/boltons/pull/13#issuecomment-93835612 . The funny thing is that floating point precision loss is definitely happening. With or without true division, in Python 2.7's native timedelta.total_seconds() as well as dateutils.total_seconds. The constants in the tests below are from manual tests on Python 2.7.6 final. 2.6 does vary slightly, but it might just be a repr change. With these tests in mind, I'm not sure why the Python issue got created in the first place. """ assert total_seconds(timedelta(microseconds=1)) == 1e-06 assert total_seconds(timedelta(microseconds=-1)) == -1e-06 assert total_seconds(timedelta(microseconds=-2)) == -2e-06 assert total_seconds(timedelta(days=2 ** 29, microseconds=1)) == 46385646796800.0 assert total_seconds(timedelta(seconds=123456.789012)) == 123456.789012 assert total_seconds(timedelta(seconds=-123456.789012)) == -123456.789012 def test_daterange_years(): new_year = date(2017, 1, 1) bit_rollover = date(2038, 1, 19) new_years_remaining = daterange(new_year, bit_rollover, step=(1, 0, 0)) assert len(list(new_years_remaining)) == 22 y2025 = date(2025, 1, 1) bakers_years_til_2025 = list(daterange(new_year, y2025, step=(1, 1, 0))) assert len(bakers_years_til_2025) == 8 assert bakers_years_til_2025[-1] == date(2024, 8, 1) assert bakers_years_til_2025[-1] == date(2024, 8, 1) years_from_2025 = list(daterange(y2025, new_year, step=(-1, 0, 0), inclusive=True)) assert years_from_2025[0] == date(2025, 1, 1) assert years_from_2025[-1] == date(2017, 1, 1) boltons-16.4.1/tests/test_jsonutils.py0000644000175000017500000000167712727357214016217 0ustar hlehle# -*- coding: utf-8 -*- import os from boltons.jsonutils import (JSONLIterator, DEFAULT_BLOCKSIZE, reverse_iter_lines) CUR_PATH = os.path.dirname(os.path.abspath(__file__)) NEWLINES_DATA_PATH = CUR_PATH + '/newlines_test_data.txt' JSONL_DATA_PATH = CUR_PATH + '/jsonl_test_data.txt' def _test_reverse_iter_lines(filename, blocksize=DEFAULT_BLOCKSIZE): fo = open(filename) reference = fo.read() fo.seek(0, os.SEEK_END) rev_lines = list(reverse_iter_lines(fo, blocksize)) assert '\n'.join(rev_lines[::-1]) == reference def test_reverse_iter_lines(): for blocksize in (1, 4, 11, 4096): _test_reverse_iter_lines(NEWLINES_DATA_PATH, blocksize) def test_jsonl_iterator(): ref = [{u'4': 4}, {u'3': 3}, {u'2': 2}, {u'1': 1}, {}] jsonl_iter = JSONLIterator(open(JSONL_DATA_PATH), reverse=True) jsonl_list = list(jsonl_iter) assert jsonl_list == ref boltons-16.4.1/tests/test_fileutils.py0000644000175000017500000000076612727357214016163 0ustar hlehle# -*- coding: utf-8 -*- from boltons.fileutils import FilePerms def test_fileperms(): up = FilePerms() up.other = '' up.user = 'xrw' up.group = 'rrrwx' try: up.other = 'nope' except ValueError: # correctly raised ValueError on invalid chars pass assert repr(up) == "FilePerms(user='rwx', group='rwx', other='')" assert up.user == 'rwx' assert oct(int(up)).endswith('770') # 0770 on py2 and 0o770 on py3 assert int(FilePerms()) == 0 boltons-16.4.1/tests/test_mathutils.py0000644000175000017500000000312212727357214016162 0ustar hlehle from pytest import raises from boltons.mathutils import ceil, floor OPTIONS = [1618, 1378, 166, 1521, 2347, 2016, 879, 2123, 269.3, 1230, 66, 425.2, 250, 2399, 2314, 439, 247, 208, 379, 1861] OPTIONS_SORTED = sorted(OPTIONS) OUT_OF_RANGE_LOWER = 60 OUT_OF_RANGE_UPPER = 2500 VALID_LOWER = 247 VALID_UPPER = 2314 VALID_BETWEEN = 248.5 def test_ceil_basic(): assert ceil(VALID_LOWER, OPTIONS) == VALID_LOWER assert ceil(VALID_UPPER, OPTIONS) == VALID_UPPER assert ceil(VALID_BETWEEN, OPTIONS) == 250 def test_ceil_sorted(): assert ceil(VALID_LOWER, OPTIONS) == ceil(VALID_LOWER, OPTIONS_SORTED) assert ceil(VALID_UPPER, OPTIONS) == ceil(VALID_UPPER, OPTIONS_SORTED) assert ceil(VALID_BETWEEN, OPTIONS) == ceil(VALID_BETWEEN, OPTIONS_SORTED) def test_ceil_oor_lower(): assert min(OPTIONS) == ceil(OUT_OF_RANGE_LOWER, OPTIONS) def test_ceil_oor_upper(): with raises(ValueError): ceil(OUT_OF_RANGE_UPPER, OPTIONS) def test_floor_basic(): assert floor(VALID_LOWER, OPTIONS) == VALID_LOWER assert floor(VALID_UPPER, OPTIONS) == VALID_UPPER assert floor(VALID_LOWER, OPTIONS) == 247 def test_floor_sorted(): assert floor(VALID_LOWER, OPTIONS) == floor(VALID_LOWER, OPTIONS_SORTED) assert floor(VALID_UPPER, OPTIONS) == floor(VALID_UPPER, OPTIONS_SORTED) assert floor(VALID_BETWEEN, OPTIONS) == floor(VALID_BETWEEN, OPTIONS_SORTED) def test_floor_oor_upper(): assert max(OPTIONS) == floor(OUT_OF_RANGE_UPPER, OPTIONS) def test_floor_oor_lower(): with raises(ValueError): floor(OUT_OF_RANGE_LOWER, OPTIONS) boltons-16.4.1/tests/__init__.py0000644000175000017500000000000012727357214014640 0ustar hlehleboltons-16.4.1/tests/test_strutils.py0000644000175000017500000000314112727357214016042 0ustar hlehle# -*- coding: utf-8 -*- import uuid from boltons import strutils def test_asciify(): ref = u'Beyoncé' b = strutils.asciify(ref) assert len(b) == len(b) assert b[-1:].decode('ascii') == 'e' def test_indent(): to_indent = '\nabc\ndef\n\nxyz\n' ref = '\n abc\n def\n\n xyz\n' assert strutils.indent(to_indent, ' ') == ref def test_is_uuid(): assert strutils.is_uuid(uuid.uuid4()) == True assert strutils.is_uuid(uuid.uuid4(), version=1) == False assert strutils.is_uuid(str(uuid.uuid4())) == True assert strutils.is_uuid(str(uuid.uuid4()), version=1) == False assert strutils.is_uuid(set('garbage')) == False def test_parse_int_list(): assert strutils.parse_int_list("1,3,5-8,10-11,15") == [1, 3, 5, 6, 7, 8, 10, 11, 15] assert strutils.parse_int_list("1,3,5-8,10-11,15,") == [1, 3, 5, 6, 7, 8, 10, 11, 15] assert strutils.parse_int_list(",1,3,5-8,10-11,15") == [1, 3, 5, 6, 7, 8, 10, 11, 15] assert strutils.parse_int_list(" 1, 3 ,5-8,10-11,15 ") == [1, 3, 5, 6, 7, 8, 10, 11, 15] assert strutils.parse_int_list("3,1,5-8,10-11,15") == [1, 3, 5, 6, 7, 8, 10, 11, 15] assert strutils.parse_int_list("5-8") == [5, 6, 7, 8] assert strutils.parse_int_list("8-5") == [5, 6, 7, 8] def test_format_int_list(): assert strutils.format_int_list([1, 3, 5, 6, 7, 8, 10, 11, 15]) == '1,3,5-8,10-11,15' assert strutils.format_int_list([5, 6, 7, 8]) == '5-8' assert strutils.format_int_list([1, 3, 5, 6, 7, 8, 10, 11, 15], delim_space=True) == '1, 3, 5-8, 10-11, 15' assert strutils.format_int_list([5, 6, 7, 8], delim_space=True) == '5-8' boltons-16.4.1/tests/test_debugutils_trace.py0000644000175000017500000000331712727357214017503 0ustar hlehle from collections import namedtuple from pytest import raises from boltons.debugutils import wrap_trace def test_trace_dict(): target = {} wrapped = wrap_trace(target) assert target is not wrapped assert isinstance(wrapped, dict) wrapped['a'] = 'A' assert target['a'] == 'A' assert len(wrapped) == len(target) wrapped.pop('a') assert 'a' not in target with raises(AttributeError): wrapped.nonexistent_attr = 'nope' return def test_trace_bytes(): target = u'Hello'.encode('ascii') wrapped = wrap_trace(target) assert target is not wrapped assert isinstance(wrapped, bytes) assert len(wrapped) == len(target) assert wrapped.decode('utf-8') == u'Hello' assert wrapped.lower() == target.lower() def test_trace_exc(): class TestException(Exception): pass target = TestException('exceptions can be a good thing') wrapped = wrap_trace(target) try: raise wrapped except TestException as te: assert te.args == target.args def test_trace_which(): class Config(object): def __init__(self, value): self.value = value config = Config('first') wrapped = wrap_trace(config, which='__setattr__') wrapped.value = 'second' assert config.value == 'second' def test_trace_namedtuple(): TargetType = namedtuple('TargetType', 'x y z') target = TargetType(1, 2, 3) wrapped = wrap_trace(target) assert wrapped == (1, 2, 3) def test_trace_oldstyle(): class Oldie: test = object() def get_test(self): return self.test oldie = Oldie() wrapped = wrap_trace(oldie) assert wrapped.get_test() is oldie.test return boltons-16.4.1/tests/test_statsutils.py0000644000175000017500000000213012727357214016365 0ustar hlehle# -*- coding: utf-8 -*- from boltons.statsutils import Stats def test_stats_basic(): da = Stats(range(20)) assert da.mean == 9.5 assert round(da.std_dev, 2) == 5.77 assert da.variance == 33.25 assert da.skewness == 0 assert round(da.kurtosis, 1) == 1.9 assert da.median == 9.5 def _test_pearson(): import random from statsutils import pearson_type def get_pt(dist): vals = [dist() for x in range(10000)] pt = pearson_type(vals) return pt for x in range(3): # pt = get_pt(dist=lambda: random.normalvariate(15, 5)) # expect 0, normal # pt = get_pt(dist=lambda: random.weibullvariate(2, 3)) # gets 1, beta, weibull not specifically supported # pt = get_pt(dist=lambda: random.gammavariate(2, 3)) # expect 3, gamma # pt = get_pt(dist=lambda: random.betavariate(2, 3)) # expect 1, beta # pt = get_pt(dist=lambda: random.expovariate(0.2)) # expect 3, beta pt = get_pt(dist=lambda: random.uniform(0.0, 10.0)) # gets 2 print('pearson type:', pt) # import pdb;pdb.set_trace() boltons-16.4.1/tests/test_gcutils.py0000644000175000017500000000137512727357214015632 0ustar hlehle# -*- coding: utf-8 -*- import sys import time if '__pypy__' not in sys.builtin_module_names: # pypy's gc really is different from boltons.gcutils import get_all, toggle_gc_postcollect def test_get_all(): class TestType(object): pass tt = TestType() assert len(get_all(TestType)) == 1 assert len(get_all(bool)) == 0 return def test_toggle_gc_postcollect(): COUNT = int(1e6) start = time.time() with toggle_gc_postcollect: x = [{} for x in range(COUNT)] no_gc_time = time.time() - start start = time.time() x = [{} for x in range(COUNT)] with_gc_time = time.time() - start time_diff = no_gc_time < with_gc_time boltons-16.4.1/tests/test_tableutils.py0000644000175000017500000000270312727357214016324 0ustar hlehle# -*- coding: utf-8 -*- from boltons.tableutils import Table def test_table_lists(): data_lists = [['id', 'name'], [1, 'John Doe'], [2, 'Dale Simmons']] t1 = Table(data_lists) assert set(t1.headers) == set(['id', 'name']) assert len(t1) == 2 assert 'John Doe' in repr(t1) T2_REF_HTML = """
id1
nameJohn Doe
""" T3_REF_HTML = """
idname
1John Doe
2Dale Simmons
3Kurt Rose
4None
""" def test_table_dicts(): data_dicts = [{'id': 1, 'name': 'John Doe'}, {'id': 2, 'name': 'Dale Simmons'}] t2 = Table.from_dict(data_dicts[0]) t3 = Table.from_dict(data_dicts) t3.extend([[3, 'Kurt Rose'], [4]]) assert set(t2.headers) == set(['id', 'name']) assert len(t2) == 1 # the sorted() stuff handles ordering differences between versions # TODO: should maybe change Table to sort the headers of dicts and such? assert sorted(t2.to_html()) == sorted(T2_REF_HTML) assert sorted(t3.to_html()) == sorted(T3_REF_HTML) assert t3.to_text() def test_table_obj(): class TestType(object): def __init__(self): self.greeting = 'hi' t4 = Table.from_object(TestType()) assert len(t4) == 1 assert 'greeting' in t4.headers boltons-16.4.1/tests/test_listutils.py0000644000175000017500000001070112727357214016205 0ustar hlehle# -*- coding: utf-8 -*- import sys from boltons.listutils import SplayList, BarrelList def test_splay_list(): splay = SplayList(range(10)) splay.swap(0, 9) assert splay[0] == 9 assert splay[-1] == 0 splay.shift(-2) assert splay[0] == 8 assert splay[-1] == 0 assert len(splay) == 10 def test_barrel_list(): bl = BarrelList() bl.insert(0, 0) assert bl[0] == 0 assert len(bl) == 1 bl.insert(1, 1) assert list(bl) == [0, 1] bl.insert(0, -1) assert list(bl) == [-1, 0, 1] bl.extend((range(int(1e5)))) assert len(bl) == (1e5 + 3) bl._balance_list(0) assert len(bl) == (1e5 + 3) bl.pop(50000) assert len(bl) == (1e5 + 3 - 1) bl2 = BarrelList(TEST_INTS) bl2.sort() assert list(bl2[:5]) == [0, 74, 80, 96, 150] assert list(bl2[:-5:-1]) == [50508, 46607, 46428, 43442] # a hundred thousand integers bl3 = BarrelList(range(int(1e5))) for i in range(10000): # move the middle ten thou to the beginning bl3.insert(0, bl3.pop(len(bl3) // 2)) assert len(bl3) == 1e5 # length didn't change assert bl3[0] == 40001 # first value changed as expected assert bl3[-1] == sorted(bl3)[-1] # last value didn't change del bl3[10:5000] assert bl3[0] == 40001 assert len(bl3) == 1e5 - (5000 - 10) # length stayed co bl3[:20:2] = range(0, -10, -1) assert bl3[6] == -3 # some truly tricky stepping/slicing works # roughly increasing random integers # [ord(i) * x for i, x in zip(os.urandom(1024), range(1024))] TEST_INTS = [0, 74, 96, 183, 456, 150, 1098, 665, 1752, 1053, 190, 561, 2244, 2964, 2534, 750, 80, 612, 3780, 2698, 1320, 4935, 5324, 3220, 672, 2925, 6474, 6507, 3892, 4814, 1470, 6913, 6112, 6072, 7854, 8540, 4212, 7770, 8702, 1404, 2240, 902, 5250, 10320, 9680, 8775, 2392, 11327, 10368, 8085, 7750, 9333, 8008, 11395, 6858, 8690, 14112, 5358, 13572, 3304, 9000, 11712, 1426, 12033, 11136, 10270, 10626, 11189, 7208, 966, 9380, 16614, 10368, 11680, 8214, 16350, 4712, 5082, 13572, 15879, 6800, 8667, 17548, 9628, 3360, 19550, 1634, 20619, 1232, 17978, 22950, 10829, 5612, 22692, 10058, 21375, 19680, 5626, 15582, 18216, 2200, 20402, 24174, 3090, 19864, 2520, 15794, 18511, 4212, 18530, 8470, 7992, 5152, 4294, 456, 6095, 26564, 9477, 14042, 7259, 14520, 28677, 9394, 5289, 7812, 1625, 17514, 7493, 24704, 903, 33150, 1834, 11352, 20615, 32562, 540, 6256, 12878, 276, 17236, 6300, 25380, 9088, 27742, 4752, 33930, 7008, 22491, 7992, 2831, 34200, 18271, 22648, 6426, 15862, 26660, 29484, 2826, 14378, 26553, 16000, 18998, 28998, 16626, 19024, 22275, 18426, 21042, 29064, 30927, 26010, 14193, 9976, 30621, 12354, 33600, 40832, 20178, 20292, 12709, 26640, 29865, 23660, 20862, 34592, 28305, 24180, 38709, 18800, 11907, 28120, 34189, 4992, 8106, 14938, 6435, 12936, 31914, 1782, 995, 9800, 28542, 22018, 27608, 30396, 28700, 26986, 50508, 5616, 46607, 2310, 41356, 26712, 8733, 43442, 33755, 21384, 40145, 26160, 46428, 30360] test_barrel_list() if __name__ == '__main__': _TUNE_SETUP = """\ from boltons.listutils import BarrelList bl = BarrelList() bl._size_factor = %s bl.extend(range(int(%s))) """ def tune(): from collections import defaultdict import gc from timeit import timeit data_size = 1e5 old_size_factor = size_factor = 512 all_times = defaultdict(list) min_times = {} step = 512 while abs(step) > 4: gc.collect() for x in range(3): tottime = timeit('bl.insert(0, bl.pop(len(bl)//2))', _TUNE_SETUP % (size_factor, data_size), number=10000) all_times[size_factor].append(tottime) min_time = round(min(all_times[size_factor]), 3) min_times[size_factor] = min_time print(size_factor, min_time, step) if min_time > (min_times[old_size_factor] + 0.002): step = -step // 2 old_size_factor = size_factor size_factor += step print(tottime) try: tune() # main() except Exception as e: import pdb;pdb.post_mortem() raise boltons-16.4.1/boltons/0000755000175000017500000000000012727357214013057 5ustar hlehleboltons-16.4.1/boltons/listutils.py0000644000175000017500000002473512727357214015500 0ustar hlehle# -*- coding: utf-8 -*- """Python's builtin :class:`list` is a very fast and efficient sequence type, but it could be better for certain access patterns, such as non-sequential insertion into a large lists. ``listutils`` provides a pure-Python solution to this problem. For utilities for working with iterables and lists, check out :mod:`iterutils`. For the a :class:`list`-based version of :class:`collections.namedtuple`, check out :mod:`namedutils`. """ from __future__ import print_function, division import operator from math import log as math_log from itertools import chain, islice try: from typeutils import make_sentinel _MISSING = make_sentinel(var_name='_MISSING') except ImportError: _MISSING = object() try: xrange except NameError: # Python 3 compat xrange = range # TODO: expose splaylist? __all__ = ['BList', 'BarrelList'] # TODO: comparators # TODO: keep track of list lengths and bisect to the right list for # faster getitem (and slightly slower setitem and delitem ops) class BarrelList(list): """The ``BarrelList`` is a :class:`list` subtype backed by many dynamically-scaled sublists, to provide better scaling and random insertion/deletion characteristics. It is a subtype of the builtin :class:`list` and has an identical API, supporting indexing, slicing, sorting, etc. If application requirements call for something more performant, consider the `blist module available on PyPI`_. The name comes by way of Kurt Rose, who said it reminded him of barrel shifters. Not sure how, but it's BList-like, so the name stuck. BList is of course a reference to `B-trees`_. Args: iterable: An optional iterable of initial values for the list. >>> blist = BList(xrange(100000)) >>> blist.pop(50000) 50000 >>> len(blist) 99999 >>> len(blist.lists) # how many underlying lists 8 >>> slice_idx = blist.lists[0][-1] >>> blist[slice_idx:slice_idx + 2] BarrelList([11637, 11638]) Slicing is supported and works just fine across list borders, returning another instance of the BarrelList. .. _blist module available on PyPI: https://pypi.python.org/pypi/blist .. _B-trees: https://en.wikipedia.org/wiki/B-tree """ _size_factor = 1520 "This size factor is the result of tuning using the tune() function below." def __init__(self, iterable=None): self.lists = [[]] if iterable: self.extend(iterable) @property def _cur_size_limit(self): len_self, size_factor = len(self), self._size_factor return int(round(size_factor * math_log(len_self + 2, 2))) def _translate_index(self, index): if index < 0: index += len(self) rel_idx, lists = index, self.lists for list_idx in range(len(lists)): len_list = len(lists[list_idx]) if rel_idx < len_list: break rel_idx -= len_list if rel_idx < 0: return None, None return list_idx, rel_idx def _balance_list(self, list_idx): if list_idx < 0: list_idx += len(self.lists) cur_list, len_self = self.lists[list_idx], len(self) size_limit = self._cur_size_limit if len(cur_list) > size_limit: half_limit = size_limit // 2 while len(cur_list) > half_limit: next_list_idx = list_idx + 1 self.lists.insert(next_list_idx, cur_list[-half_limit:]) del cur_list[-half_limit:] return True return False def insert(self, index, item): if len(self.lists) == 1: self.lists[0].insert(index, item) self._balance_list(0) else: list_idx, rel_idx = self._translate_index(index) if list_idx is None: raise IndexError() self.lists[list_idx].insert(rel_idx, item) self._balance_list(list_idx) return def append(self, item): self.lists[-1].append(item) def extend(self, iterable): self.lists[-1].extend(iterable) def pop(self, *a): lists = self.lists if len(lists) == 1 and not a: return self.lists[0].pop() index = a and a[0] if index == () or index is None or index == -1: ret = lists[-1].pop() if len(lists) > 1 and not lists[-1]: lists.pop() else: list_idx, rel_idx = self._translate_index(index) if list_idx is None: raise IndexError() ret = lists[list_idx].pop(rel_idx) self._balance_list(list_idx) return ret def iter_slice(self, start, stop, step=None): iterable = self # TODO: optimization opportunities abound # start_list_idx, stop_list_idx = 0, len(self.lists) if start is None: start = 0 if stop is None: stop = len(self) if step is not None and step < 0: step = -step start, stop = -start, -stop - 1 iterable = reversed(self) if start < 0: start += len(self) # start_list_idx, start_rel_idx = self._translate_index(start) if stop < 0: stop += len(self) # stop_list_idx, stop_rel_idx = self._translate_index(stop) return islice(iterable, start, stop, step) def del_slice(self, start, stop, step=None): if step is not None and abs(step) > 1: # punt new_list = chain(self.iter_slice(0, start, step), self.iter_slice(stop, None, step)) self.lists[0][:] = new_list self._balance_list(0) return if start is None: start = 0 if stop is None: stop = len(self) start_list_idx, start_rel_idx = self._translate_index(start) stop_list_idx, stop_rel_idx = self._translate_index(stop) if start_list_idx is None: raise IndexError() if stop_list_idx is None: raise IndexError() if start_list_idx == stop_list_idx: del self.lists[start_list_idx][start_rel_idx:stop_rel_idx] elif start_list_idx < stop_list_idx: del self.lists[start_list_idx + 1:stop_list_idx] del self.lists[start_list_idx][start_rel_idx:] del self.lists[stop_list_idx][:stop_rel_idx] else: assert False, ('start list index should never translate to' ' greater than stop list index') __delslice__ = del_slice @classmethod def from_iterable(cls, it): return cls(it) def __iter__(self): return chain(*self.lists) def __reversed__(self): return chain.from_iterable(reversed(l) for l in reversed(self.lists)) def __len__(self): return sum([len(l) for l in self.lists]) def __contains__(self, item): for cur in self.lists: if item in cur: return True return False def __getitem__(self, index): try: start, stop, step = index.start, index.stop, index.step except AttributeError: index = operator.index(index) else: iter_slice = self.iter_slice(start, stop, step) ret = self.from_iterable(iter_slice) return ret list_idx, rel_idx = self._translate_index(index) if list_idx is None: raise IndexError() return self.lists[list_idx][rel_idx] def __delitem__(self, index): try: start, stop, step = index.start, index.stop, index.step except AttributeError: index = operator.index(index) else: self.del_slice(start, stop, step) return list_idx, rel_idx = self._translate_index(index) if list_idx is None: raise IndexError() del self.lists[list_idx][rel_idx] def __setitem__(self, index, item): try: start, stop, step = index.start, index.stop, index.step except AttributeError: index = operator.index(index) else: if len(self.lists) == 1: self.lists[0][index] = item else: tmp = list(self) tmp[index] = item self.lists[:] = [tmp] self._balance_list(0) return list_idx, rel_idx = self._translate_index(index) if list_idx is None: raise IndexError() self.lists[list_idx][rel_idx] = item def __getslice__(self, start, stop): iter_slice = self.iter_slice(start, stop, 1) return self.from_iterable(iter_slice) def __setslice__(self, start, stop, sequence): if len(self.lists) == 1: self.lists[0][start:stop] = sequence else: tmp = list(self) tmp[start:stop] = sequence self.lists[:] = [tmp] self._balance_list(0) return def __repr__(self): return '%s(%r)' % (self.__class__.__name__, list(self)) def sort(self): # poor pythonist's mergesort, it's faster than sorted(self) # when the lists' average length is greater than 512. if len(self.lists) == 1: self.lists[0].sort() else: for li in self.lists: li.sort() tmp_sorted = sorted(chain.from_iterable(self.lists)) del self.lists[:] self.lists[0] = tmp_sorted self._balance_list(0) def reverse(self): for cur in self.lists: cur.reverse() self.lists.reverse() def count(self, item): return sum([cur.count(item) for cur in self.lists]) def index(self, item): len_accum = 0 for cur in self.lists: try: rel_idx = cur.index(item) return len_accum + rel_idx except ValueError: len_accum += len(cur) raise ValueError('%r is not in list' % (item,)) BList = BarrelList class SplayList(list): """Like a `splay tree`_, the SplayList facilitates moving higher utility items closer to the front of the list for faster access. .. _splay tree: https://en.wikipedia.org/wiki/Splay_tree """ def shift(self, item_index, dest_index=0): if item_index == dest_index: return item = self.pop(item_index) self.insert(dest_index, item) def swap(self, item_index, dest_index): self[dest_index], self[item_index] = self[item_index], self[dest_index] boltons-16.4.1/boltons/namedutils.py0000644000175000017500000003304512727357214015603 0ustar hlehle# -*- coding: utf-8 -*- """\ The ``namedutils`` module defines two lightweight container types: :class:`namedtuple` and :class:`namedlist`. Both are subtypes of built-in sequence types, which are very fast and efficient. They simply add named attribute accessors for specific indexes within themselves. The :class:`namedtuple` is identical to the built-in :class:`collections.namedtuple`, with a couple of enhancements, including a ``__repr__`` more suitable to inheritance. The :class:`namedlist` is the mutable counterpart to the :class:`namedtuple`, and is much faster and lighter-weight than full-blown :class:`object`. Consider this if you're implementing nodes in a tree, graph, or other mutable data structure. If you want an even skinnier approach, you'll probably have to look to C. """ from __future__ import print_function import sys as _sys try: from collections import OrderedDict except ImportError: # backwards compatibility (2.6 has no OrderedDict) OrderedDict = dict from keyword import iskeyword as _iskeyword from operator import itemgetter as _itemgetter try: basestring def exec_(code, global_env): exec("exec code in global_env") except NameError: basestring = (str, bytes) # Python 3 compat def exec_(code, global_env): exec(code, global_env) __all__ = ['namedlist', 'namedtuple'] # Tiny templates _repr_tmpl = '{name}=%r' _imm_field_tmpl = '''\ {name} = _property(_itemgetter({index:d}), doc='Alias for field {index:d}') ''' _m_field_tmpl = '''\ {name} = _property(_itemgetter({index:d}), _itemsetter({index:d}), doc='Alias for field {index:d}') ''' ################################################################# ### namedtuple ################################################################# _namedtuple_tmpl = '''\ class {typename}(tuple): '{typename}({arg_list})' __slots__ = () _fields = {field_names!r} def __new__(_cls, {arg_list}): # TODO: tweak sig to make more extensible 'Create new instance of {typename}({arg_list})' return _tuple.__new__(_cls, ({arg_list})) @classmethod def _make(cls, iterable, new=_tuple.__new__, len=len): 'Make a new {typename} object from a sequence or iterable' result = new(cls, iterable) if len(result) != {num_fields:d}: raise TypeError('Expected {num_fields:d}' ' arguments, got %d' % len(result)) return result def __repr__(self): 'Return a nicely formatted representation string' tmpl = self.__class__.__name__ + '({repr_fmt})' return tmpl % self def _asdict(self): 'Return a new OrderedDict which maps field names to their values' return OrderedDict(zip(self._fields, self)) def _replace(_self, **kwds): 'Return a new {typename} object replacing field(s) with new values' result = _self._make(map(kwds.pop, {field_names!r}, _self)) if kwds: raise ValueError('Got unexpected field names: %r' % kwds.keys()) return result def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) __dict__ = _property(_asdict) def __getstate__(self): 'Exclude the OrderedDict from pickling' # wat pass {field_defs} ''' def namedtuple(typename, field_names, verbose=False, rename=False): """Returns a new subclass of tuple with named fields. >>> Point = namedtuple('Point', ['x', 'y']) >>> Point.__doc__ # docstring for the new class 'Point(x, y)' >>> p = Point(11, y=22) # instantiate with pos args or keywords >>> p[0] + p[1] # indexable like a plain tuple 33 >>> x, y = p # unpack like a regular tuple >>> x, y (11, 22) >>> p.x + p.y # fields also accessible by name 33 >>> d = p._asdict() # convert to a dictionary >>> d['x'] 11 >>> Point(**d) # convert from a dictionary Point(x=11, y=22) >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields Point(x=100, y=22) """ # Validate the field names. At the user's option, either generate an error # message or automatically replace the field name with a valid name. if isinstance(field_names, basestring): field_names = field_names.replace(',', ' ').split() field_names = [str(x) for x in field_names] if rename: seen = set() for index, name in enumerate(field_names): if (not all(c.isalnum() or c == '_' for c in name) or _iskeyword(name) or not name or name[0].isdigit() or name.startswith('_') or name in seen): field_names[index] = '_%d' % index seen.add(name) for name in [typename] + field_names: if not all(c.isalnum() or c == '_' for c in name): raise ValueError('Type names and field names can only contain ' 'alphanumeric characters and underscores: %r' % name) if _iskeyword(name): raise ValueError('Type names and field names cannot be a ' 'keyword: %r' % name) if name[0].isdigit(): raise ValueError('Type names and field names cannot start with ' 'a number: %r' % name) seen = set() for name in field_names: if name.startswith('_') and not rename: raise ValueError('Field names cannot start with an underscore: ' '%r' % name) if name in seen: raise ValueError('Encountered duplicate field name: %r' % name) seen.add(name) # Fill-in the class template fmt_kw = {'typename': typename} fmt_kw['field_names'] = tuple(field_names) fmt_kw['num_fields'] = len(field_names) fmt_kw['arg_list'] = repr(tuple(field_names)).replace("'", "")[1:-1] fmt_kw['repr_fmt'] = ', '.join(_repr_tmpl.format(name=name) for name in field_names) fmt_kw['field_defs'] = '\n'.join(_imm_field_tmpl.format(index=index, name=name) for index, name in enumerate(field_names)) class_definition = _namedtuple_tmpl.format(**fmt_kw) if verbose: print(class_definition) # Execute the template string in a temporary namespace and support # tracing utilities by setting a value for frame.f_globals['__name__'] namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, OrderedDict=OrderedDict, _property=property, _tuple=tuple) try: exec_(class_definition, namespace) except SyntaxError as e: raise SyntaxError(e.message + ':\n' + class_definition) result = namespace[typename] # For pickling to work, the __module__ variable needs to be set to the frame # where the named tuple is created. Bypass this step in environments where # sys._getframe is not defined (Jython for example) or sys._getframe is not # defined for arguments greater than 0 (IronPython). try: frame = _sys._getframe(1) result.__module__ = frame.f_globals.get('__name__', '__main__') except (AttributeError, ValueError): pass return result ################################################################# ### namedlist ################################################################# _namedlist_tmpl = '''\ class {typename}(list): '{typename}({arg_list})' __slots__ = () _fields = {field_names!r} def __new__(_cls, {arg_list}): # TODO: tweak sig to make more extensible 'Create new instance of {typename}({arg_list})' return _list.__new__(_cls, ({arg_list})) def __init__(self, {arg_list}): # tuple didn't need this but list does return _list.__init__(self, ({arg_list})) @classmethod def _make(cls, iterable, new=_list, len=len): 'Make a new {typename} object from a sequence or iterable' # why did this function exist? why not just star the # iterable like below? result = cls(*iterable) if len(result) != {num_fields:d}: raise TypeError('Expected {num_fields:d} arguments,' ' got %d' % len(result)) return result def __repr__(self): 'Return a nicely formatted representation string' tmpl = self.__class__.__name__ + '({repr_fmt})' return tmpl % tuple(self) def _asdict(self): 'Return a new OrderedDict which maps field names to their values' return OrderedDict(zip(self._fields, self)) def _replace(_self, **kwds): 'Return a new {typename} object replacing field(s) with new values' result = _self._make(map(kwds.pop, {field_names!r}, _self)) if kwds: raise ValueError('Got unexpected field names: %r' % kwds.keys()) return result def __getnewargs__(self): 'Return self as a plain list. Used by copy and pickle.' return tuple(self) __dict__ = _property(_asdict) def __getstate__(self): 'Exclude the OrderedDict from pickling' # wat pass {field_defs} ''' def namedlist(typename, field_names, verbose=False, rename=False): """Returns a new subclass of list with named fields. >>> Point = namedlist('Point', ['x', 'y']) >>> Point.__doc__ # docstring for the new class 'Point(x, y)' >>> p = Point(11, y=22) # instantiate with pos args or keywords >>> p[0] + p[1] # indexable like a plain list 33 >>> x, y = p # unpack like a regular list >>> x, y (11, 22) >>> p.x + p.y # fields also accessible by name 33 >>> d = p._asdict() # convert to a dictionary >>> d['x'] 11 >>> Point(**d) # convert from a dictionary Point(x=11, y=22) >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields Point(x=100, y=22) """ # Validate the field names. At the user's option, either generate an error # message or automatically replace the field name with a valid name. if isinstance(field_names, basestring): field_names = field_names.replace(',', ' ').split() field_names = [str(x) for x in field_names] if rename: seen = set() for index, name in enumerate(field_names): if (not all(c.isalnum() or c == '_' for c in name) or _iskeyword(name) or not name or name[0].isdigit() or name.startswith('_') or name in seen): field_names[index] = '_%d' % index seen.add(name) for name in [typename] + field_names: if not all(c.isalnum() or c == '_' for c in name): raise ValueError('Type names and field names can only contain ' 'alphanumeric characters and underscores: %r' % name) if _iskeyword(name): raise ValueError('Type names and field names cannot be a ' 'keyword: %r' % name) if name[0].isdigit(): raise ValueError('Type names and field names cannot start with ' 'a number: %r' % name) seen = set() for name in field_names: if name.startswith('_') and not rename: raise ValueError('Field names cannot start with an underscore: ' '%r' % name) if name in seen: raise ValueError('Encountered duplicate field name: %r' % name) seen.add(name) # Fill-in the class template fmt_kw = {'typename': typename} fmt_kw['field_names'] = tuple(field_names) fmt_kw['num_fields'] = len(field_names) fmt_kw['arg_list'] = repr(tuple(field_names)).replace("'", "")[1:-1] fmt_kw['repr_fmt'] = ', '.join(_repr_tmpl.format(name=name) for name in field_names) fmt_kw['field_defs'] = '\n'.join(_m_field_tmpl.format(index=index, name=name) for index, name in enumerate(field_names)) class_definition = _namedlist_tmpl.format(**fmt_kw) if verbose: print(class_definition) def _itemsetter(key): def _itemsetter(obj, value): obj[key] = value return _itemsetter # Execute the template string in a temporary namespace and support # tracing utilities by setting a value for frame.f_globals['__name__'] namespace = dict(_itemgetter=_itemgetter, _itemsetter=_itemsetter, __name__='namedlist_%s' % typename, OrderedDict=OrderedDict, _property=property, _list=list) try: exec_(class_definition, namespace) except SyntaxError as e: raise SyntaxError(e.message + ':\n' + class_definition) result = namespace[typename] # For pickling to work, the __module__ variable needs to be set to # the frame where the named list is created. Bypass this step in # environments where sys._getframe is not defined (Jython for # example) or sys._getframe is not defined for arguments greater # than 0 (IronPython). try: frame = _sys._getframe(1) result.__module__ = frame.f_globals.get('__name__', '__main__') except (AttributeError, ValueError): pass return result boltons-16.4.1/boltons/statsutils.py0000644000175000017500000006654512727357214015670 0ustar hlehle# -*- coding: utf-8 -*- """``statsutils`` provides tools aimed primarily at descriptive statistics for data analysis, such as :func:`mean` (average), :func:`median`, :func:`variance`, and many others, The :class:`Stats` type provides all the main functionality of the ``statsutils`` module. A :class:`Stats` object wraps a given dataset, providing all statistical measures as property attributes. These attributes cache their results, which allows efficient computation of multiple measures, as many measures rely on other measures. For example, relative standard deviation (:attr:`Stats.rel_std_dev`) relies on both the mean and standard deviation. The Stats object caches those results so no rework is done. The :class:`Stats` type's attributes have module-level counterparts for convenience when the computation reuse advantages do not apply. >>> stats = Stats(range(42)) >>> stats.mean 20.5 >>> mean(range(42)) 20.5 Statistics is a large field, and ``statsutils`` is focused on a few basic techniques that are useful in software. The following is a brief introduction to those techniques. For a more in-depth introduction, `Statistics for Software `_, an article I wrote on the topic. It introduces key terminology vital to effective usage of statistics. Statistical moments ------------------- Python programmers are probably familiar with the concept of the *mean* or *average*, which gives a rough quantitiative middle value by which a sample can be can be generalized. However, the mean is just the first of four `moment`_-based measures by which a sample or distribution can be measured. The four `Standardized moments`_ are: 1. `Mean`_ - :func:`mean` - theoretical middle value 2. `Variance`_ - :func:`variance` - width of value dispersion 3. `Skewness`_ - :func:`skewness` - symmetry of distribution 4. `Kurtosis`_ - :func:`kurtosis` - "peakiness" or "long-tailed"-ness For more information check out `the Moment article on Wikipedia`_. .. _moment: https://en.wikipedia.org/wiki/Moment_(mathematics) .. _Standardized moments: https://en.wikipedia.org/wiki/Standardized_moment .. _Mean: https://en.wikipedia.org/wiki/Mean .. _Variance: https://en.wikipedia.org/wiki/Variance .. _Skewness: https://en.wikipedia.org/wiki/Skewness .. _Kurtosis: https://en.wikipedia.org/wiki/Kurtosis .. _the Moment article on Wikipedia: https://en.wikipedia.org/wiki/Moment_(mathematics) Keep in mind that while these moments can give a bit more insight into the shape and distribution of data, they do not guarantee a complete picture. Wildly different datasets can have the same values for all four moments, so generalize wisely. Robust statistics ----------------- Moment-based statistics are notorious for being easily skewed by outliers. The whole field of robust statistics aims to mitigate this dilemma. ``statsutils`` also includes several robust statistical methods: * `Median`_ - The middle value of a sorted dataset * `Trimean`_ - Another robust measure of the data's central tendency * `Median Absolute Deviation`_ (MAD) - A robust measure of variability, a natural counterpart to :func:`variance`. * `Trimming`_ - Reducing a dataset to only the middle majority of data is a simple way of making other estimators more robust. .. _Median: https://en.wikipedia.org/wiki/Median .. _Trimean: https://en.wikipedia.org/wiki/Trimean .. _Median Absolute Deviation: https://en.wikipedia.org/wiki/Median_absolute_deviation .. _Trimming: https://en.wikipedia.org/wiki/Trimmed_estimator Online and Offline Statistics ----------------------------- Unrelated to computer networking, `online`_ statistics involve calculating statistics in a `streaming`_ fashion, without all the data being available. The :class:`Stats` type is meant for the more traditional offline statistics when all the data is available. For pure-Python online statistics accumulators, look at the `Lithoxyl`_ system instrumentation package. .. _Online: https://en.wikipedia.org/wiki/Online_algorithm .. _streaming: https://en.wikipedia.org/wiki/Streaming_algorithm .. _Lithoxyl: https://github.com/mahmoud/lithoxyl """ from __future__ import print_function import bisect from math import floor, ceil class _StatsProperty(object): def __init__(self, name, func): self.name = name self.func = func self.internal_name = '_' + name doc = func.__doc__ or '' pre_doctest_doc, _, _ = doc.partition('>>>') self.__doc__ = pre_doctest_doc def __get__(self, obj, objtype=None): if obj is None: return self if not obj.data: return obj.default try: return getattr(obj, self.internal_name) except AttributeError: setattr(obj, self.internal_name, self.func(obj)) return getattr(obj, self.internal_name) class Stats(object): """The ``Stats`` type is used to represent a group of unordered statistical datapoints for calculations such as mean, median, and variance. Args: data (list): List or other iterable containing numeric values. default (float): A value to be returned when a given statistical measure is not defined. 0.0 by default, but ``float('nan')`` is appropriate for stricter applications. use_copy (bool): By default Stats objects copy the initial data into a new list to avoid issues with modifications. Pass ``False`` to disable this behavior. is_sorted (bool): Presorted data can skip an extra sorting step for a little speed boost. Defaults to False. """ def __init__(self, data, default=0.0, use_copy=True, is_sorted=False): self._use_copy = use_copy self._is_sorted = is_sorted if use_copy: self.data = list(data) else: self.data = data self.default = default cls = self.__class__ self._prop_attr_names = [a for a in dir(self) if isinstance(getattr(cls, a, None), _StatsProperty)] self._pearson_precision = 0 def __len__(self): return len(self.data) def __iter__(self): return iter(self.data) def _get_sorted_data(self): """When using a copy of the data, it's better to have that copy be sorted, but we do it lazily using this method, in case no sorted measures are used. I.e., if median is never called, sorting would be a waste. When not using a copy, it's presumed that all optimizations are on the user. """ if not self._use_copy: return sorted(self.data) elif not self._is_sorted: self.data.sort() return self.data def clear_cache(self): for attr_name in self._prop_attr_names: attr_name = getattr(self.__class__, attr_name).internal_name if not hasattr(self, attr_name): continue delattr(self, attr_name) def _calc_count(self): """The number of items in this Stats object. Returns the same as :func:`len` on a Stats object, but provided for pandas terminology parallelism. >>> Stats(range(20)).count 20 """ return len(self.data) count = _StatsProperty('count', _calc_count) def _calc_mean(self): """ The arithmetic mean, or "average". Sum of the values divided by the number of values. >>> mean(range(20)) 9.5 >>> mean(list(range(19)) + [949]) # 949 is an arbitrary outlier 56.0 """ return sum(self.data, 0.0) / len(self.data) mean = _StatsProperty('mean', _calc_mean) def _calc_max(self): """ The maximum value present in the data. >>> Stats([2, 1, 3]).max 3 """ if self._is_sorted: return self.data[-1] return max(self.data) max = _StatsProperty('max', _calc_max) def _calc_min(self): """ The minimum value present in the data. >>> Stats([2, 1, 3]).min 1 """ if self._is_sorted: return self.data[0] return min(self.data) min = _StatsProperty('min', _calc_min) def _calc_median(self): """ The median is either the middle value or the average of the two middle values of a sample. Compared to the mean, it's generally more resilient to the presence of outliers in the sample. >>> median([2, 1, 3]) 2 >>> median(range(97)) 48 >>> median(list(range(96)) + [1066]) # 1066 is an arbitrary outlier 48 """ return self._get_quantile(self._get_sorted_data(), 0.5) median = _StatsProperty('median', _calc_median) def _calc_iqr(self): """Inter-quartile range (IQR) is the difference between the 75th percentile and 25th percentile. IQR is a robust measure of dispersion, like standard deviation, but safer to compare between datasets, as it is less influenced by outliers. >>> iqr([1, 2, 3, 4, 5]) 2 >>> iqr(range(1001)) 500 """ return self.get_quantile(0.75) - self.get_quantile(0.25) iqr = _StatsProperty('iqr', _calc_iqr) def _calc_trimean(self): """The trimean is a robust measure of central tendency, like the median, that takes the weighted average of the median and the upper and lower quartiles. >>> trimean([2, 1, 3]) 2.0 >>> trimean(range(97)) 48.0 >>> trimean(list(range(96)) + [1066]) # 1066 is an arbitrary outlier 48.0 """ sorted_data = self._get_sorted_data() gq = lambda q: self._get_quantile(sorted_data, q) return (gq(0.25) + (2 * gq(0.5)) + gq(0.75)) / 4.0 trimean = _StatsProperty('trimean', _calc_trimean) def _calc_variance(self): """\ Variance is the average of the squares of the difference between each value and the mean. >>> variance(range(97)) 784.0 """ return mean(self._get_pow_diffs(2)) variance = _StatsProperty('variance', _calc_variance) def _calc_std_dev(self): """\ Standard deviation. Square root of the variance. >>> std_dev(range(97)) 28.0 """ return self.variance ** 0.5 std_dev = _StatsProperty('std_dev', _calc_std_dev) def _calc_median_abs_dev(self): """\ Median Absolute Deviation is a robust measure of statistical dispersion: http://en.wikipedia.org/wiki/Median_absolute_deviation >>> median_abs_dev(range(97)) 24.0 """ sorted_vals = sorted(self.data) x = float(median(sorted_vals)) # programmatically defined below return median([abs(x - v) for v in sorted_vals]) median_abs_dev = _StatsProperty('median_abs_dev', _calc_median_abs_dev) mad = median_abs_dev # convenience def _calc_rel_std_dev(self): """\ Standard deviation divided by the absolute value of the average. http://en.wikipedia.org/wiki/Relative_standard_deviation >>> print('%1.3f' % rel_std_dev(range(97))) 0.583 """ abs_mean = abs(self.mean) if abs_mean: return self.std_dev / abs_mean else: return self.default rel_std_dev = _StatsProperty('rel_std_dev', _calc_rel_std_dev) def _calc_skewness(self): """\ Indicates the asymmetry of a curve. Positive values mean the bulk of the values are on the left side of the average and vice versa. http://en.wikipedia.org/wiki/Skewness See the module docstring for more about statistical moments. >>> skewness(range(97)) # symmetrical around 48.0 0.0 >>> left_skewed = skewness(list(range(97)) + list(range(10))) >>> right_skewed = skewness(list(range(97)) + list(range(87, 97))) >>> round(left_skewed, 3), round(right_skewed, 3) (0.114, -0.114) """ data, s_dev = self.data, self.std_dev if len(data) > 1 and s_dev > 0: return (sum(self._get_pow_diffs(3)) / float((len(data) - 1) * (s_dev ** 3))) else: return self.default skewness = _StatsProperty('skewness', _calc_skewness) def _calc_kurtosis(self): """\ Indicates how much data is in the tails of the distribution. The result is always positive, with the normal "bell-curve" distribution having a kurtosis of 3. http://en.wikipedia.org/wiki/Kurtosis See the module docstring for more about statistical moments. >>> kurtosis(range(9)) 1.99125 With a kurtosis of 1.99125, [0, 1, 2, 3, 4, 5, 6, 7, 8] is more centrally distributed than the normal curve. """ data, s_dev = self.data, self.std_dev if len(data) > 1 and s_dev > 0: return (sum(self._get_pow_diffs(4)) / float((len(data) - 1) * (s_dev ** 4))) else: return 0.0 kurtosis = _StatsProperty('kurtosis', _calc_kurtosis) def _calc_pearson_type(self): precision = self._pearson_precision skewness = self.skewness kurtosis = self.kurtosis beta1 = skewness ** 2.0 beta2 = kurtosis * 1.0 # TODO: range checks? c0 = (4 * beta2) - (3 * beta1) c1 = skewness * (beta2 + 3) c2 = (2 * beta2) - (3 * beta1) - 6 if round(c1, precision) == 0: if round(beta2, precision) == 3: return 0 # Normal else: if beta2 < 3: return 2 # Symmetric Beta elif beta2 > 3: return 7 elif round(c2, precision) == 0: return 3 # Gamma else: k = c1 ** 2 / (4 * c0 * c2) if k < 0: return 1 # Beta raise RuntimeError('missed a spot') pearson_type = _StatsProperty('pearson_type', _calc_pearson_type) @staticmethod def _get_quantile(sorted_data, q): data, n = sorted_data, len(sorted_data) idx = q / 1.0 * (n - 1) idx_f, idx_c = int(floor(idx)), int(ceil(idx)) if idx_f == idx_c: return data[idx_f] return (data[idx_f] * (idx_c - idx)) + (data[idx_c] * (idx - idx_f)) def get_quantile(self, q): """ >>> Stats(range(100)).get_quantile(0.5) 49.5 """ q = float(q) if not 0.0 <= q <= 1.0: raise ValueError('expected q between 0.0 and 1.0, not %r' % q) elif not self.data: return self.default return self._get_quantile(self._get_sorted_data(), q) def get_zscore(self, value): """Get the z-score for *value* in the group. If the standard deviation is 0, 0 inf or -inf will be returned to indicate whether the value is equal to, greater than or below the group's mean. """ mean = self.mean if self.std_dev == 0: if value == mean: return 0 if value > mean: return float('inf') if value < mean: return float('-inf') return (float(value) - mean) / self.std_dev def trim_relative(self, amount=0.15): """A utility function used to cut a proportion of values off each end of a list of values. This has the effect of limiting the effect of outliers. Args: amount (float): A value between 0.0 and 0.5 to trim off of each side of the data. .. note: This operation modifies the data in-place. It does not make or return a copy. """ trim = float(amount) if not 0.0 <= trim < 0.5: raise ValueError('expected amount between 0.0 and 0.5, not %r' % trim) size = len(self.data) size_diff = int(size * trim) if size_diff == 0.0: return self.data = self._get_sorted_data()[size_diff:-size_diff] self.clear_cache() def _get_pow_diffs(self, power): """ A utility function used for calculating statistical moments. """ m = self.mean return [(v - m) ** power for v in self.data] def _get_bin_bounds(self, count=None, with_max=False): if not self.data: return [0.0] # TODO: raise? data = self.data len_data, min_data, max_data = len(data), min(data), max(data) if len_data < 4: if not count: count = len_data dx = (max_data - min_data) / float(count) bins = [min_data + (dx * i) for i in range(count)] elif count is None: # freedman algorithm for fixed-width bin selection q25, q75 = self.get_quantile(0.25), self.get_quantile(0.75) dx = 2 * (q75 - q25) / (len_data ** (1 / 3.0)) bin_count = max(1, int(ceil((max_data - min_data) / dx))) bins = [min_data + (dx * i) for i in range(bin_count + 1)] bins = [b for b in bins if b < max_data] else: dx = (max_data - min_data) / float(count) bins = [min_data + (dx * i) for i in range(count)] if with_max: bins.append(float(max_data)) return bins def get_histogram_counts(self, bins=None, **kw): """Produces a list of ``(bin, count)`` pairs comprising a histogram of the Stats object's data, using fixed-width bins. See :meth:`Stats.format_histogram` for more details. Args: bins (int): maximum number of bins, or list of floating-point bin boundaries. Defaults to the output of Freedman's algorithm. bin_digits (int): Number of digits used to round down the bin boundaries. Defaults to 1. The output of this method can be stored and/or modified, and then passed to :func:`statsutils.format_histogram_counts` to achieve the same text formatting as the :meth:`~Stats.format_histogram` method. This can be useful for snapshotting over time. """ bin_digits = int(kw.pop('bin_digits', 1)) if kw: raise TypeError('unexpected keyword arguments: %r' % kw.keys()) if not bins: bins = self._get_bin_bounds() else: try: bin_count = int(bins) except TypeError: try: bins = [float(x) for x in bins] except Exception: raise ValueError('bins expected integer bin count or list' ' of float bin boundaries, not %r' % bins) if self.min < bins[0]: bins = [self.min] + bins else: bins = self._get_bin_bounds(bin_count) # floor and ceil really should have taken ndigits, like round() round_factor = 10.0 ** bin_digits bins = [floor(b * round_factor) / round_factor for b in bins] bins = sorted(set(bins)) idxs = [bisect.bisect(bins, d) - 1 for d in self.data] count_map = {} # would have used Counter, but py26 support for idx in idxs: try: count_map[idx] += 1 except KeyError: count_map[idx] = 1 bin_counts = [(b, count_map.get(i, 0)) for i, b in enumerate(bins)] return bin_counts def format_histogram(self, bins=None, **kw): """Produces a textual histogram of the data, using fixed-width bins, allowing for simple visualization, even in console environments. >>> data = list(range(20)) + list(range(5, 15)) + [10] >>> print(Stats(data).format_histogram()) 0.0: 5 ################################ 4.4: 8 ################################################### 8.9: 11 ###################################################################### 13.3: 5 ################################ 17.8: 2 ############# In this histogram, five values are between 0.0 and 4.4, eight are between 4.4 and 8.9, and two values lie between 17.8 and the max. You can specify the number of bins, or provide a list of bin boundaries themselves. If no bins are provided, as in the example above, `Freedman's algorithm`_ for bin selection is used. Args: bins (int): Maximum number of bins for the histogram. Also accepts a list of floating-point bin boundaries. If the minimum boundary is still greater than the minimum value in the data, that boundary will be implicitly added. Defaults to the bin boundaries returned by `Freedman's algorithm`_. bin_digits (int): Number of digits to round each bin to. Note that bins are always rounded down to avoid clipping any data. Defaults to 1. width (int): integer number of columns in the longest line in the histogram. Defaults to console width on Python 3.3+, or 80 if that is not available. format_bin (callable): Called on each bin to create a label for the final output. Use this function to add units, such as "ms" for milliseconds. Should you want something more programmatically reusable, see the :meth:`~Stats.get_histogram_counts` method, the output of is used by format_histogram. The :meth:`~Stats.describe` method is another useful summarization method, albeit less visual. .. _Freedman's algorithm: https://en.wikipedia.org/wiki/Freedman%E2%80%93Diaconis_rule """ width = kw.pop('width', None) format_bin = kw.pop('format_bin', None) bin_counts = self.get_histogram_counts(bins=bins, **kw) return format_histogram_counts(bin_counts, width=width, format_bin=format_bin) def describe(self, quantiles=None, format=None): """Provides standard summary statistics for the data in the Stats object, in one of several convenient formats. Args: quantiles (list): A list of numeric values to use as quantiles in the resulting summary. All values must be 0.0-1.0, with 0.5 representing the median. Defaults to ``[0.25, 0.5, 0.75]``, representing the standard quartiles. format (str): Controls the return type of the function, with one of three valid values: ``"dict"`` gives back a :class:`dict` with the appropriate keys and values. ``"list"`` is a list of key-value pairs in an order suitable to pass to an OrderedDict or HTML table. ``"text"`` converts the values to text suitable for printing, as seen below. Here is the information returned by a default ``describe``, as presented in the ``"text"`` format: >>> stats = Stats(range(1, 8)) >>> print(stats.describe(format='text')) count: 7 mean: 4.0 std_dev: 2.0 mad: 2.0 min: 1 0.25: 2.5 0.5: 4 0.75: 5.5 max: 7 For more advanced descriptive statistics, check out my blog post on the topic `Statistics for Software `_. """ if format is None: format = 'dict' elif format not in ('dict', 'list', 'text'): raise ValueError('invalid format for describe,' ' expected one of "dict"/"list"/"text", not %r' % format) quantiles = quantiles or [0.25, 0.5, 0.75] q_items = [] for q in quantiles: q_val = self.get_quantile(q) q_items.append((str(q), q_val)) items = [('count', self.count), ('mean', self.mean), ('std_dev', self.std_dev), ('mad', self.mad), ('min', self.min)] items.extend(q_items) items.append(('max', self.max)) if format == 'dict': ret = dict(items) elif format == 'list': ret = items elif format == 'text': ret = '\n'.join(['%s%s' % ((label + ':').ljust(10), val) for label, val in items]) return ret def describe(data, quantiles=None, format=None): """A convenience function to get standard summary statistics useful for describing most data. See :meth:`Stats.describe` for more details. >>> print(describe(range(7), format='text')) count: 7 mean: 3.0 std_dev: 2.0 mad: 2.0 min: 0 0.25: 1.5 0.5: 3 0.75: 4.5 max: 6 See :meth:`Stats.format_histogram` for another very useful summarization that uses textual visualization. """ return Stats(data).describe(quantiles=quantiles, format=format) def _get_conv_func(attr_name): def stats_helper(data, default=0.0): return getattr(Stats(data, default=default, use_copy=False), attr_name) return stats_helper for attr_name, attr in list(Stats.__dict__.items()): if isinstance(attr, _StatsProperty): if attr_name in ('max', 'min', 'count'): # don't shadow builtins continue if attr_name in ('mad',): # convenience aliases continue func = _get_conv_func(attr_name) func.__doc__ = attr.func.__doc__ globals()[attr_name] = func delattr(Stats, '_calc_' + attr_name) # cleanup del attr del attr_name del func def format_histogram_counts(bin_counts, width=None, format_bin=None): """The formatting logic behind :meth:`Stats.format_histogram`, which takes the output of :meth:`Stats.get_histogram_counts`, and passes them to this function. Args: bin_counts (list): A list of bin values to counts. width (int): Number of character columns in the text output, defaults to 80 or console width in Python 3.3+. format_bin (callable): Used to convert bin values into string labels. """ lines = [] if not format_bin: format_bin = lambda v: v if not width: try: import shutil # python 3 convenience width = shutil.get_terminal_size()[0] except Exception: width = 80 bins = [b for b, _ in bin_counts] count_max = max([count for _, count in bin_counts]) count_cols = len(str(count_max)) labels = ['%s' % format_bin(b) for b in bins] label_cols = max([len(l) for l in labels]) tmp_line = '%s: %s #' % ('x' * label_cols, count_max) bar_cols = max(width - len(tmp_line), 3) line_k = float(bar_cols) / count_max tmpl = "{label:>{label_cols}}: {count:>{count_cols}} {bar}" for label, (bin_val, count) in zip(labels, bin_counts): bar_len = int(round(count * line_k)) bar = ('#' * bar_len) or '|' line = tmpl.format(label=label, label_cols=label_cols, count=count, count_cols=count_cols, bar=bar) lines.append(line) return '\n'.join(lines) boltons-16.4.1/boltons/formatutils.py0000644000175000017500000002575312727357214016016 0ustar hlehle# -*- coding: utf-8 -*- """`PEP 3101`_ introduced the :meth:`str.format` method, and what would later be called "new-style" string formatting. For the sake of explicit correctness, it is probably best to refer to Python's dual string formatting capabilities as *bracket-style* and *percent-style*. There is overlap, but one does not replace the other. * Bracket-style is more pluggable, slower, and uses a method. * Percent-style is simpler, faster, and uses an operator. Bracket-style formatting brought with it a much more powerful toolbox, but it was far from a full one. :meth:`str.format` uses `more powerful syntax`_, but `the tools and idioms`_ for working with that syntax are not well-developed nor well-advertised. ``formatutils`` adds several functions for working with bracket-style format strings: * :class:`DeferredValue`: Defer fetching or calculating a value until format time. * :func:`get_format_args`: Parse the positional and keyword arguments out of a format string. * :func:`tokenize_format_str`: Tokenize a format string into literals and :class:`BaseFormatField` objects. * :func:`construct_format_field_str`: Assists in progammatic construction of format strings. * :func:`infer_positional_format_args`: Converts anonymous references in 2.7+ format strings to explicit positional arguments suitable for usage with Python 2.6. .. _more powerful syntax: https://docs.python.org/2/library/string.html#format-string-syntax .. _the tools and idioms: https://docs.python.org/2/library/string.html#string-formatting .. _PEP 3101: https://www.python.org/dev/peps/pep-3101/ """ # TODO: also include percent-formatting utils? # TODO: include lithoxyl.formatters.Formatter (or some adaptation)? from __future__ import print_function import re from string import Formatter __all__ = ['DeferredValue', 'get_format_args', 'tokenize_format_str', 'construct_format_field_str', 'infer_positional_format_args', 'BaseFormatField'] _pos_farg_re = re.compile('({{)|' # escaped open-brace '(}})|' # escaped close-brace '({[:!.\[}])') # anon positional format arg def construct_format_field_str(fname, fspec, conv): """ Constructs a format field string from the field name, spec, and conversion character (``fname``, ``fspec``, ``conv``). See Python String Formatting for more info. """ if fname is None: return '' ret = '{' + fname if conv: ret += '!' + conv if fspec: ret += ':' + fspec ret += '}' return ret def split_format_str(fstr): """Does very basic spliting of a format string, returns a list of strings. For full tokenization, see :func:`tokenize_format_str`. """ ret = [] for lit, fname, fspec, conv in Formatter().parse(fstr): if fname is None: ret.append((lit, None)) continue field_str = construct_format_field_str(fname, fspec, conv) ret.append((lit, field_str)) return ret def infer_positional_format_args(fstr): """Takes format strings with anonymous positional arguments, (e.g., "{}" and {:d}), and converts them into numbered ones for explicitness and compatibility with 2.6. Returns a string with the inferred positional arguments. """ # TODO: memoize ret, max_anon = '', 0 # look for {: or {! or {. or {[ or {} start, end, prev_end = 0, 0, 0 for match in _pos_farg_re.finditer(fstr): start, end, group = match.start(), match.end(), match.group() if prev_end < start: ret += fstr[prev_end:start] prev_end = end if group == '{{' or group == '}}': ret += group continue ret += '{%s%s' % (max_anon, group[1:]) max_anon += 1 ret += fstr[prev_end:] return ret # This approach is hardly exhaustive but it works for most builtins _INTCHARS = 'bcdoxXn' _FLOATCHARS = 'eEfFgGn%' _TYPE_MAP = dict([(x, int) for x in _INTCHARS] + [(x, float) for x in _FLOATCHARS]) _TYPE_MAP['s'] = str def get_format_args(fstr): """ Turn a format string into two lists of arguments referenced by the format string. One is positional arguments, and the other is named arguments. Each element of the list includes the name and the nominal type of the field. # >>> get_format_args("{noun} is {1:d} years old{punct}") # ([(1, )], [('noun', ), ('punct', )]) # XXX: Py3k >>> get_format_args("{noun} is {1:d} years old{punct}") == \ ([(1, int)], [('noun', str), ('punct', str)]) True """ # TODO: memoize formatter = Formatter() fargs, fkwargs, _dedup = [], [], set() def _add_arg(argname, type_char='s'): if argname not in _dedup: _dedup.add(argname) argtype = _TYPE_MAP.get(type_char, str) # TODO: unicode try: fargs.append((int(argname), argtype)) except ValueError: fkwargs.append((argname, argtype)) for lit, fname, fspec, conv in formatter.parse(fstr): if fname is not None: type_char = fspec[-1:] fname_list = re.split('[.[]', fname) if len(fname_list) > 1: raise ValueError('encountered compound format arg: %r' % fname) try: base_fname = fname_list[0] assert base_fname except (IndexError, AssertionError): raise ValueError('encountered anonymous positional argument') _add_arg(fname, type_char) for sublit, subfname, _, _ in formatter.parse(fspec): # TODO: positional and anon args not allowed here. if subfname is not None: _add_arg(subfname) return fargs, fkwargs def tokenize_format_str(fstr, resolve_pos=True): """Takes a format string, turns it into a list of alternating string literals and :class:`BaseFormatField` tokens. By default, also infers anonymous positional references into explict, numbered positional references. To disable this behavior set *resolve_pos* to ``False``. """ ret = [] if resolve_pos: fstr = infer_positional_format_args(fstr) formatter = Formatter() for lit, fname, fspec, conv in formatter.parse(fstr): if lit: ret.append(lit) if fname is None: continue ret.append(BaseFormatField(fname, fspec, conv)) return ret class BaseFormatField(object): """A class representing a reference to an argument inside of a bracket-style format string. For instance, in ``"{greeting}, world!"``, there is a field named "greeting". These fields can have many options applied to them. See the Python docs on `Format String Syntax`_ for the full details. .. _Format String Syntax: https://docs.python.org/2/library/string.html#string-formatting """ def __init__(self, fname, fspec='', conv=None): self.set_fname(fname) self.set_fspec(fspec) self.set_conv(conv) def set_fname(self, fname): "Set the field name." path_list = re.split('[.[]', fname) # TODO self.base_name = path_list[0] self.fname = fname self.subpath = path_list[1:] self.is_positional = not self.base_name or self.base_name.isdigit() def set_fspec(self, fspec): "Set the field spec." fspec = fspec or '' subfields = [] for sublit, subfname, _, _ in Formatter().parse(fspec): if subfname is not None: subfields.append(subfname) self.subfields = subfields self.fspec = fspec self.type_char = fspec[-1:] self.type_func = _TYPE_MAP.get(self.type_char, str) def set_conv(self, conv): """There are only two built-in converters: ``s`` and ``r``. They are somewhat rare and appearlike ``"{ref!r}"``.""" # TODO self.conv = conv self.conv_func = None # TODO @property def fstr(self): "The current state of the field in string format." return construct_format_field_str(self.fname, self.fspec, self.conv) def __repr__(self): cn = self.__class__.__name__ args = [self.fname] if self.conv is not None: args.extend([self.fspec, self.conv]) elif self.fspec != '': args.append(self.fspec) args_repr = ', '.join([repr(a) for a in args]) return '%s(%s)' % (cn, args_repr) def __str__(self): return self.fstr _UNSET = object() class DeferredValue(object): """:class:`DeferredValue` is a wrapper type, used to defer computing values which would otherwise be expensive to stringify and format. This is most valuable in areas like logging, where one would not want to waste time formatting a value for a log message which will subsequently be filtered because the message's log level was DEBUG and the logger was set to only emit CRITICAL messages. The :class:``DeferredValue`` is initialized with a callable that takes no arguments and returns the value, which can be of any type. By default DeferredValue only calls that callable once, and future references will get a cached value. This behavior can be disabled by setting *cache_value* to ``False``. Args: func (function): A callable that takes no arguments and computes the value being represented. cache_value (bool): Whether subsequent usages will call *func* again. Defaults to ``True``. >>> import sys >>> dv = DeferredValue(lambda: len(sys._current_frames())) >>> output = "works great in all {0} threads!".format(dv) PROTIP: To keep lines shorter, use: ``from formatutils import DeferredValue as DV`` """ def __init__(self, func, cache_value=True): self.func = func self.cache_value = True self._value = _UNSET def get_value(self): """Computes, optionally caches, and returns the value of the *func*. If ``get_value()`` has been called before, a cached value may be returned depending on the *cache_value* option passed to the constructor. """ if self._value is _UNSET or not self.cache_value: value = self.func() if self.cache_value: self._value = value return value def __int__(self): return int(self.get_value()) def __float__(self): return float(self.get_value()) def __str__(self): return str(self.get_value()) def __unicode__(self): return unicode(self.get_value()) def __repr__(self): return repr(self.get_value()) def __format__(self, fmt): value = self.get_value() pt = fmt[-1:] # presentation type type_conv = _TYPE_MAP.get(pt, str) try: return value.__format__(fmt) except (ValueError, TypeError): # TODO: this may be overkill return type_conv(value).__format__(fmt) # end formatutils.py boltons-16.4.1/boltons/debugutils.py0000644000175000017500000002153312727357214015604 0ustar hlehle# -*- coding: utf-8 -*- """ A small set of utilities useful for debugging misbehaving applications. Currently this focuses on ways to use :mod:`pdb`, the built-in Python debugger. """ import sys import time try: basestring from repr import Repr except NameError: basestring = (str, bytes) # py3 from reprlib import Repr try: from typeutils import make_sentinel _UNSET = make_sentinel(var_name='_UNSET') except ImportError: _UNSET = object() __all__ = ['pdb_on_signal', 'pdb_on_exception'] def pdb_on_signal(signalnum=None): """Installs a signal handler for *signalnum*, which defaults to ``SIGINT``, or keyboard interrupt/ctrl-c. This signal handler launches a :mod:`pdb` breakpoint. Results vary in concurrent systems, but this technique can be useful for debugging infinite loops, or easily getting into deep call stacks. Args: signalnum (int): The signal number of the signal to handle with pdb. Defaults to :mod:`signal.SIGINT`, see :mod:`signal` for more information. """ import pdb import signal if not signalnum: signalnum = signal.SIGINT old_handler = signal.getsignal(signalnum) def pdb_int_handler(sig, frame): signal.signal(signalnum, old_handler) pdb.set_trace() pdb_on_signal(signalnum) # use 'u' to find your code and 'h' for help signal.signal(signalnum, pdb_int_handler) return def pdb_on_exception(limit=100): """Installs a handler which, instead of exiting, attaches a post-mortem pdb console whenever an unhandled exception is encountered. Args: limit (int): the max number of stack frames to display when printing the traceback A similar effect can be achieved from the command-line using the following command:: python -m pdb your_code.py But ``pdb_on_exception`` allows you to do this conditionally and within your application. To restore default behavior, just do:: sys.excepthook = sys.__excepthook__ """ import pdb import sys import traceback def pdb_excepthook(exc_type, exc_val, exc_tb): traceback.print_tb(exc_tb, limit=limit) pdb.post_mortem(exc_tb) sys.excepthook = pdb_excepthook return _repr_obj = Repr() _repr_obj.maxstring = 50 _repr_obj.maxother = 50 brief_repr = _repr_obj.repr # events: call, return, get, set, del, raise def trace_print_hook(event, label, obj, attr_name, args=(), kwargs={}, result=_UNSET): fargs = (event.ljust(6), time.time(), label.rjust(10), obj.__class__.__name__, attr_name) if event == 'get': tmpl = '%s %s - %s - %s.%s -> %s' fargs += (brief_repr(result),) elif event == 'set': tmpl = '%s %s - %s - %s.%s = %s' fargs += (brief_repr(args[0]),) elif event == 'del': tmpl = '%s %s - %s - %s.%s' else: # call/return/raise tmpl = '%s %s - %s - %s.%s(%s)' fargs += (', '.join([brief_repr(a) for a in args]),) if kwargs: tmpl = '%s %s - %s - %s.%s(%s, %s)' fargs += (', '.join(['%s=%s' % (k, brief_repr(v)) for k, v in kwargs.items()]),) if result is not _UNSET: tmpl += ' -> %s' fargs += (brief_repr(result),) print(tmpl % fargs) return def wrap_trace(obj, hook=trace_print_hook, which=None, events=None, label=None): """Monitor an object for interactions. Whenever code calls a method, gets an attribute, or sets an attribute, an event is called. By default the trace output is printed, but a custom tracing *hook* can be passed. Args: obj (object): New- or old-style object to be traced. Built-in objects like lists and dicts also supported. hook (callable): A function called once for every event. See below for details. which (str): One or more attribute names to trace, or a function accepting attribute name and value, and returing True/False. events (str): One or more kinds of events to call *hook* on. Expected values are ``['get', 'set', 'del', 'call', 'raise', 'return']``. Defaults to all events. label (str): A name to associate with the traced object Defaults to hexadecimal memory address, similar to repr. The object returned is not the same object as the one passed in. It will not pass identity checks. However, it will pass :func:`isinstance` checks, as it is a new instance of a new subtype of the object passed. """ # other actions: pdb.set_trace, print, aggregate, aggregate_return # (like aggregate but with the return value) # TODO: test classmethod/staticmethod/property # TODO: wrap __dict__ for old-style classes? if isinstance(which, basestring): which_func = lambda attr_name, attr_val: attr_name == which elif callable(getattr(which, '__contains__', None)): which_func = lambda attr_name, attr_val: attr_name in which elif which is None or callable(which): which_func = which else: raise TypeError('expected attr name(s) or callable, not: %r' % which) label = label or hex(id(obj)) if isinstance(events, basestring): events = [events] do_get = not events or 'get' in events do_set = not events or 'set' in events do_del = not events or 'del' in events do_call = not events or 'call' in events do_raise = not events or 'raise' in events do_return = not events or 'return' in events def wrap_method(attr_name, func, _hook=hook, _label=label): def wrapped(*a, **kw): a = a[1:] if do_call: hook(event='call', label=_label, obj=obj, attr_name=attr_name, args=a, kwargs=kw) if do_raise: try: ret = func(*a, **kw) except: if not hook(event='raise', label=_label, obj=obj, attr_name=attr_name, args=a, kwargs=kw, result=sys.exc_info()): raise else: ret = func(*a, **kw) if do_return: hook(event='return', label=_label, obj=obj, attr_name=attr_name, args=a, kwargs=kw, result=ret) return ret wrapped.__name__ = func.__name__ wrapped.__doc__ = func.__doc__ try: wrapped.__module__ = func.__module__ except Exception: pass try: if func.__dict__: wrapped.__dict__.update(func.__dict__) except Exception: pass return wrapped def __getattribute__(self, attr_name): ret = type(obj).__getattribute__(obj, attr_name) if callable(ret): # wrap any bound methods ret = type(obj).__getattribute__(self, attr_name) if do_get: hook('get', label, obj, attr_name, (), {}, result=ret) return ret def __setattr__(self, attr_name, value): type(obj).__setattr__(obj, attr_name, value) if do_set: hook('set', label, obj, attr_name, (value,), {}) return def __delattr__(self, attr_name): type(obj).__delattr__(obj, attr_name) if do_del: hook('del', label, obj, attr_name, (), {}) return attrs = {} for attr_name in dir(obj): try: attr_val = getattr(obj, attr_name) except Exception: continue if not callable(attr_val) or attr_name in ('__new__',): continue elif which_func and not which_func(attr_name, attr_val): continue if attr_name == '__getattribute__': wrapped_method = __getattribute__ elif attr_name == '__setattr__': wrapped_method = __setattr__ elif attr_name == '__delattr__': wrapped_method = __delattr__ else: wrapped_method = wrap_method(attr_name, attr_val) attrs[attr_name] = wrapped_method cls_name = obj.__class__.__name__ if cls_name == cls_name.lower(): type_name = 'traced_' + cls_name else: type_name = 'Traced' + cls_name if hasattr(obj, '__mro__'): bases = (obj.__class__,) else: # need new-style class for even basic wrapping of callables to # work. getattribute won't work for old-style classes of course. bases = (obj.__class__, object) trace_type = type(type_name, bases, attrs) for cls in trace_type.__mro__: try: return cls.__new__(trace_type) except Exception: pass raise TypeError('unable to wrap_trace %r instance %r' % (obj.__class__, obj)) if __name__ == '__main__': obj = wrap_trace({}) obj['hi'] = 'hello' obj.fail import pdb;pdb.set_trace() boltons-16.4.1/boltons/iterutils.py0000644000175000017500000010020212727357214015450 0ustar hlehle# -*- coding: utf-8 -*- """:mod:`itertools` is full of great examples of Python generator usage. However, there are still some critical gaps. ``iterutils`` fills many of those gaps with featureful, tested, and Pythonic solutions. Many of the functions below have two versions, one which returns an iterator (denoted by the ``*_iter`` naming pattern), and a shorter-named convenience form that returns a list. Some of the following are based on examples in itertools docs. """ import math import random import itertools from collections import Mapping, Sequence, Set, ItemsView try: from typeutils import make_sentinel _UNSET = make_sentinel('_UNSET') _REMAP_EXIT = make_sentinel('_REMAP_EXIT') except ImportError: _REMAP_EXIT = object() _UNSET = object() try: from itertools import izip except ImportError: # Python 3 compat basestring = (str, bytes) izip, xrange = zip, range def is_iterable(obj): """Similar in nature to :func:`callable`, ``is_iterable`` returns ``True`` if an object is `iterable`_, ``False`` if not. >>> is_iterable([]) True >>> is_iterable(object()) False .. _iterable: https://docs.python.org/2/glossary.html#term-iterable """ try: iter(obj) except TypeError: return False return True def is_scalar(obj): """A near-mirror of :func:`is_iterable`. Returns ``False`` if an object is an iterable container type. Strings are considered scalar as well, because strings are more often treated as whole values as opposed to iterables of 1-character substrings. >>> is_scalar(object()) True >>> is_scalar(range(10)) False >>> is_scalar('hello') True """ return not is_iterable(obj) or isinstance(obj, basestring) def is_collection(obj): """The opposite of :func:`is_scalar`. Returns ``True`` if an object is an iterable other than a string. >>> is_collection(object()) False >>> is_collection(range(10)) True >>> is_collection('hello') False """ return is_iterable(obj) and not isinstance(obj, basestring) def split(src, sep=None, maxsplit=None): """Splits an iterable based on a separator. Like :meth:`str.split`, but for all iterables. Returns a list of lists. >>> split(['hi', 'hello', None, None, 'sup', None, 'soap', None]) [['hi', 'hello'], ['sup'], ['soap']] See :func:`split_iter` docs for more info. """ return list(split_iter(src, sep, maxsplit)) def split_iter(src, sep=None, maxsplit=None): """Splits an iterable based on a separator, *sep*, a max of *maxsplit* times (no max by default). *sep* can be: * a single value * an iterable of separators * a single-argument callable that returns True when a separator is encountered ``split_iter()`` yields lists of non-separator values. A separator will never appear in the output. >>> list(split_iter(['hi', 'hello', None, None, 'sup', None, 'soap', None])) [['hi', 'hello'], ['sup'], ['soap']] Note that ``split_iter`` is based on :func:`str.split`, so if *sep* is ``None``, ``split()`` **groups** separators. If empty lists are desired between two contiguous ``None`` values, simply use ``sep=[None]``: >>> list(split_iter(['hi', 'hello', None, None, 'sup', None])) [['hi', 'hello'], ['sup']] >>> list(split_iter(['hi', 'hello', None, None, 'sup', None], sep=[None])) [['hi', 'hello'], [], ['sup'], []] Using a callable separator: >>> falsy_sep = lambda x: not x >>> list(split_iter(['hi', 'hello', None, '', 'sup', False], falsy_sep)) [['hi', 'hello'], [], ['sup'], []] See :func:`split` for a list-returning version. """ if not is_iterable(src): raise TypeError('expected an iterable') if maxsplit is not None: maxsplit = int(maxsplit) if maxsplit == 0: yield [src] return if callable(sep): sep_func = sep elif not is_scalar(sep): sep = frozenset(sep) sep_func = lambda x: x in sep else: sep_func = lambda x: x == sep cur_group = [] split_count = 0 for s in src: if maxsplit is not None and split_count >= maxsplit: sep_func = lambda x: False if sep_func(s): if sep is None and not cur_group: # If sep is none, str.split() "groups" separators # check the str.split() docs for more info continue split_count += 1 yield cur_group cur_group = [] else: cur_group.append(s) if cur_group or sep is not None: yield cur_group return def chunked(src, size, count=None, **kw): """Returns a list of *count* chunks, each with *size* elements, generated from iterable *src*. If *src* is not evenly divisible by *size*, the final chunk will have fewer than *size* elements. Provide the *fill* keyword argument to provide a pad value and enable padding, otherwise no padding will take place. >>> chunked(range(10), 3) [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] >>> chunked(range(10), 3, fill=None) [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, None, None]] >>> chunked(range(10), 3, count=2) [[0, 1, 2], [3, 4, 5]] See :func:`chunked_iter` for more info. """ chunk_iter = chunked_iter(src, size, **kw) if count is None: return list(chunk_iter) else: return list(itertools.islice(chunk_iter, count)) def chunked_iter(src, size, **kw): """Generates *size*-sized chunks from *src* iterable. Unless the optional *fill* keyword argument is provided, iterables not even divisible by *size* will have a final chunk that is smaller than *size*. >>> list(chunked_iter(range(10), 3)) [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] >>> list(chunked_iter(range(10), 3, fill=None)) [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, None, None]] Note that ``fill=None`` in fact uses ``None`` as the fill value. """ # TODO: add count kwarg? if not is_iterable(src): raise TypeError('expected an iterable') size = int(size) if size <= 0: raise ValueError('expected a positive integer chunk size') do_fill = True try: fill_val = kw.pop('fill') except KeyError: do_fill = False fill_val = None if kw: raise ValueError('got unexpected keyword arguments: %r' % kw.keys()) if not src: return postprocess = lambda chk: chk if isinstance(src, basestring): postprocess = lambda chk, _sep=type(src)(): _sep.join(chk) src_iter = iter(src) while True: cur_chunk = list(itertools.islice(src_iter, size)) if not cur_chunk: break lc = len(cur_chunk) if lc < size and do_fill: cur_chunk[lc:] = [fill_val] * (size - lc) yield postprocess(cur_chunk) return def pairwise(src): """Convenience function for calling :func:`windowed` on *src*, with *size* set to 2. >>> pairwise(range(5)) [(0, 1), (1, 2), (2, 3), (3, 4)] >>> pairwise([]) [] The number of pairs is always one less than the number of elements in the iterable passed in, except on empty inputs, which returns an empty list. """ return windowed(src, 2) def pairwise_iter(src): """Convenience function for calling :func:`windowed_iter` on *src*, with *size* set to 2. >>> list(pairwise_iter(range(5))) [(0, 1), (1, 2), (2, 3), (3, 4)] >>> list(pairwise_iter([])) [] The number of pairs is always one less than the number of elements in the iterable passed in, or zero, when *src* is empty. """ return windowed_iter(src, 2) def windowed(src, size): """Returns tuples with exactly length *size*. If the iterable is too short to make a window of length *size*, no tuples are returned. See :func:`windowed_iter` for more. """ return list(windowed_iter(src, size)) def windowed_iter(src, size): """Returns tuples with length *size* which represent a sliding window over iterable *src*. >>> list(windowed_iter(range(7), 3)) [(0, 1, 2), (1, 2, 3), (2, 3, 4), (3, 4, 5), (4, 5, 6)] If the iterable is too short to make a window of length *size*, then no window tuples are returned. >>> list(windowed_iter(range(3), 5)) [] """ # TODO: lists? (for consistency) tees = itertools.tee(src, size) try: for i, t in enumerate(tees): for _ in xrange(i): next(t) except StopIteration: return izip([]) return izip(*tees) def xfrange(stop, start=None, step=1.0): """Same as :func:`frange`, but generator-based instead of returning a list. >>> tuple(xfrange(1, 3, step=0.75)) (1.0, 1.75, 2.5) See :func:`frange` for more details. """ if not step: raise ValueError('step must be non-zero') if start is None: start, stop = 0.0, stop * 1.0 else: # swap when all args are used stop, start = start * 1.0, stop * 1.0 cur = start while cur < stop: yield cur cur += step def frange(stop, start=None, step=1.0): """A :func:`range` clone for float-based ranges. >>> frange(5) [0.0, 1.0, 2.0, 3.0, 4.0] >>> frange(6, step=1.25) [0.0, 1.25, 2.5, 3.75, 5.0] >>> frange(100.5, 101.5, 0.25) [100.5, 100.75, 101.0, 101.25] >>> frange(5, 0) [] >>> frange(5, 0, step=-1.25) [5.0, 3.75, 2.5, 1.25] """ if not step: raise ValueError('step must be non-zero') if start is None: start, stop = 0.0, stop * 1.0 else: # swap when all args are used stop, start = start * 1.0, stop * 1.0 count = int(math.ceil((stop - start) / step)) ret = [None] * count if not ret: return ret ret[0] = start for i in xrange(1, count): ret[i] = ret[i - 1] + step return ret def backoff(start, stop, count=None, factor=2.0, jitter=False): """Returns a list of geometrically-increasing floating-point numbers, suitable for usage with `exponential backoff`_. Exactly like :func:`backoff_iter`, but without the ``'repeat'`` option for *count*. See :func:`backoff_iter` for more details. .. _exponential backoff: https://en.wikipedia.org/wiki/Exponential_backoff >>> backoff(1, 10) [1.0, 2.0, 4.0, 8.0, 10.0] """ if count == 'repeat': raise ValueError("'repeat' supported in backoff_iter, not backoff") return list(backoff_iter(start, stop, count=count, factor=factor, jitter=jitter)) def backoff_iter(start, stop, count=None, factor=2.0, jitter=False): """Generates a sequence of geometrically-increasing floats, suitable for usage with `exponential backoff`_. Starts with *start*, increasing by *factor* until *stop* is reached, optionally stopping iteration once *count* numbers are yielded. *factor* defaults to 2. In general retrying with properly-configured backoff creates a better-behaved component for a larger service ecosystem. .. _exponential backoff: https://en.wikipedia.org/wiki/Exponential_backoff >>> list(backoff_iter(1.0, 10.0, count=5)) [1.0, 2.0, 4.0, 8.0, 10.0] >>> list(backoff_iter(1.0, 10.0, count=8)) [1.0, 2.0, 4.0, 8.0, 10.0, 10.0, 10.0, 10.0] >>> list(backoff_iter(0.25, 100.0, factor=10)) [0.25, 2.5, 25.0, 100.0] A simplified usage example: .. code-block:: python for timeout in backoff_iter(0.25, 5.0): try: res = network_call() break except Exception as e: log(e) time.sleep(timeout) An enhancement for large-scale systems would be to add variation, or *jitter*, to timeout values. This is done to avoid a thundering herd on the receiving end of the network call. Finally, for *count*, the special value ``'repeat'`` can be passed to continue yielding indefinitely. Args: start (float): Positive number for baseline. stop (float): Positive number for maximum. count (int): Number of steps before stopping iteration. Defaults to the number of steps between *start* and *stop*. Pass the string, `'repeat'`, to continue iteration indefinitely. factor (float): Rate of exponential increase. Defaults to `2.0`, e.g., `[1, 2, 4, 8, 16]`. jitter (float): A factor between `-1.0` and `1.0`, used to uniformly randomize and thus spread out timeouts in a distributed system, avoiding rhythm effects. Positive values use the base backoff curve as a maximum, negative values use the curve as a minimum. Set to 1.0 or `True` for a jitter approximating Ethernet's time-tested backoff solution. Defaults to `False`. """ start = float(start) stop = float(stop) factor = float(factor) if start < 0.0: raise ValueError('expected start >= 0, not %r' % start) if factor < 1.0: raise ValueError('expected factor >= 1.0, not %r' % factor) if stop == 0.0: raise ValueError('expected stop >= 0') if stop < start: raise ValueError('expected stop >= start, not %r' % stop) if count is None: denom = start if start else 1 count = 1 + math.ceil(math.log(stop/denom, factor)) count = count if start else count + 1 if count != 'repeat' and count < 0: raise ValueError('count must be positive or "repeat", not %r' % count) if jitter: jitter = float(jitter) if not (-1.0 <= jitter <= 1.0): raise ValueError('expected jitter -1 <= j <= 1, not: %r' % jitter) cur, i = start, 0 while count == 'repeat' or i < count: if not jitter: cur_ret = cur elif jitter: cur_ret = cur - (cur * jitter * random.random()) yield cur_ret i += 1 if cur == 0: cur = 1 elif cur < stop: cur *= factor if cur > stop: cur = stop return def bucketize(src, key=None): """Group values in the *src* iterable by the value returned by *key*, which defaults to :class:`bool`, grouping values by truthiness. >>> bucketize(range(5)) {False: [0], True: [1, 2, 3, 4]} >>> is_odd = lambda x: x % 2 == 1 >>> bucketize(range(5), is_odd) {False: [0, 2, 4], True: [1, 3]} Value lists are not deduplicated: >>> bucketize([None, None, None, 'hello']) {False: [None, None, None], True: ['hello']} Note in these examples there were at most two keys, ``True`` and ``False``, and each key present has a list with at least one item. See :func:`partition` for a version specialized for binary use cases. """ if not is_iterable(src): raise TypeError('expected an iterable') if key is None: key = bool if not callable(key): raise TypeError('expected callable key function') ret = {} for val in src: keyval = key(val) ret.setdefault(keyval, []).append(val) return ret def partition(src, key=None): """No relation to :meth:`str.partition`, ``partition`` is like :func:`bucketize`, but for added convenience returns a tuple of ``(truthy_values, falsy_values)``. >>> nonempty, empty = partition(['', '', 'hi', '', 'bye']) >>> nonempty ['hi', 'bye'] *key* defaults to :class:`bool`, but can be carefully overridden to use any function that returns either ``True`` or ``False``. >>> import string >>> is_digit = lambda x: x in string.digits >>> decimal_digits, hexletters = partition(string.hexdigits, is_digit) >>> ''.join(decimal_digits), ''.join(hexletters) ('0123456789', 'abcdefABCDEF') """ bucketized = bucketize(src, key) return bucketized.get(True, []), bucketized.get(False, []) def unique(src, key=None): """``unique()`` returns a list of unique values, as determined by *key*, in the order they first appeared in the input iterable, *src*. >>> ones_n_zeros = '11010110001010010101010' >>> ''.join(unique(ones_n_zeros)) '10' See :func:`unique_iter` docs for more details. """ return list(unique_iter(src, key)) def unique_iter(src, key=None): """Yield unique elements from the iterable, *src*, based on *key*, in the order in which they first appeared in *src*. >>> repetitious = [1, 2, 3] * 10 >>> list(unique_iter(repetitious)) [1, 2, 3] By default, *key* is the object itself, but *key* can either be a callable or, for convenience, a string name of the attribute on which to uniqueify objects, falling back on identity when the attribute is not present. >>> pleasantries = ['hi', 'hello', 'ok', 'bye', 'yes'] >>> list(unique_iter(pleasantries, key=lambda x: len(x))) ['hi', 'hello', 'bye'] """ if not is_iterable(src): raise TypeError('expected an iterable, not %r' % type(src)) if key is None: key_func = lambda x: x elif callable(key): key_func = key elif isinstance(key, basestring): key_func = lambda x: getattr(x, key, x) else: raise TypeError('"key" expected a string or callable, not %r' % key) seen = set() for i in src: k = key_func(i) if k not in seen: seen.add(k) yield i return def one(src, default=None, key=None): """Along the same lines as builtins, :func:`all` and :func:`any`, and similar to :func:`first`, ``one()`` returns the single object in the given iterable *src* that evaluates to ``True``, as determined by callable *key*. If unset, *key* defaults to :class:`bool`. If no such objects are found, *default* is returned. If *default* is not passed, ``None`` is returned. If *src* has more than one object that evaluates to ``True``, or if there is no object that fulfills such condition, return ``False``. It's like an `XOR`_ over an iterable. >>> one((True, False, False)) True >>> one((True, False, True)) >>> one((0, 0, 'a')) 'a' >>> one((0, False, None)) >>> one((True, True), default=False) False >>> bool(one(('', 1))) True >>> one((10, 20, 30, 42), key=lambda i: i > 40) 42 See `Martín Gaitán's original repo`_ for further use cases. .. _Martín Gaitán's original repo: https://github.com/mgaitan/one .. _XOR: https://en.wikipedia.org/wiki/Exclusive_or """ the_one = default for i in src: if key(i) if key else i: if the_one: return default the_one = i return the_one def first(iterable, default=None, key=None): """Return first element of *iterable* that evaluates to ``True``, else return ``None`` or optional *default*. Similar to :func:`one`. >>> first([0, False, None, [], (), 42]) 42 >>> first([0, False, None, [], ()]) is None True >>> first([0, False, None, [], ()], default='ohai') 'ohai' >>> import re >>> m = first(re.match(regex, 'abc') for regex in ['b.*', 'a(.*)']) >>> m.group(1) 'bc' The optional *key* argument specifies a one-argument predicate function like that used for *filter()*. The *key* argument, if supplied, should be in keyword form. For example, finding the first even number in an iterable: >>> first([1, 1, 3, 4, 5], key=lambda x: x % 2 == 0) 4 Contributed by Hynek Schlawack, author of `the original standalone module`_. .. _the original standalone module: https://github.com/hynek/first """ if key is None: for el in iterable: if el: return el else: for el in iterable: if key(el): return el return default def same(iterable, ref=_UNSET): """``same()`` returns ``True`` when all values in *iterable* are equal to one another, or optionally a reference value, *ref*. Similar to :func:`all` and :func:`any` in that it evaluates an iterable and returns a :class:`bool`. ``same()`` returns ``True`` for empty iterables. >>> same([]) True >>> same([1]) True >>> same(['a', 'a', 'a']) True >>> same(range(20)) False >>> same([[], []]) True >>> same([[], []], ref='test') False """ iterator = iter(iterable) if ref is _UNSET: try: ref = next(iterator) except StopIteration: return True # those that were there were all equal for val in iterator: if val != ref: return False # short circuit on first unequal value return True def default_visit(path, key, value): # print('visit(%r, %r, %r)' % (path, key, value)) return key, value # enable the extreme: monkeypatching iterutils with a different default_visit _orig_default_visit = default_visit def default_enter(path, key, value): # print('enter(%r, %r)' % (key, value)) try: iter(value) except TypeError: return value, False if isinstance(value, basestring): return value, False elif isinstance(value, Mapping): return value.__class__(), ItemsView(value) elif isinstance(value, Sequence): return value.__class__(), enumerate(value) elif isinstance(value, Set): return value.__class__(), enumerate(value) return value, False def default_exit(path, key, old_parent, new_parent, new_items): # print('exit(%r, %r, %r, %r, %r)' # % (path, key, old_parent, new_parent, new_items)) ret = new_parent if isinstance(new_parent, Mapping): new_parent.update(new_items) elif isinstance(new_parent, Sequence): vals = [v for i, v in new_items] try: new_parent.extend(vals) except AttributeError: ret = new_parent.__class__(vals) # tuples elif isinstance(new_parent, Set): vals = [v for i, v in new_items] try: new_parent.update(new_items) except AttributeError: ret = new_parent.__class__(vals) # frozensets else: raise RuntimeError('unexpected iterable type: %r' % type(new_parent)) return ret def remap(root, visit=default_visit, enter=default_enter, exit=default_exit, **kwargs): """The remap ("recursive map") function is used to traverse and transform nested structures. Lists, tuples, sets, and dictionaries are just a few of the data structures nested into heterogenous tree-like structures that are so common in programming. Unfortunately, Python's built-in ways to manipulate collections are almost all flat. List comprehensions may be fast and succinct, but they do not recurse, making it tedious to apply quick changes or complex transforms to real-world data. remap goes where list comprehensions cannot. Here's an example of removing all Nones from some data: >>> from pprint import pprint >>> reviews = {'Star Trek': {'TNG': 10, 'DS9': 8.5, 'ENT': None}, ... 'Babylon 5': 6, 'Dr. Who': None} >>> pprint(remap(reviews, lambda p, k, v: v is not None)) {'Babylon 5': 6, 'Star Trek': {'DS9': 8.5, 'TNG': 10}} Notice how both Nones have been removed despite the nesting in the dictionary. Not bad for a one-liner, and that's just the beginning. See `this remap cookbook`_ for more delicious recipes. .. _this remap cookbook: http://sedimental.org/remap.html remap takes four main arguments: the object to traverse and three optional callables which determine how the remapped object will be created. Args: root: The target object to traverse. By default, remap supports iterables like :class:`list`, :class:`tuple`, :class:`dict`, and :class:`set`, but any object traversable by *enter* will work. visit (callable): This function is called on every item in *root*. It must accept three positional arguments, *path*, *key*, and *value*. *path* is simply a tuple of parents' keys. *visit* should return the new key-value pair. It may also return ``True`` as shorthand to keep the old item unmodified, or ``False`` to drop the item from the new structure. *visit* is called after *enter*, on the new parent. The *visit* function is called for every item in root, including duplicate items. For traversable values, it is called on the new parent object, after all its children have been visited. The default visit behavior simply returns the key-value pair unmodified. enter (callable): This function controls which items in *root* are traversed. It accepts the same arguments as *visit*: the path, the key, and the value of the current item. It returns a pair of the blank new parent, and an iterator over the items which should be visited. If ``False`` is returned instead of an iterator, the value will not be traversed. The *enter* function is only called once per unique value. The default enter behavior support mappings, sequences, and sets. Strings and all other iterables will not be traversed. exit (callable): This function determines how to handle items once they have been visited. It gets the same three arguments as the other functions -- *path*, *key*, *value* -- plus two more: the blank new parent object returned from *enter*, and a list of the new items, as remapped by *visit*. Like *enter*, the *exit* function is only called once per unique value. The default exit behavior is to simply add all new items to the new parent, e.g., using :meth:`list.extend` and :meth:`dict.update` to add to the new parent. Immutable objects, such as a :class:`tuple` or :class:`namedtuple`, must be recreated from scratch, but use the same type as the new parent passed back from the *enter* function. reraise_visit (bool): A pragmatic convenience for the *visit* callable. When set to ``False``, remap ignores any errors raised by the *visit* callback. Items causing exceptions are kept. See examples for more details. remap is designed to cover the majority of cases with just the *visit* callable. While passing in multiple callables is very empowering, remap is designed so very few cases should require passing more than one function. When passing *enter* and *exit*, it's common and easiest to build on the default behavior. Simply add ``from boltons.iterutils import default_enter`` (or ``default_exit``), and have your enter/exit function call the default behavior before or after your custom logic. See `this example`_. Duplicate and self-referential objects (aka reference loops) are automatically handled internally, `as shown here`_. .. _this example: http://sedimental.org/remap.html#sort_all_lists .. _as shown here: http://sedimental.org/remap.html#corner_cases """ # TODO: improve argument formatting in sphinx doc # TODO: enter() return (False, items) to continue traverse but cancel copy? if not callable(visit): raise TypeError('visit expected callable, not: %r' % visit) if not callable(enter): raise TypeError('enter expected callable, not: %r' % enter) if not callable(exit): raise TypeError('exit expected callable, not: %r' % exit) reraise_visit = kwargs.pop('reraise_visit', True) if kwargs: raise TypeError('unexpected keyword arguments: %r' % kwargs.keys()) path, registry, stack = (), {}, [(None, root)] new_items_stack = [] while stack: key, value = stack.pop() id_value = id(value) if key is _REMAP_EXIT: key, new_parent, old_parent = value id_value = id(old_parent) path, new_items = new_items_stack.pop() value = exit(path, key, old_parent, new_parent, new_items) registry[id_value] = value if not new_items_stack: continue elif id_value in registry: value = registry[id_value] else: res = enter(path, key, value) try: new_parent, new_items = res except TypeError: # TODO: handle False? raise TypeError('enter should return a tuple of (new_parent,' ' items_iterator), not: %r' % res) if new_items is not False: # traverse unless False is explicitly passed registry[id_value] = new_parent new_items_stack.append((path, [])) if value is not root: path += (key,) stack.append((_REMAP_EXIT, (key, new_parent, value))) if new_items: stack.extend(reversed(list(new_items))) continue if visit is _orig_default_visit: # avoid function call overhead by inlining identity operation visited_item = (key, value) else: try: visited_item = visit(path, key, value) except Exception: if reraise_visit: raise visited_item = True if visited_item is False: continue # drop elif visited_item is True: visited_item = (key, value) # TODO: typecheck? # raise TypeError('expected (key, value) from visit(),' # ' not: %r' % visited_item) try: new_items_stack[-1][1].append(visited_item) except IndexError: raise TypeError('expected remappable root, not: %r' % root) return value class PathAccessError(KeyError, IndexError, TypeError): # TODO: could maybe get fancy with an isinstance # TODO: should accept an idx argument def __init__(self, exc, seg, path): self.exc = exc self.seg = seg self.path = path def __repr__(self): cn = self.__class__.__name__ return '%s(%r, %r, %r)' % (cn, self.exc, self.seg, self.path) def __str__(self): return ('could not access %r from path %r, got error: %r' % (self.seg, self.path, self.exc)) def get_path(root, path, default=_UNSET): """EAFP is great, but the error message on this isn't: var_key = 'last_key' x['key'][-1]['other_key'][var_key] KeyError: 'last_key' One of get_path's chief aims is to have a good exception that is better than a plain old KeyError: 'missing_key' """ # TODO: integrate default # TODO: listify kwarg? to allow indexing into sets # TODO: raise better error on not iterable? if isinstance(path, basestring): path = path.split('.') cur = root for seg in path: try: cur = cur[seg] except (KeyError, IndexError) as exc: raise PathAccessError(exc, seg, path) except TypeError as exc: # either string index in a list, or a parent that # doesn't support indexing try: seg = int(seg) cur = cur[seg] except (ValueError, KeyError, IndexError, TypeError): raise PathAccessError(exc, seg, path) return cur # TODO: get_path/set_path # TODO: recollect() # TODO: reiter() """ May actually be faster to do an isinstance check for a str path $ python -m timeit -s "x = [1]" "x[0]" 10000000 loops, best of 3: 0.0207 usec per loop $ python -m timeit -s "x = [1]" "try: x[0] \nexcept: pass" 10000000 loops, best of 3: 0.029 usec per loop $ python -m timeit -s "x = [1]" "try: x[1] \nexcept: pass" 1000000 loops, best of 3: 0.315 usec per loop # setting up try/except is fast, only around 0.01us # actually triggering the exception takes almost 10x as long $ python -m timeit -s "x = [1]" "isinstance(x, basestring)" 10000000 loops, best of 3: 0.141 usec per loop $ python -m timeit -s "x = [1]" "isinstance(x, str)" 10000000 loops, best of 3: 0.131 usec per loop $ python -m timeit -s "x = [1]" "try: x.split('.')\n except: pass" 1000000 loops, best of 3: 0.443 usec per loop $ python -m timeit -s "x = [1]" "try: x.split('.') \nexcept AttributeError: pass" 1000000 loops, best of 3: 0.544 usec per loop """ boltons-16.4.1/boltons/dictutils.py0000644000175000017500000005743412727357214015452 0ustar hlehle# -*- coding: utf-8 -*- """Python has a very powerful mapping type at its core: the :class:`dict` type. While versatile and featureful, the :class:`dict` prioritizes simplicity and performance. As a result, it does not retain the order of item insertion [1]_, nor does it store multiple values per key. It is a fast, unordered 1:1 mapping. The :class:`OrderedMultiDict` contrasts to the built-in :class:`dict`, as a relatively maximalist, ordered 1:n subtype of :class:`dict`. Virtually every feature of :class:`dict` has been retooled to be intuitive in the face of this added complexity. Additional methods have been added, such as :class:`collections.Counter`-like functionality. A prime advantage of the :class:`OrderedMultiDict` (OMD) is its non-destructive nature. Data can be added to an :class:`OMD` without being rearranged or overwritten. The property can allow the developer to work more freely with the data, as well as make more assumptions about where input data will end up in the output, all without any extra work. One great example of this is the :meth:`OMD.inverted()` method, which returns a new OMD with the values as keys and the keys as values. All the data and the respective order is still represented in the inverted form, all from an operation which would be outright wrong and reckless with a built-in :class:`dict` or :class:`collections.OrderedDict`. The OMD has been performance tuned to be suitable for a wide range of usages, including as a basic unordered MultiDict. Special thanks to `Mark Williams`_ for all his help. .. [1] As of 2015, `basic dicts on PyPy are ordered `_. .. _Mark Williams: https://github.com/markrwilliams """ from collections import KeysView, ValuesView, ItemsView try: from itertools import izip_longest except ImportError: from itertools import zip_longest as izip_longest try: from typeutils import make_sentinel _MISSING = make_sentinel(var_name='_MISSING') except ImportError: _MISSING = object() PREV, NEXT, KEY, VALUE, SPREV, SNEXT = range(6) __all__ = ['MultiDict', 'OMD', 'OrderedMultiDict'] try: profile except NameError: profile = lambda x: x class OrderedMultiDict(dict): """A MultiDict is a dictionary that can have multiple values per key and the OrderedMultiDict (OMD) is a MultiDict that retains original insertion order. Common use cases include: * handling query strings parsed from URLs * inverting a dictionary to create a reverse index (values to keys) * stacking data from multiple dictionaries in a non-destructive way The OrderedMultiDict constructor is identical to the built-in :class:`dict`, and overall the API is constitutes an intuitive superset of the built-in type: >>> omd = OrderedMultiDict() >>> omd['a'] = 1 >>> omd['b'] = 2 >>> omd.add('a', 3) >>> omd.get('a') 3 >>> omd.getlist('a') [1, 3] Some non-:class:`dict`-like behaviors also make an appearance, such as support for :func:`reversed`: >>> list(reversed(omd)) ['b', 'a'] Note that unlike some other MultiDicts, this OMD gives precedence to the most recent value added. ``omd['a']`` refers to ``3``, not ``1``. >>> omd OrderedMultiDict([('a', 1), ('b', 2), ('a', 3)]) >>> omd.poplast('a') 3 >>> omd OrderedMultiDict([('a', 1), ('b', 2)]) >>> omd.pop('a') 1 >>> omd OrderedMultiDict([('b', 2)]) Note that calling :func:`dict` on an OMD results in a dict of keys to *lists* of values: >>> from pprint import pprint as pp # ensuring proper key ordering >>> omd = OrderedMultiDict([('a', 1), ('b', 2), ('a', 3)]) >>> pp(dict(omd)) {'a': [1, 3], 'b': [2]} Note that modifying those lists will modify the OMD. If you want a safe-to-modify or flat dictionary, use :meth:`OrderedMultiDict.todict()`. >>> pp(omd.todict()) {'a': 3, 'b': 2} >>> pp(omd.todict(multi=True)) {'a': [1, 3], 'b': [2]} With ``multi=False``, items appear with the keys in to original insertion order, alongside the most-recently inserted value for that key. >>> OrderedMultiDict([('a', 1), ('b', 2), ('a', 3)]).items(multi=False) [('a', 3), ('b', 2)] """ def __init__(self, *args, **kwargs): if len(args) > 1: raise TypeError('%s expected at most 1 argument, got %s' % (self.__class__.__name__, len(args))) super(OrderedMultiDict, self).__init__() self._clear_ll() if args: self.update_extend(args[0]) if kwargs: self.update(kwargs) def _clear_ll(self): try: _map = self._map except AttributeError: _map = self._map = {} self.root = [] _map.clear() self.root[:] = [self.root, self.root, None] def _insert(self, k, v): root = self.root cells = self._map.setdefault(k, []) last = root[PREV] cell = [last, root, k, v] last[NEXT] = root[PREV] = cell cells.append(cell) def add(self, k, v): """Add a single value *v* under a key *k*. Existing values under *k* are preserved. """ values = super(OrderedMultiDict, self).setdefault(k, []) self._insert(k, v) values.append(v) def addlist(self, k, v): """Add an iterable of values underneath a specific key, preserving any values already under that key. >>> omd = OrderedMultiDict([('a', -1)]) >>> omd.addlist('a', range(3)) >>> omd OrderedMultiDict([('a', -1), ('a', 0), ('a', 1), ('a', 2)]) Called ``addlist`` for consistency with :meth:`getlist`, but tuples and other sequences and iterables work. """ self_insert = self._insert values = super(OrderedMultiDict, self).setdefault(k, []) for subv in v: self_insert(k, subv) values.extend(v) def get(self, k, default=None): """Return the value for key *k* if present in the dictionary, else *default*. If *default* is not given, ``None`` is returned. This method never raises a :exc:`KeyError`. To get all values under a key, use :meth:`OrderedMultiDict.getlist`. """ return super(OrderedMultiDict, self).get(k, [default])[-1] def getlist(self, k, default=_MISSING): """Get all values for key *k* as a list, if *k* is in the dictionary, else *default*. The list returned is a copy and can be safely mutated. If *default* is not given, an empty :class:`list` is returned. """ try: return super(OrderedMultiDict, self).__getitem__(k)[:] except KeyError: if default is _MISSING: return [] return default def clear(self): "Empty the dictionary." super(OrderedMultiDict, self).clear() self._clear_ll() def setdefault(self, k, default=_MISSING): """If key *k* is in the dictionary, return its value. If not, insert *k* with a value of *default* and return *default*. *default* defaults to ``None``. See :meth:`dict.setdefault` for more information. """ if not super(OrderedMultiDict, self).__contains__(k): self[k] = None if default is _MISSING else default return self[k] def copy(self): "Return a shallow copy of the dictionary." return self.__class__(self.iteritems(multi=True)) @classmethod def fromkeys(cls, keys, default=None): """Create a dictionary from a list of keys, with all the values set to *default*, or ``None`` if *default* is not set. """ return cls([(k, default) for k in keys]) def update(self, E, **F): """Add items from a dictionary or iterable (and/or keyword arguments), overwriting values under an existing key. See :meth:`dict.update` for more details. """ # E and F are throwback names to the dict() __doc__ if E is self: return self_add = self.add if isinstance(E, OrderedMultiDict): for k in E: if k in self: del self[k] for k, v in E.iteritems(multi=True): self_add(k, v) elif hasattr(E, 'keys'): for k in E.keys(): self[k] = E[k] else: seen = set() seen_add = seen.add for k, v in E: if k not in seen and k in self: del self[k] seen_add(k) self_add(k, v) for k in F: self[k] = F[k] return def update_extend(self, E, **F): """Add items from a dictionary, iterable, and/or keyword arguments without overwriting existing items present in the dictionary. Like :meth:`update`, but adds to existing keys instead of overwriting them. """ if E is self: iterator = iter(E.items()) elif isinstance(E, OrderedMultiDict): iterator = E.iteritems(multi=True) elif hasattr(E, 'keys'): iterator = ((k, E[k]) for k in E.keys()) else: iterator = E self_add = self.add for k, v in iterator: self_add(k, v) def __setitem__(self, k, v): if super(OrderedMultiDict, self).__contains__(k): self._remove_all(k) self._insert(k, v) super(OrderedMultiDict, self).__setitem__(k, [v]) def __getitem__(self, k): return super(OrderedMultiDict, self).__getitem__(k)[-1] def __delitem__(self, k): super(OrderedMultiDict, self).__delitem__(k) self._remove_all(k) def __eq__(self, other): if self is other: return True try: if len(other) != len(self): return False except TypeError: return False if isinstance(other, OrderedMultiDict): selfi = self.iteritems(multi=True) otheri = other.iteritems(multi=True) zipped_items = izip_longest(selfi, otheri, fillvalue=(None, None)) for (selfk, selfv), (otherk, otherv) in zipped_items: if selfk != otherk or selfv != otherv: return False if not(next(selfi, _MISSING) is _MISSING and next(otheri, _MISSING) is _MISSING): # leftovers (TODO: watch for StopIteration?) return False return True elif hasattr(other, 'keys'): for selfk in self: try: other[selfk] == self[selfk] except KeyError: return False return True return False def __ne__(self, other): return not (self == other) def pop(self, k, default=_MISSING): """Remove all values under key *k*, returning the most-recently inserted value. Raises :exc:`KeyError` if the key is not present and no *default* is provided. """ try: return self.popall(k)[-1] except KeyError: if default is _MISSING: raise KeyError(k) return default def popall(self, k, default=_MISSING): """Remove all values under key *k*, returning them in the form of a list. Raises :exc:`KeyError` if the key is not present and no *default* is provided. """ super_self = super(OrderedMultiDict, self) if super_self.__contains__(k): self._remove_all(k) if default is _MISSING: return super_self.pop(k) return super_self.pop(k, default) def poplast(self, k=_MISSING, default=_MISSING): """Remove and return the most-recently inserted value under the key *k*, or the most-recently inserted key if *k* is not provided. If no values remain under *k*, it will be removed from the OMD. Raises :exc:`KeyError` if *k* is not present in the dictionary, or the dictionary is empty. """ if k is _MISSING: if self: k = self.root[PREV][KEY] else: raise KeyError('empty %r' % type(self)) try: self._remove(k) except KeyError: if default is _MISSING: raise KeyError(k) return default values = super(OrderedMultiDict, self).__getitem__(k) v = values.pop() if not values: super(OrderedMultiDict, self).__delitem__(k) return v def _remove(self, k): values = self._map[k] cell = values.pop() cell[PREV][NEXT], cell[NEXT][PREV] = cell[NEXT], cell[PREV] if not values: del self._map[k] def _remove_all(self, k): values = self._map[k] while values: cell = values.pop() cell[PREV][NEXT], cell[NEXT][PREV] = cell[NEXT], cell[PREV] del self._map[k] def iteritems(self, multi=False): """Iterate over the OMD's items in insertion order. By default, yields only the most-recently inserted value for each key. Set *multi* to ``True`` to get all inserted items. """ root = self.root curr = root[NEXT] if multi: while curr is not root: yield curr[KEY], curr[VALUE] curr = curr[NEXT] else: for key in self.iterkeys(): yield key, self[key] def iterkeys(self, multi=False): """Iterate over the OMD's keys in insertion order. By default, yields each key once, according to the most recent insertion. Set *multi* to ``True`` to get all keys, including duplicates, in insertion order. """ root = self.root curr = root[NEXT] if multi: while curr is not root: yield curr[KEY] curr = curr[NEXT] else: yielded = set() yielded_add = yielded.add while curr is not root: k = curr[KEY] if k not in yielded: yielded_add(k) yield k curr = curr[NEXT] def itervalues(self, multi=False): """Iterate over the OMD's values in insertion order. By default, yields the most-recently inserted value per unique key. Set *multi* to ``True`` to get all values according to insertion order. """ for k, v in self.iteritems(multi=multi): yield v def todict(self, multi=False): """Gets a basic :class:`dict` of the items in this dictionary. Keys are the same as the OMD, values are the most recently inserted values for each key. Setting the *multi* arg to ``True`` is yields the same result as calling :class:`dict` on the OMD, except that all the value lists are copies that can be safely mutated. """ if multi: return dict([(k, self.getlist(k)) for k in self]) return dict([(k, self[k]) for k in self]) def sorted(self, key=None, reverse=False): """Similar to the built-in :func:`sorted`, except this method returns a new :class:`OrderedMultiDict` sorted by the provided key function, optionally reversed. Args: key (callable): A callable to determine the sort key of each element. The callable should expect an **item** (key-value pair tuple). reverse (bool): Set to ``True`` to reverse the ordering. >>> omd = OrderedMultiDict(zip(range(3), range(3))) >>> omd.sorted(reverse=True) OrderedMultiDict([(2, 2), (1, 1), (0, 0)]) Note that the key function receives an **item** (key-value tuple), so the recommended signature looks like: >>> omd = OrderedMultiDict(zip('hello', 'world')) >>> omd.sorted(key=lambda i: i[1]) # i[0] is the key, i[1] is the val OrderedMultiDict([('o', 'd'), ('l', 'l'), ('e', 'o'), ('h', 'w')]) """ cls = self.__class__ return cls(sorted(self.iteritems(), key=key, reverse=reverse)) def sortedvalues(self, key=None, reverse=False): """Returns a copy of the :class:`OrderedMultiDict` with the same keys in the same order as the original OMD, but the values within each keyspace have been sorted according to *key* and *reverse*. Args: key (callable): A single-argument callable to determine the sort key of each element. The callable should expect an **item** (key-value pair tuple). reverse (bool): Set to ``True`` to reverse the ordering. >>> omd = OrderedMultiDict() >>> omd.addlist('even', [6, 2]) >>> omd.addlist('odd', [1, 5]) >>> omd.add('even', 4) >>> omd.add('odd', 3) >>> somd = omd.sortedvalues() >>> somd.getlist('even') [2, 4, 6] >>> somd.keys(multi=True) == omd.keys(multi=True) True >>> omd == somd False >>> somd OrderedMultiDict([('even', 2), ('even', 4), ('odd', 1), ('odd', 3), ('even', 6), ('odd', 5)]) As demonstrated above, contents and key order are retained. Only value order changes. """ try: superself_iteritems = super(OrderedMultiDict, self).iteritems() except AttributeError: superself_iteritems = super(OrderedMultiDict, self).items() # (not reverse) because they pop off in reverse order for reinsertion sorted_val_map = dict([(k, sorted(v, key=key, reverse=(not reverse))) for k, v in superself_iteritems]) ret = self.__class__() for k in self.iterkeys(multi=True): ret.add(k, sorted_val_map[k].pop()) return ret def inverted(self): """Returns a new :class:`OrderedMultiDict` with values and keys swapped, like creating dictionary transposition or reverse index. Insertion order is retained and all keys and values are represented in the output. >>> omd = OMD([(0, 2), (1, 2)]) >>> omd.inverted().getlist(2) [0, 1] Inverting twice yields a copy of the original: >>> omd.inverted().inverted() OrderedMultiDict([(0, 2), (1, 2)]) """ return self.__class__((v, k) for k, v in self.iteritems(multi=True)) def counts(self): """Returns a mapping from key to number of values inserted under that key. Like :py:class:`collections.Counter`, but returns a new :class:`OrderedMultiDict`. """ # Returns an OMD because Counter/OrderedDict may not be # available, and neither Counter nor dict maintain order. super_getitem = super(OrderedMultiDict, self).__getitem__ return self.__class__((k, len(super_getitem(k))) for k in self) def keys(self, multi=False): """Returns a list containing the output of :meth:`iterkeys`. See that method's docs for more details. """ return list(self.iterkeys(multi=multi)) def values(self, multi=False): """Returns a list containing the output of :meth:`itervalues`. See that method's docs for more details. """ return list(self.itervalues(multi=multi)) def items(self, multi=False): """Returns a list containing the output of :meth:`iteritems`. See that method's docs for more details. """ return list(self.iteritems(multi=multi)) def __iter__(self): return self.iterkeys() def __reversed__(self): root = self.root curr = root[PREV] lengths = {} lengths_sd = lengths.setdefault get_values = super(OrderedMultiDict, self).__getitem__ while curr is not root: k = curr[KEY] vals = get_values(k) if lengths_sd(k, 1) == len(vals): yield k lengths[k] += 1 curr = curr[PREV] def __repr__(self): cn = self.__class__.__name__ kvs = ', '.join([repr((k, v)) for k, v in self.iteritems(multi=True)]) return '%s([%s])' % (cn, kvs) def viewkeys(self): "OMD.viewkeys() -> a set-like object providing a view on OMD's keys" return KeysView(self) def viewvalues(self): "OMD.viewvalues() -> an object providing a view on OMD's values" return ValuesView(self) def viewitems(self): "OMD.viewitems() -> a set-like object providing a view on OMD's items" return ItemsView(self) # A couple of convenient aliases OMD = OrderedMultiDict MultiDict = OrderedMultiDict class FastIterOrderedMultiDict(OrderedMultiDict): """An OrderedMultiDict backed by a skip list. Iteration over keys is faster and uses constant memory but adding duplicate key-value pairs is slower. Brainchild of Mark Williams. """ def _clear_ll(self): # TODO: always reset objects? (i.e., no else block below) try: _map = self._map except AttributeError: _map = self._map = {} self.root = [] _map.clear() self.root[:] = [self.root, self.root, None, None, self.root, self.root] def _insert(self, k, v): root = self.root empty = [] cells = self._map.setdefault(k, empty) last = root[PREV] if cells is empty: cell = [last, root, k, v, last, root] # was the last one skipped? if last[SPREV][SNEXT] is root: last[SPREV][SNEXT] = cell last[NEXT] = last[SNEXT] = root[PREV] = root[SPREV] = cell cells.append(cell) else: # if the previous was skipped, go back to the cell that # skipped it sprev = last[SPREV] if (last[SPREV][SNEXT] is not last) else last cell = [last, root, k, v, sprev, root] # skip me last[SNEXT] = root last[NEXT] = root[PREV] = root[SPREV] = cell cells.append(cell) def _remove(self, k): cells = self._map[k] cell = cells.pop() if not cells: del self._map[k] cell[PREV][SNEXT] = cell[SNEXT] if cell[PREV][SPREV][SNEXT] is cell: cell[PREV][SPREV][SNEXT] = cell[NEXT] elif cell[SNEXT] is cell[NEXT]: cell[SPREV][SNEXT], cell[SNEXT][SPREV] = cell[SNEXT], cell[SPREV] cell[PREV][NEXT], cell[NEXT][PREV] = cell[NEXT], cell[PREV] def _remove_all(self, k): cells = self._map.pop(k) while cells: cell = cells.pop() if cell[PREV][SPREV][SNEXT] is cell: cell[PREV][SPREV][SNEXT] = cell[NEXT] elif cell[SNEXT] is cell[NEXT]: cell[SPREV][SNEXT], cell[SNEXT][SPREV] = cell[SNEXT], cell[SPREV] cell[PREV][NEXT], cell[NEXT][PREV] = cell[NEXT], cell[PREV] cell[PREV][SNEXT] = cell[SNEXT] def iteritems(self, multi=False): next_link = NEXT if multi else SNEXT root = self.root curr = root[next_link] while curr is not root: yield curr[KEY], curr[VALUE] curr = curr[next_link] def iterkeys(self, multi=False): next_link = NEXT if multi else SNEXT root = self.root curr = root[next_link] while curr is not root: yield curr[KEY] curr = curr[next_link] def __reversed__(self): root = self.root curr = root[PREV] while curr is not root: if curr[SPREV][SNEXT] is not curr: curr = curr[SPREV] if curr is root: break yield curr[KEY] curr = curr[PREV] # end dictutils.py boltons-16.4.1/boltons/excutils.py0000644000175000017500000001664212727357214015302 0ustar hlehle# -*- coding: utf-8 -*- import sys import traceback import linecache from collections import namedtuple # TODO: last arg or first arg? (last arg makes it harder to *args # into, but makes it more readable in the default exception # __repr__ output) # TODO: Multiexception wrapper __all__ = ['ExceptionCauseMixin'] class ExceptionCauseMixin(Exception): """ A mixin class for wrapping an exception in another exception, or otherwise indicating an exception was caused by another exception. This is most useful in concurrent or failure-intolerant scenarios, where just because one operation failed, doesn't mean the remainder should be aborted, or that it's the appropriate time to raise exceptions. This is still a work in progress, but an example use case at the bottom of this module. NOTE: when inheriting, you will probably want to put the ExceptionCauseMixin first. Builtin exceptions are not good about calling super() """ cause = None def __new__(cls, *args, **kw): cause = None if args and isinstance(args[0], Exception): cause, args = args[0], args[1:] ret = super(ExceptionCauseMixin, cls).__new__(cls, *args, **kw) ret.cause = cause if cause is None: return ret root_cause = getattr(cause, 'root_cause', None) if root_cause is None: ret.root_cause = cause else: ret.root_cause = root_cause full_trace = getattr(cause, 'full_trace', None) if full_trace is not None: ret.full_trace = list(full_trace) ret._tb = list(cause._tb) ret._stack = list(cause._stack) return ret try: exc_type, exc_value, exc_tb = sys.exc_info() if exc_type is None and exc_value is None: return ret if cause is exc_value or root_cause is exc_value: # handles when cause is the current exception or when # there are multiple wraps while handling the original # exception, but a cause was never provided ret._tb = _extract_from_tb(exc_tb) ret._stack = _extract_from_frame(exc_tb.tb_frame) ret.full_trace = ret._stack[:-1] + ret._tb finally: del exc_tb return ret def get_str(self): """ Get formatted the formatted traceback and exception message. This function exists separately from __str__() because __str__() is somewhat specialized for the built-in traceback module's particular usage. """ ret = [] trace_str = self._get_trace_str() if trace_str: ret.extend(['Traceback (most recent call last):\n', trace_str]) ret.append(self._get_exc_str()) return ''.join(ret) def _get_message(self): args = getattr(self, 'args', []) if self.cause: args = args[1:] if args and args[0]: return args[0] return '' def _get_trace_str(self): if not self.cause: return super(ExceptionCauseMixin, self).__repr__() if self.full_trace: return ''.join(traceback.format_list(self.full_trace)) return '' def _get_exc_str(self, incl_name=True): cause_str = _format_exc(self.root_cause) message = self._get_message() ret = [] if incl_name: ret = [self.__class__.__name__, ': '] if message: ret.extend([message, ' (caused by ', cause_str, ')']) else: ret.extend([' caused by ', cause_str]) return ''.join(ret) def __str__(self): if not self.cause: return super(ExceptionCauseMixin, self).__str__() trace_str = self._get_trace_str() ret = [] if trace_str: message = self._get_message() if message: ret.extend([message, ' --- ']) ret.extend(['Wrapped traceback (most recent call last):\n', trace_str, self._get_exc_str(incl_name=True)]) return ''.join(ret) else: return self._get_exc_str(incl_name=False) def _format_exc(exc, message=None): if message is None: message = exc exc_str = traceback._format_final_exc_line(exc.__class__.__name__, message) return exc_str.rstrip() _BaseTBItem = namedtuple('_BaseTBItem', 'filename, lineno, name, line') class _TBItem(_BaseTBItem): def __repr__(self): ret = super(_TBItem, self).__repr__() ret += ' <%r>' % self.frame_id return ret class _DeferredLine(object): def __init__(self, filename, lineno, module_globals=None): self.filename = filename self.lineno = lineno module_globals = module_globals or {} self.module_globals = dict([(k, v) for k, v in module_globals.items() if k in ('__name__', '__loader__')]) def __eq__(self, other): return (self.lineno, self.filename) == (other.lineno, other.filename) def __ne__(self, other): return (self.lineno, self.filename) != (other.lineno, other.filename) def __str__(self): if hasattr(self, '_line'): return self._line linecache.checkcache(self.filename) line = linecache.getline(self.filename, self.lineno, self.module_globals) if line: line = line.strip() else: line = None self._line = line return line def __repr__(self): return repr(str(self)) def __len__(self): return len(str(self)) def strip(self): return str(self).strip() def _extract_from_frame(f=None, limit=None): ret = [] if f is None: f = sys._getframe(1) # cross-impl yadayada if limit is None: limit = getattr(sys, 'tracebacklimit', 1000) n = 0 while f is not None and n < limit: filename = f.f_code.co_filename lineno = f.f_lineno name = f.f_code.co_name line = _DeferredLine(filename, lineno, f.f_globals) item = _TBItem(filename, lineno, name, line) item.frame_id = id(f) ret.append(item) f = f.f_back n += 1 ret.reverse() return ret def _extract_from_tb(tb, limit=None): ret = [] if limit is None: limit = getattr(sys, 'tracebacklimit', 1000) n = 0 while tb is not None and n < limit: filename = tb.tb_frame.f_code.co_filename lineno = tb.tb_lineno name = tb.tb_frame.f_code.co_name line = _DeferredLine(filename, lineno, tb.tb_frame.f_globals) item = _TBItem(filename, lineno, name, line) item.frame_id = id(tb.tb_frame) ret.append(item) tb = tb.tb_next n += 1 return ret # An Example/Prototest: class MathError(ExceptionCauseMixin, ValueError): pass def whoops_math(): return 1/0 def math_lol(n=0): if n < 3: return math_lol(n=n+1) try: return whoops_math() except ZeroDivisionError as zde: exc = MathError(zde, 'ya done messed up') raise exc def main(): try: math_lol() except ValueError as me: exc = MathError(me, 'hi') raise exc if __name__ == '__main__': try: main() except Exception: import pdb;pdb.post_mortem() raise boltons-16.4.1/boltons/typeutils.py0000644000175000017500000000726112727357214015501 0ustar hlehle# -*- coding: utf-8 -*- """Python's built-in :mod:`functools` module builds several useful utilities on top of Python's first-class function support. ``typeutils`` attempts to do the same for metaprogramming with types and instances. """ from collections import deque _issubclass = issubclass def make_sentinel(name='_MISSING', var_name=None): """Creates and returns a new **instance** of a new class, suitable for usage as a "sentinel", a kind of singleton often used to indicate a value is missing when ``None`` is a valid input. Args: name (str): Name of the Sentinel var_name (str): Set this name to the name of the variable in its respective module enable pickleability. >>> make_sentinel(var_name='_MISSING') _MISSING The most common use cases here in boltons are as default values for optional function arguments, partly because of its less-confusing appearance in automatically generated documentation. Sentinels also function well as placeholders in queues and linked lists. .. note:: By design, additional calls to ``make_sentinel`` with the same values will not produce equivalent objects. >>> make_sentinel('TEST') == make_sentinel('TEST') False >>> type(make_sentinel('TEST')) == type(make_sentinel('TEST')) False """ class Sentinel(object): def __init__(self): self.name = name self.var_name = var_name def __repr__(self): if self.var_name: return self.var_name return '%s(%r)' % (self.__class__.__name__, self.name) if var_name: def __reduce__(self): return self.var_name def __nonzero__(self): return False return Sentinel() def issubclass(subclass, baseclass): """Just like the built-in :func:`issubclass`, this function checks whether *subclass* is inherited from *baseclass*. Unlike the built-in function, this ``issubclass`` will simply return ``False`` if either argument is not suitable (e.g., if *subclass* is not an instance of :class:`type`), instead of raising :exc:`TypeError`. Args: subclass (type): The target class to check. baseclass (type): The base class *subclass* will be checked against. >>> class MyObject(object): pass ... >>> issubclass(MyObject, object) # always a fun fact True >>> issubclass('hi', 'friend') False """ try: return _issubclass(subclass, baseclass) except TypeError: return False def get_all_subclasses(cls): """Recursively finds and returns a :class:`list` of all types inherited from *cls*. >>> class A(object): ... pass ... >>> class B(A): ... pass ... >>> class C(B): ... pass ... >>> class D(A): ... pass ... >>> [t.__name__ for t in get_all_subclasses(A)] ['B', 'D', 'C'] >>> [t.__name__ for t in get_all_subclasses(B)] ['C'] """ try: to_check = deque(cls.__subclasses__()) except (AttributeError, TypeError): raise TypeError('expected type object, not %r' % cls) seen, ret = set(), [] while to_check: cur = to_check.popleft() if cur in seen: continue ret.append(cur) seen.add(cur) to_check.extend(cur.__subclasses__()) return ret class classproperty(object): """Much like a :class:`property`, but the wrapped get function is a class method. For simplicity, only read-only properties are implemented. """ def __init__(self, fn): self.fn = fn def __get__(self, instance, cls): return self.fn(cls) boltons-16.4.1/boltons/ecoutils.py0000644000175000017500000003227012727357214015264 0ustar hlehle# -*- coding: utf-8 -*- """The larger a programming ecosystem gets, the greater the chances of runtime variability become. Currently, Python is one of the most widely deployed high-level programming environments available, making it a viable target for all manner of application. But it's important to know what you're working with. Some basic variations that are common among development machines: * **Executable runtime**: CPython, PyPy, Jython, etc., plus build date and compiler * **Language version**: 2.4, 2.5, 2.6, 2.7... 3.3, 3.4, 3.5 * **Host operating system**: Windows, OS X, Ubuntu, Debian, CentOS, RHEL, etc. * **Features**: 64-bit, IPv6, Unicode character support (UCS-2/UCS-4) * **Built-in library support**: OpenSSL, threading, SQLite, zlib * **User environment**: umask, ulimit, working directory path * **Machine info**: CPU count, hostname, filesystem encoding See the full example profile below for more. ecoutils was created to quantify that variability. ecoutils quickly produces an information-dense description of critical runtime factors, with minimal side effects. In short, ecoutils is like browser and user agent analytics, but for Python environments. Transmission and collection --------------------------- The data is all JSON serializable, and is suitable for sending to a central analytics server. An HTTP-backed service for this can be found at: https://github.com/mahmoud/espymetrics/ Notable omissions ----------------- Due to space constraints (and possibly latency constraints), the following information is deemed not dense enough, and thus omitted: * :data:`sys.path` * full :mod:`sysconfig` * environment variables (:data:`os.environ`) Compatibility ------------- So far ecoutils has has been tested on Python 2.4, 2.5, 2.6, 2.7, 3.4, 3.5, and PyPy. Various versions have been tested on Ubuntu, Debian, RHEL, OS X, FreeBSD, and Windows 7. .. note:: Boltons typically only support back to Python 2.6, but due to its nature, ecoutils extends backwards compatibility to Python 2.4 and 2.5. Profile generation ------------------ Profiles are generated by :func:`ecoutils.get_profile`. When run as a module, ecoutils will call :func:`~ecoutils.get_profile` and print a profile in JSON format:: $ python -m boltons.ecoutils { "_eco_version": "1.0.0", "cpu_count": 4, "cwd": "/home/mahmoud/projects/boltons", "fs_encoding": "UTF-8", "guid": "6b139e7bbf5ad4ed8d4063bf6235b4d2", "hostfqdn": "mahmoud-host", "hostname": "mahmoud-host", "linux_dist_name": "Ubuntu", "linux_dist_version": "14.04", "python": { "argv": "boltons/ecoutils.py", "bin": "/usr/bin/python", "build_date": "Jun 22 2015 17:58:13", "compiler": "GCC 4.8.2", "features": { "64bit": true, "expat": "expat_2.1.0", "ipv6": true, "openssl": "OpenSSL 1.0.1f 6 Jan 2014", "readline": true, "sqlite": "3.8.2", "threading": true, "tkinter": "8.6", "unicode_wide": true, "zlib": "1.2.8" }, "version": "2.7.6 (default, Jun 22 2015, 17:58:13) [GCC 4.8.2]", "version_info": [ 2, 7, 6, "final", 0 ] }, "time_utc": "2016-05-24 07:59:40.473140", "time_utc_offset": -8.0, "ulimit_hard": 4096, "ulimit_soft": 1024, "umask": "002", "uname": { "machine": "x86_64", "node": "mahmoud-host", "processor": "x86_64", "release": "3.13.0-85-generic", "system": "Linux", "version": "#129-Ubuntu SMP Thu Mar 17 20:50:15 UTC 2016" }, "username": "mahmoud" } ``pip install boltons`` and try it yourself! """ import re import os import sys import time import pprint import random import socket import struct import getpass import datetime import platform ECO_VERSION = '1.0.1' # see version history below PY_GT_2 = sys.version_info[0] > 2 try: getrandbits = random.SystemRandom().getrandbits HAVE_URANDOM = True except Exception: HAVE_URANDOM = False getrandbits = random.getrandbits # 128-bit GUID just like a UUID, but backwards compatible to 2.4 INSTANCE_ID = hex(getrandbits(128))[2:-1].lower() IS_64BIT = struct.calcsize("P") > 4 HAVE_UCS4 = getattr(sys, 'maxunicode', 0) > 65536 HAVE_READLINE = True try: import readline except Exception: HAVE_READLINE = False try: import sqlite3 SQLITE_VERSION = sqlite3.sqlite_version except Exception: # note: 2.5 and older have sqlite, but not sqlite3 SQLITE_VERSION = '' try: import ssl try: OPENSSL_VERSION = ssl.OPENSSL_VERSION except AttributeError: # This is a conservative estimate for Python <2.6 # SSL module added in 2006, when 0.9.7 was standard OPENSSL_VERSION = 'OpenSSL >0.8.0' except Exception: OPENSSL_VERSION = '' try: if PY_GT_2: import tkinter else: import Tkinter as tkinter TKINTER_VERSION = str(tkinter.TkVersion) except Exception: TKINTER_VERSION = '' try: import zlib ZLIB_VERSION = zlib.ZLIB_VERSION except Exception: ZLIB_VERSION = '' try: from xml.parsers import expat EXPAT_VERSION = expat.EXPAT_VERSION except Exception: EXPAT_VERSION = '' try: from multiprocessing import cpu_count CPU_COUNT = cpu_count() except Exception: CPU_COUNT = 0 try: import threading HAVE_THREADING = True except Exception: HAVE_THREADING = False try: HAVE_IPV6 = socket.has_ipv6 except Exception: HAVE_IPV6 = False try: from resource import getrlimit, RLIMIT_NOFILE RLIMIT_FDS_SOFT, RLIMIT_FDS_HARD = getrlimit(RLIMIT_NOFILE) except Exception: RLIMIT_FDS_SOFT, RLIMIT_FDS_HARD = 0, 0 START_TIME_INFO = {'time_utc': str(datetime.datetime.utcnow()), 'time_utc_offset': -time.timezone / 3600.0} def get_python_info(): ret = {} ret['argv'] = _escape_shell_args(sys.argv) ret['bin'] = sys.executable # Even though compiler/build_date are already here, they're # actually parsed from the version string. So, in the rare case of # the unparsable version string, we're still transmitting it. ret['version'] = ' '.join(sys.version.split()) ret['compiler'] = platform.python_compiler() ret['build_date'] = platform.python_build()[1] ret['version_info'] = list(sys.version_info) ret['features'] = {'openssl': OPENSSL_VERSION, 'expat': EXPAT_VERSION, 'sqlite': SQLITE_VERSION, 'tkinter': TKINTER_VERSION, 'zlib': ZLIB_VERSION, 'unicode_wide': HAVE_UCS4, 'readline': HAVE_READLINE, '64bit': IS_64BIT, 'ipv6': HAVE_IPV6, 'threading': HAVE_THREADING, 'urandom': HAVE_URANDOM} return ret def get_profile(**kwargs): """The main entrypoint to ecoutils. Calling this will return a JSON-serializable dictionary of information about the current process. It is very unlikely that the information returned will change during the lifetime of the process, and in most cases the majority of the information stays the same between runs as well. :func:`get_profile` takes one optional keyword argument, *scrub*, a :class:`bool` that, if True, blanks out identifiable information. This includes current working directory, hostname, Python executable path, command-line arguments, and username. Values are replaced with '-', but for compatibility keys remain in place. """ scrub = kwargs.pop('scrub', False) if kwargs: raise TypeError('unexpected keyword arguments: %r' % (kwargs.keys(),)) ret = {} try: ret['username'] = getpass.getuser() except Exception: ret['username'] = '' ret['guid'] = str(INSTANCE_ID) ret['hostname'] = socket.gethostname() ret['hostfqdn'] = socket.getfqdn() uname = platform.uname() ret['uname'] = {'system': uname[0], 'node': uname[1], 'release': uname[2], # linux: distro name 'version': uname[3], # linux: kernel version 'machine': uname[4], 'processor': uname[5]} try: linux_dist = platform.linux_distribution() except Exception: linux_dist = ('', '', '') ret['linux_dist_name'] = linux_dist[0] ret['linux_dist_version'] = linux_dist[1] ret['cpu_count'] = CPU_COUNT ret['fs_encoding'] = sys.getfilesystemencoding() ret['ulimit_soft'] = RLIMIT_FDS_SOFT ret['ulimit_hard'] = RLIMIT_FDS_HARD ret['cwd'] = os.getcwd() ret['umask'] = oct(os.umask(os.umask(2))).rjust(3, '0') ret['python'] = get_python_info() ret.update(START_TIME_INFO) ret['_eco_version'] = ECO_VERSION if scrub: # mask identifiable information ret['cwd'] = '-' ret['hostname'] = '-' ret['hostfqdn'] = '-' ret['python']['bin'] = '-' ret['python']['argv'] = '-' ret['uname']['node'] = '-' ret['username'] = '-' return ret _real_safe_repr = pprint._safe_repr def _fake_json_dumps(val, indent=2): # never do this. this is a hack for Python 2.4. Python 2.5 added # the json module for a reason. def _fake_safe_repr(*a, **kw): res, is_read, is_rec = _real_safe_repr(*a, **kw) if res == 'None': res = 'null' if res == 'True': res = 'true' if res == 'False': res = 'false' if not (res.startswith("'") or res.startswith("u'")): res = res else: if res.startswith('u'): res = res[1:] contents = res[1:-1] contents = contents.replace('"', '').replace(r'\"', '') res = '"' + contents + '"' return res, is_read, is_rec pprint._safe_repr = _fake_safe_repr try: ret = pprint.pformat(val, indent=indent) finally: pprint._safe_repr = _real_safe_repr return ret def get_profile_json(indent=False): if indent: indent = 2 else: indent = 0 try: import json def dumps(val, indent): if indent: return json.dumps(val, sort_keys=True, indent=indent) return json.dumps(val, sort_keys=True) except ImportError: def dumps(val, indent): ret = _fake_json_dumps(val, indent=indent) if not indent: ret = re.sub('\n\s*', ' ', ret) return ret data_dict = get_profile() return dumps(data_dict, indent) def main(): print(get_profile_json(indent=True)) ############################################# # The shell escaping copied in from strutils ############################################# def _escape_shell_args(args, sep=' ', style=None): if not style: if sys.platform == 'win32': style = 'cmd' else: style = 'sh' if style == 'sh': return _args2sh(args, sep=sep) elif style == 'cmd': return _args2cmd(args, sep=sep) raise ValueError("style expected one of 'cmd' or 'sh', not %r" % style) _find_sh_unsafe = re.compile(r'[^a-zA-Z0-9_@%+=:,./-]').search def _args2sh(args, sep=' '): # see strutils ret_list = [] for arg in args: if not arg: ret_list.append("''") continue if _find_sh_unsafe(arg) is None: ret_list.append(arg) continue # use single quotes, and put single quotes into double quotes # the string $'b is then quoted as '$'"'"'b' ret_list.append("'" + arg.replace("'", "'\"'\"'") + "'") return ' '.join(ret_list) def _args2cmd(args, sep=' '): # see strutils result = [] needquote = False for arg in args: bs_buf = [] # Add a space to separate this argument from the others if result: result.append(' ') needquote = (" " in arg) or ("\t" in arg) or not arg if needquote: result.append('"') for c in arg: if c == '\\': # Don't know if we need to double yet. bs_buf.append(c) elif c == '"': # Double backslashes. result.append('\\' * len(bs_buf)*2) bs_buf = [] result.append('\\"') else: # Normal char if bs_buf: result.extend(bs_buf) bs_buf = [] result.append(c) # Add remaining backslashes, if any. if bs_buf: result.extend(bs_buf) if needquote: result.extend(bs_buf) result.append('"') return ''.join(result) ############################ # End shell escaping code ############################ if __name__ == '__main__': main() """ ecoutils protocol version history --------------------------------- The version is ECO_VERSION module-level constant, and _eco_version key in the dictionary returned from ecoutils.get_profile(). 1.0.1 - (boltons version 16.3.2+) Remove uuid dependency and add HAVE_URANDOM 1.0.0 - (boltons version 16.3.0-16.3.1) Initial release """ boltons-16.4.1/boltons/mathutils.py0000644000175000017500000000314712727357214015450 0ustar hlehle"""This module provides useful math functions on top of Python's built-in :mod:`math` module. """ from math import ceil as _ceil, floor as _floor import bisect def ceil(x, options=None): """Return the ceiling of *x*. If *options* is set, return the smallest integer or float from *options* that is greater than or equal to *x*. Args: x (int or float): Number to be tested. options (iterable): Optional iterable of arbitrary numbers (ints or floats). >>> VALID_CABLE_CSA = [1.5, 2.5, 4, 6, 10, 25, 35, 50] >>> ceil(3.5, options=VALID_CABLE_CSA) 4 >>> ceil(4, options=VALID_CABLE_CSA) 4 """ if options is None: return _ceil(x) options = sorted(options) i = bisect.bisect_left(options, x) if i == len(options): raise ValueError("no ceil options greater than or equal to: %r" % x) return options[i] def floor(x, options=None): """Return the floor of *x*. If *options* is set, return the largest integer or float from *options* that is less than or equal to *x*. Args: x (int or float): Number to be tested. options (iterable): Optional iterable of arbitrary numbers (ints or floats). >>> VALID_CABLE_CSA = [1.5, 2.5, 4, 6, 10, 25, 35, 50] >>> floor(3.5, options=VALID_CABLE_CSA) 2.5 >>> floor(2.5, options=VALID_CABLE_CSA) 2.5 """ if options is None: return _floor(x) options = sorted(options) i = bisect.bisect_right(options, x) if not i: raise ValueError("no floor options less than or equal to: %r" % x) return options[i - 1] boltons-16.4.1/boltons/timeutils.py0000644000175000017500000004463312727357214015462 0ustar hlehle# -*- coding: utf-8 -*- """Python's :mod:`datetime` module provides some of the most complex and powerful primitives in the Python standard library. Time is nontrivial, but thankfully its support is first-class in Python. ``dateutils`` provides some additional tools for working with time. Additionally, timeutils provides a few basic utilities for working with timezones in Python. The Python :mod:`datetime` module's documentation describes how to create a :class:`~datetime.datetime`-compatible :class:`~datetime.tzinfo` subtype. It even provides a few examples. The following module defines usable forms of the timezones in those docs, as well as a couple other useful ones, :data:`UTC` (aka GMT) and :data:`LocalTZ` (representing the local timezone as configured in the operating system). For timezones beyond these, as well as a higher degree of accuracy in corner cases, check out `pytz`_. .. _pytz: https://pypi.python.org/pypi/pytz """ import re import time import bisect import operator from datetime import tzinfo, timedelta, date, datetime def total_seconds(td): """For those with older versions of Python, a pure-Python implementation of Python 2.7's :meth:`~datetime.timedelta.total_seconds`. Args: td (datetime.timedelta): The timedelta to convert to seconds. Returns: float: total number of seconds >>> td = timedelta(days=4, seconds=33) >>> total_seconds(td) 345633.0 """ a_milli = 1000000.0 td_ds = td.seconds + (td.days * 86400) # 24 * 60 * 60 td_micro = td.microseconds + (td_ds * a_milli) return td_micro / a_milli def dt_to_timestamp(dt): """Converts from a :class:`~datetime.datetime` object to an integer timestamp, suitable interoperation with :func:`time.time` and other `Epoch-based timestamps`. .. _Epoch-based timestamps: https://en.wikipedia.org/wiki/Unix_time >>> abs(round(time.time() - dt_to_timestamp(datetime.utcnow()), 2)) 0.0 ``dt_to_timestamp`` supports both timezone-aware and naïve :class:`~datetime.datetime` objects. Note that it assumes naïve datetime objects are implied UTC, such as those generated with :meth:`datetime.datetime.utcnow`. If your datetime objects are local time, such as those generated with :meth:`datetime.datetime.now`, first convert it using the :meth:`datetime.datetime.replace` method with ``tzinfo=`` :class:`LocalTZ` object in this module, then pass the result of that to ``dt_to_timestamp``. """ if dt.tzinfo: td = dt - EPOCH_AWARE else: td = dt - EPOCH_NAIVE return total_seconds(td) _NONDIGIT_RE = re.compile('\D') def isoparse(iso_str): """Parses the limited subset of `ISO8601-formatted time`_ strings as returned by :meth:`datetime.datetime.isoformat`. >>> epoch_dt = datetime.utcfromtimestamp(0) >>> iso_str = epoch_dt.isoformat() >>> print(iso_str) 1970-01-01T00:00:00 >>> isoparse(iso_str) datetime.datetime(1970, 1, 1, 0, 0) >>> utcnow = datetime.utcnow() >>> utcnow == isoparse(utcnow.isoformat()) True For further datetime parsing, see the `iso8601`_ package for strict ISO parsing and `dateutil`_ package for loose parsing and more. .. _ISO8601-formatted time: https://en.wikipedia.org/wiki/ISO_8601 .. _iso8601: https://pypi.python.org/pypi/iso8601 .. _dateutil: https://pypi.python.org/pypi/python-dateutil """ dt_args = [int(p) for p in _NONDIGIT_RE.split(iso_str)] return datetime(*dt_args) _BOUNDS = [(0, timedelta(seconds=1), 'second'), (1, timedelta(seconds=60), 'minute'), (1, timedelta(seconds=3600), 'hour'), (1, timedelta(days=1), 'day'), (1, timedelta(days=7), 'week'), (2, timedelta(days=30), 'month'), (1, timedelta(days=365), 'year')] _BOUNDS = [(b[0] * b[1], b[1], b[2]) for b in _BOUNDS] _BOUND_DELTAS = [b[0] for b in _BOUNDS] _FLOAT_PATTERN = r'[+-]?\ *(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?' _PARSE_TD_RE = re.compile("((?P%s)\s*(?P\w)\w*)" % _FLOAT_PATTERN) _PARSE_TD_KW_MAP = dict([(unit[0], unit + 's') for _, _, unit in reversed(_BOUNDS[:-2])]) def parse_timedelta(text): """Robustly parses a short text description of a time period into a :class:`datetime.timedelta`. Supports weeks, days, hours, minutes, and seconds, with or without decimal points: Args: text (str): Text to parse. Returns: datetime.timedelta Raises: ValueError: on parse failure. >>> parse_td('1d 2h 3.5m 0s') datetime.timedelta(1, 7410) Also supports full words and whitespace. >>> parse_td('2 weeks 1 day') datetime.timedelta(15) Negative times are supported, too: >>> parse_td('-1.5 weeks 3m 20s') datetime.timedelta(-11, 43400) """ td_kwargs = {} for match in _PARSE_TD_RE.finditer(text): value, unit = match.group('value'), match.group('unit') try: unit_key = _PARSE_TD_KW_MAP[unit] except KeyError: raise ValueError('invalid time unit %r, expected one of %r' % (unit, _PARSE_TD_KW_MAP.keys())) try: value = float(value) except ValueError: raise ValueError('invalid time value for unit %r: %r' % (unit, value)) td_kwargs[unit_key] = value return timedelta(**td_kwargs) parse_td = parse_timedelta # legacy alias def _cardinalize_time_unit(unit, value): # removes dependency on strutils; nice and simple because # all time units cardinalize normally if value == 1: return unit return unit + 's' def decimal_relative_time(d, other=None, ndigits=0, cardinalize=True): """Get a tuple representing the relative time difference between two :class:`~datetime.datetime` objects or one :class:`~datetime.datetime` and now. Args: d (datetime): The first datetime object. other (datetime): An optional second datetime object. If unset, defaults to the current time as determined :meth:`datetime.utcnow`. ndigits (int): The number of decimal digits to round to, defaults to ``0``. cardinalize (bool): Whether to pluralize the time unit if appropriate, defaults to ``True``. Returns: (float, str): A tuple of the :class:`float` difference and respective unit of time, pluralized if appropriate and *cardinalize* is set to ``True``. Unlike :func:`relative_time`, this method's return is amenable to localization into other languages and custom phrasing and formatting. >>> now = datetime.utcnow() >>> decimal_relative_time(now - timedelta(days=1, seconds=3600), now) (1.0, 'day') >>> decimal_relative_time(now - timedelta(seconds=0.002), now, ndigits=5) (0.002, 'seconds') >>> decimal_relative_time(now, now - timedelta(days=900), ndigits=1) (-2.5, 'years') """ if other is None: other = datetime.utcnow() diff = other - d diff_seconds = total_seconds(diff) abs_diff = abs(diff) b_idx = bisect.bisect(_BOUND_DELTAS, abs_diff) - 1 bbound, bunit, bname = _BOUNDS[b_idx] f_diff = diff_seconds / total_seconds(bunit) rounded_diff = round(f_diff, ndigits) if cardinalize: return rounded_diff, _cardinalize_time_unit(bname, abs(rounded_diff)) return rounded_diff, bname def relative_time(d, other=None, ndigits=0): """Get a string representation of the difference between two :class:`~datetime.datetime` objects or one :class:`~datetime.datetime` and the current time. Handles past and future times. Args: d (datetime): The first datetime object. other (datetime): An optional second datetime object. If unset, defaults to the current time as determined :meth:`datetime.utcnow`. ndigits (int): The number of decimal digits to round to, defaults to ``0``. Returns: A short English-language string. >>> now = datetime.utcnow() >>> relative_time(now, ndigits=1) '0 seconds ago' >>> relative_time(now - timedelta(days=1, seconds=36000), ndigits=1) '1.4 days ago' >>> relative_time(now + timedelta(days=7), now, ndigits=1) '1 week from now' """ drt, unit = decimal_relative_time(d, other, ndigits, cardinalize=True) phrase = 'ago' if drt < 0: phrase = 'from now' return '%g %s %s' % (abs(drt), unit, phrase) def strpdate(string, format): """Parse the date string according to the format in `format`. Returns a :class:`date` object. Internally, :meth:`datetime.strptime` is used to parse the string and thus conversion specifiers for time fields (e.g. `%H`) may be provided; these will be parsed but ignored. Args: string (str): The date string to be parsed. format (str): The `strptime`_-style date format string. Returns: datetime.date .. _`strptime`: https://docs.python.org/2/library/datetime.html#strftime-strptime-behavior >>> strpdate('2016-02-14', '%Y-%m-%d') datetime.date(2016, 2, 14) >>> strpdate('26/12 (2015)', '%d/%m (%Y)') datetime.date(2015, 12, 26) >>> strpdate('20151231 23:59:59', '%Y%m%d %H:%M:%S') datetime.date(2015, 12, 31) >>> strpdate('20160101 00:00:00.001', '%Y%m%d %H:%M:%S.%f') datetime.date(2016, 1, 1) """ whence = datetime.strptime(string, format) return whence.date() def daterange(start, stop, step=1, inclusive=False): """In the spirit of :func:`range` and :func:`xrange`, the `daterange` generator that yields a sequence of :class:`~datetime.date` objects, starting at *start*, incrementing by *step*, until *stop* is reached. When *inclusive* is True, the final date may be *stop*, **if** *step* falls evenly on it. By default, *step* is one day. See details below for many more details. Args: start (datetime.date): The starting date The first value in the sequence. stop (datetime.date): The stopping date. By default not included in return. Can be `None` to yield an infinite sequence. step (int): The value to increment *start* by to reach *stop*. Can be an :class:`int` number of days, a :class:`datetime.timedelta`, or a :class:`tuple` of integers, `(year, month, day)`. Positive and negative *step* values are supported. inclusive (bool): Whether or not the *stop* date can be returned. *stop* is only returned when a *step* falls evenly on it. >>> christmas = date(year=2015, month=12, day=25) >>> boxing_day = date(year=2015, month=12, day=26) >>> new_year = date(year=2016, month=1, day=1) >>> for day in daterange(christmas, new_year): ... print(repr(day)) datetime.date(2015, 12, 25) datetime.date(2015, 12, 26) datetime.date(2015, 12, 27) datetime.date(2015, 12, 28) datetime.date(2015, 12, 29) datetime.date(2015, 12, 30) datetime.date(2015, 12, 31) >>> for day in daterange(christmas, boxing_day): ... print(repr(day)) datetime.date(2015, 12, 25) >>> for day in daterange(date(2017, 5, 1), date(2017, 8, 1), ... step=(0, 1, 0), inclusive=True): ... print(repr(day)) datetime.date(2017, 5, 1) datetime.date(2017, 6, 1) datetime.date(2017, 7, 1) datetime.date(2017, 8, 1) *Be careful when using stop=None, as this will yield an infinite sequence of dates.* """ if not isinstance(start, date): raise TypeError("start expected datetime.date instance") if stop and not isinstance(stop, date): raise TypeError("stop expected datetime.date instance or None") try: y_step, m_step, d_step = step except TypeError: y_step, m_step, d_step = 0, 0, step else: y_step, m_step = int(y_step), int(m_step) if isinstance(d_step, int): d_step = timedelta(days=int(d_step)) elif isinstance(d_step, timedelta): pass else: raise ValueError('step expected int, timedelta, or tuple' ' (year, month, day), not: %r' % step) if stop is None: finished = lambda t: False elif start < stop: finished = operator.gt if inclusive else operator.ge else: finished = operator.lt if inclusive else operator.le now = start while not finished(now, stop): yield now if y_step or m_step: m_y_step, cur_month = divmod(now.month + m_step, 12) now = now.replace(year=now.year + y_step + m_y_step, month=cur_month or 12) now = now + d_step return # Timezone support (brought in from tzutils) ZERO = timedelta(0) HOUR = timedelta(hours=1) class ConstantTZInfo(tzinfo): """ A :class:`~datetime.tzinfo` subtype whose *offset* remains constant (no daylight savings). Args: name (str): Name of the timezone. offset (datetime.timedelta): Offset of the timezone. """ def __init__(self, name="ConstantTZ", offset=ZERO): self.name = name self.offset = offset @property def utcoffset_hours(self): return total_seconds(self.offset) / (60 * 60) def utcoffset(self, dt): return self.offset def tzname(self, dt): return self.name def dst(self, dt): return ZERO def __repr__(self): cn = self.__class__.__name__ return '%s(name=%r, offset=%r)' % (cn, self.name, self.offset) UTC = ConstantTZInfo('UTC') EPOCH_AWARE = datetime.fromtimestamp(0, UTC) EPOCH_NAIVE = datetime.utcfromtimestamp(0) class LocalTZInfo(tzinfo): """The ``LocalTZInfo`` type takes data available in the time module about the local timezone and makes a practical :class:`datetime.tzinfo` to represent the timezone settings of the operating system. For a more in-depth integration with the operating system, check out `tzlocal`_. It builds on `pytz`_ and implements heuristics for many versions of major operating systems to provide the official ``pytz`` tzinfo, instead of the LocalTZ generalization. .. _tzlocal: https://pypi.python.org/pypi/tzlocal .. _pytz: https://pypi.python.org/pypi/pytz """ _std_offset = timedelta(seconds=-time.timezone) _dst_offset = _std_offset if time.daylight: _dst_offset = timedelta(seconds=-time.altzone) def is_dst(self, dt): dt_t = (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.weekday(), 0, -1) local_t = time.localtime(time.mktime(dt_t)) return local_t.tm_isdst > 0 def utcoffset(self, dt): if self.is_dst(dt): return self._dst_offset return self._std_offset def dst(self, dt): if self.is_dst(dt): return self._dst_offset - self._std_offset return ZERO def tzname(self, dt): return time.tzname[self.is_dst(dt)] def __repr__(self): return '%s()' % self.__class__.__name__ LocalTZ = LocalTZInfo() def _first_sunday_on_or_after(dt): days_to_go = 6 - dt.weekday() if days_to_go: dt += timedelta(days_to_go) return dt # US DST Rules # # This is a simplified (i.e., wrong for a few cases) set of rules for US # DST start and end times. For a complete and up-to-date set of DST rules # and timezone definitions, visit the Olson Database (or try pytz): # http://www.twinsun.com/tz/tz-link.htm # http://sourceforge.net/projects/pytz/ (might not be up-to-date) # # In the US, since 2007, DST starts at 2am (standard time) on the second # Sunday in March, which is the first Sunday on or after Mar 8. DSTSTART_2007 = datetime(1, 3, 8, 2) # and ends at 2am (DST time; 1am standard time) on the first Sunday of Nov. DSTEND_2007 = datetime(1, 11, 1, 1) # From 1987 to 2006, DST used to start at 2am (standard time) on the first # Sunday in April and to end at 2am (DST time; 1am standard time) on the last # Sunday of October, which is the first Sunday on or after Oct 25. DSTSTART_1987_2006 = datetime(1, 4, 1, 2) DSTEND_1987_2006 = datetime(1, 10, 25, 1) # From 1967 to 1986, DST used to start at 2am (standard time) on the last # Sunday in April (the one on or after April 24) and to end at 2am (DST time; # 1am standard time) on the last Sunday of October, which is the first Sunday # on or after Oct 25. DSTSTART_1967_1986 = datetime(1, 4, 24, 2) DSTEND_1967_1986 = DSTEND_1987_2006 class USTimeZone(tzinfo): """Copied directly from the Python docs, the ``USTimeZone`` is a :class:`datetime.tzinfo` subtype used to create the :data:`Eastern`, :data:`Central`, :data:`Mountain`, and :data:`Pacific` tzinfo types. """ def __init__(self, hours, reprname, stdname, dstname): self.stdoffset = timedelta(hours=hours) self.reprname = reprname self.stdname = stdname self.dstname = dstname def __repr__(self): return self.reprname def tzname(self, dt): if self.dst(dt): return self.dstname else: return self.stdname def utcoffset(self, dt): return self.stdoffset + self.dst(dt) def dst(self, dt): if dt is None or dt.tzinfo is None: # An exception may be sensible here, in one or both cases. # It depends on how you want to treat them. The default # fromutc() implementation (called by the default astimezone() # implementation) passes a datetime with dt.tzinfo is self. return ZERO assert dt.tzinfo is self # Find start and end times for US DST. For years before 1967, return # ZERO for no DST. if 2006 < dt.year: dststart, dstend = DSTSTART_2007, DSTEND_2007 elif 1986 < dt.year < 2007: dststart, dstend = DSTSTART_1987_2006, DSTEND_1987_2006 elif 1966 < dt.year < 1987: dststart, dstend = DSTSTART_1967_1986, DSTEND_1967_1986 else: return ZERO start = _first_sunday_on_or_after(dststart.replace(year=dt.year)) end = _first_sunday_on_or_after(dstend.replace(year=dt.year)) # Can't compare naive to aware objects, so strip the timezone # from dt first. if start <= dt.replace(tzinfo=None) < end: return HOUR else: return ZERO Eastern = USTimeZone(-5, "Eastern", "EST", "EDT") Central = USTimeZone(-6, "Central", "CST", "CDT") Mountain = USTimeZone(-7, "Mountain", "MST", "MDT") Pacific = USTimeZone(-8, "Pacific", "PST", "PDT") boltons-16.4.1/boltons/setutils.py0000644000175000017500000003260612727357214015314 0ustar hlehle# -*- coding: utf-8 -*- """\ The :class:`set` type brings brings the practical expressiveness of set theory to Python. It has a very rich API overall, but lacks a couple of fundamental features. For one, sets are not ordered. On top of this, sets are not indexable, i.e, ``my_set[8]`` will raise an :exc:`TypeError`. The :class:`IndexedSet` type remedies both of these issues without compromising on the excellent complexity characteristics of Python's built-in set implementation. """ from __future__ import print_function from bisect import bisect_left from itertools import chain, islice from collections import MutableSet import operator try: from typeutils import make_sentinel _MISSING = make_sentinel(var_name='_MISSING') except ImportError: _MISSING = object() __all__ = ['IndexedSet'] _COMPACTION_FACTOR = 8 # TODO: inherit from set() # TODO: .discard_many(), .remove_many() # TODO: raise exception on non-set params? # TODO: technically reverse operators should probably reverse the # order of the 'other' inputs and put self last (to try and maintain # insertion order) class IndexedSet(MutableSet): """``IndexedSet`` is a :class:`collections.MutableSet` that maintains insertion order and uniqueness of inserted elements. It's a hybrid type, mostly like an OrderedSet, but also :class:`list`-like, in that it supports indexing and slicing. Args: other (iterable): An optional iterable used to initialize the set. >>> x = IndexedSet(list(range(4)) + list(range(8))) >>> x IndexedSet([0, 1, 2, 3, 4, 5, 6, 7]) >>> x - set(range(2)) IndexedSet([2, 3, 4, 5, 6, 7]) >>> x[-1] 7 >>> fcr = IndexedSet('freecreditreport.com') >>> ''.join(fcr[:fcr.index('.')]) 'frecditpo' Standard set operators and interoperation with :class:`set` are all supported: >>> fcr & set('cash4gold.com') IndexedSet(['c', 'd', 'o', '.', 'm']) As you can see, the ``IndexedSet`` is almost like a ``UniqueList``, retaining only one copy of a given value, in the order it was first added. For the curious, the reason why IndexedSet does not support setting items based on index (i.e, ``__setitem__()``), consider the following dilemma:: my_indexed_set = [A, B, C, D] my_indexed_set[2] = A At this point, a set requires only one *A*, but a :class:`list` would overwrite *C*. Overwriting *C* would change the length of the list, meaning that ``my_indexed_set[2]`` would not be *A*, as expected with a list, but rather *D*. So, no ``__setitem__()``. Otherwise, the API strives to be as complete a union of the :class:`list` and :class:`set` APIs as possible. """ def __init__(self, other=None): self.item_index_map = dict() self.item_list = [] self.dead_indices = [] self._compactions = 0 self._c_max_size = 0 if other: self.update(other) # internal functions @property def _dead_index_count(self): return len(self.item_list) - len(self.item_index_map) def _compact(self): if not self.dead_indices: return self._compactions += 1 dead_index_count = self._dead_index_count items, index_map = self.item_list, self.item_index_map self._c_max_size = max(self._c_max_size, len(items)) for i, item in enumerate(self): items[i] = item index_map[item] = i del items[-dead_index_count:] del self.dead_indices[:] def _cull(self): ded = self.dead_indices if not ded: return items, ii_map = self.item_list, self.item_index_map if not ii_map: del items[:] del ded[:] elif len(ded) > 384: self._compact() elif self._dead_index_count > (len(items) / _COMPACTION_FACTOR): self._compact() elif items[-1] is _MISSING: # get rid of dead right hand side num_dead = 1 while items[-(num_dead + 1)] is _MISSING: num_dead += 1 if ded and ded[-1][1] == len(items): del ded[-1] del items[-num_dead:] def _get_real_index(self, index): if index < 0: index += len(self) if not self.dead_indices: return index real_index = index for d_start, d_stop in self.dead_indices: if real_index < d_start: break real_index += d_stop - d_start return real_index def _add_dead(self, start, stop=None): # TODO: does not handle when the new interval subsumes # multiple existing intervals dints = self.dead_indices if stop is None: stop = start + 1 cand_int = [start, stop] if not dints: dints.append(cand_int) return int_idx = bisect_left(dints, cand_int) dint = dints[int_idx - 1] d_start, d_stop = dint if start <= d_start <= stop: dint[0] = start elif start <= d_stop <= stop: dint[1] = stop else: dints.insert(int_idx, cand_int) return # common operations (shared by set and list) def __len__(self): return len(self.item_index_map) def __contains__(self, item): return item in self.item_index_map def __iter__(self): return (item for item in self.item_list if item is not _MISSING) def __reversed__(self): item_list = self.item_list return (item for item in reversed(item_list) if item is not _MISSING) def __repr__(self): return '%s(%r)' % (self.__class__.__name__, list(self)) def __eq__(self, other): if isinstance(other, IndexedSet): return len(self) == len(other) and list(self) == list(other) return set(self) == set(other) @classmethod def from_iterable(cls, it): "from_iterable(it) -> create a set from an iterable" return cls(it) # set operations def add(self, item): "add(item) -> add item to the set" if item not in self.item_index_map: self.item_index_map[item] = len(self.item_list) self.item_list.append(item) def remove(self, item): "remove(item) -> remove item from the set, raises if not present" try: didx = self.item_index_map.pop(item) except KeyError: raise KeyError(item) self.item_list[didx] = _MISSING self._add_dead(didx) self._cull() def discard(self, item): "discard(item) -> discard item from the set (does not raise)" try: self.remove(item) except KeyError: pass def clear(self): "clear() -> empty the set" del self.item_list[:] del self.dead_indices[:] self.item_index_map.clear() def isdisjoint(self, other): "isdisjoint(other) -> return True if no overlap with other" iim = self.item_index_map for k in other: if k in iim: return False return True def issubset(self, other): "issubset(other) -> return True if other contains this set" if len(other) < len(self): return False for k in self.item_index_map: if k not in other: return False return True def issuperset(self, other): "issuperset(other) -> return True if set contains other" if len(other) > len(self): return False iim = self.item_index_map for k in other: if k not in iim: return False return True def union(self, *others): "union(*others) -> return a new set containing this set and others" return self.from_iterable(chain(self, *others)) def iter_intersection(self, *others): "iter_intersection(*others) -> iterate over elements also in others" for k in self: for other in others: if k not in other: break else: yield k return def intersection(self, *others): "intersection(*others) -> get a set with overlap of this and others" if len(others) == 1: other = others[0] return self.from_iterable(k for k in self if k in other) return self.from_iterable(self.iter_intersection(*others)) def iter_difference(self, *others): "iter_difference(*others) -> iterate over elements not in others" for k in self: for other in others: if k in other: break else: yield k return def difference(self, *others): "difference(*others) -> get a new set with elements not in others" if len(others) == 1: other = others[0] return self.from_iterable(k for k in self if k not in other) return self.from_iterable(self.iter_difference(*others)) def symmetric_difference(self, *others): "symmetric_difference(*others) -> XOR set of this and others" ret = self.union(*others) return ret.difference(self.intersection(*others)) __or__ = __ror__ = union __and__ = __rand__ = intersection __sub__ = __rsub__ = difference __xor__ = __rxor__ = symmetric_difference # in-place set operations def update(self, *others): "update(*others) -> add values from one or more iterables" if not others: return # raise? elif len(others) == 1: other = others[0] else: other = chain(others) for o in other: self.add(o) def intersection_update(self, *others): "intersection_update(*others) -> discard self.difference(*others)" for val in self.difference(*others): self.discard(val) def difference_update(self, *others): "difference_update(*others) -> discard self.intersection(*others)" if self in others: self.clear() for val in self.intersection(*others): self.discard(val) def symmetric_difference_update(self, other): # note singular 'other' "symmetric_difference_update(other) -> in-place XOR with other" if self is other: self.clear() for val in other: if val in self: self.discard(val) else: self.add(val) def __ior__(self, *others): self.update(*others) return self def __iand__(self, *others): self.intersection_update(*others) return self def __isub__(self, *others): self.difference_update(*others) return self def __ixor__(self, *others): self.symmetric_difference_update(*others) return self def iter_slice(self, start, stop, step=None): "iterate over a slice of the set" iterable = self if start is not None: start = self._get_real_index(start) if stop is not None: stop = self._get_real_index(stop) if step is not None and step < 0: step = -step iterable = reversed(self) return islice(iterable, start, stop, step) # list operations def __getitem__(self, index): try: start, stop, step = index.start, index.stop, index.step except AttributeError: index = operator.index(index) else: iter_slice = self.iter_slice(start, stop, step) return self.from_iterable(iter_slice) if index < 0: index += len(self) real_index = self._get_real_index(index) try: ret = self.item_list[real_index] except IndexError: raise IndexError('IndexedSet index out of range') return ret def pop(self, index=None): "pop(index) -> remove the item at a given index (-1 by default)" item_index_map = self.item_index_map len_self = len(item_index_map) if index is None or index == -1 or index == len_self - 1: ret = self.item_list.pop() del item_index_map[ret] else: real_index = self._get_real_index(index) ret = self.item_list[real_index] self.item_list[real_index] = _MISSING del item_index_map[ret] self._add_dead(real_index) self._cull() return ret def count(self, val): "count(val) -> count number of instances of value (0 or 1)" if val in self.item_index_map: return 1 return 0 def reverse(self): "reverse() -> reverse the contents of the set in-place" reversed_list = list(reversed(self)) self.item_list[:] = reversed_list for i, item in enumerate(self.item_list): self.item_index_map[item] = i del self.dead_indices[:] def sort(self): "sort() -> sort the contents of the set in-place" sorted_list = sorted(self) if sorted_list == self.item_list: return self.item_list[:] = sorted_list for i, item in enumerate(self.item_list): self.item_index_map[item] = i del self.dead_indices[:] def index(self, val): "index(val) -> get the index of a value, raises if not present" try: return self.item_index_map[val] except KeyError: cn = self.__class__.__name__ raise ValueError('%r is not in %s' % (val, cn)) boltons-16.4.1/boltons/easterutils.py0000644000175000017500000000053312727357214015776 0ustar hlehle# -*- coding: utf-8 -*- from __future__ import print_function def gobs_program(): """ A pure-Python implementation of Gob's Algorithm (2006). A brief explanation can be found here: https://www.youtube.com/watch?v=JbnjusltDHk """ while True: print("Penus", end=" ") if __name__ == '__main__': gobs_program() boltons-16.4.1/boltons/tbutils.py0000644000175000017500000007026212727357214015126 0ustar hlehle# -*- coding: utf-8 -*- """One of the oft-cited tenets of Python is that it is better to ask forgiveness than permission. That is, there are many cases where it is more inclusive and correct to handle exceptions than spend extra lines and execution time checking for conditions. This philosophy makes good exception handling features all the more important. Unfortunately Python's :mod:`traceback` module is woefully behind the times. The ``tbutils`` module provides two disparate but complementary featuresets: 1. With :class:`ExceptionInfo` and :class:`TracebackInfo`, the ability to extract, construct, manipulate, format, and serialize exceptions, tracebacks, and callstacks. 2. With :class:`ParsedException`, the ability to find and parse tracebacks from captured output such as logs and stdout. There is also the :class:`ContextualTracebackInfo` variant of :class:`TracebackInfo`, which includes much more information from each frame of the callstack, including values of locals and neighboring lines of code. """ from __future__ import print_function import re import sys import linecache if str is bytes: # py2 text = unicode else: # py3 text = str # TODO: chaining primitives? what are real use cases where these help? # TODO: print_* for backwards compatability # __all__ = ['extract_stack', 'extract_tb', 'format_exception', # 'format_exception_only', 'format_list', 'format_stack', # 'format_tb', 'print_exc', 'format_exc', 'print_exception', # 'print_last', 'print_stack', 'print_tb'] __all__ = ['ExceptionInfo', 'TracebackInfo', 'Callpoint', 'ContextualExceptionInfo', 'ContextualTracebackInfo', 'ContextualCallpoint', 'print_exception', 'ParsedException'] class Callpoint(object): """The Callpoint is a lightweight object used to represent a single entry in the code of a call stack. It stores the code-related metadata of a given frame. Available attributes are the same as the parameters below. Args: func_name (str): the function name lineno (int): the line number module_name (str): the module name module_path (str): the filesystem path of the module lasti (int): the index of bytecode execution line (str): the single-line code content (if available) """ __slots__ = ('func_name', 'lineno', 'module_name', 'module_path', 'lasti', 'line') def __init__(self, module_name, module_path, func_name, lineno, lasti, line=None): self.func_name = func_name self.lineno = lineno self.module_name = module_name self.module_path = module_path self.lasti = lasti self.line = line def to_dict(self): "Get a :class:`dict` copy of the Callpoint. Useful for serialization." ret = {} for slot in self.__slots__: try: ret[slot] = getattr(self, slot) except AttributeError: pass return ret @classmethod def from_current(cls, level=1): "Creates a Callpoint from the location of the calling function." frame = sys._getframe(level) return cls.from_frame(frame) @classmethod def from_frame(cls, frame): "Create a Callpoint object from data extracted from the given frame." func_name = frame.f_code.co_name lineno = frame.f_lineno module_name = frame.f_globals.get('__name__', '') module_path = frame.f_code.co_filename lasti = frame.f_lasti line = _DeferredLine(module_path, lineno, frame.f_globals) return cls(module_name, module_path, func_name, lineno, lasti, line=line) @classmethod def from_tb(cls, tb): """Create a Callpoint from the traceback of the current exception. Main difference with :meth:`from_frame` is that ``lineno`` and ``lasti`` come from the traceback, which is to say the line that failed in the try block, not the line currently being executed (in the except block). """ func_name = tb.tb_frame.f_code.co_name lineno = tb.tb_lineno lasti = tb.tb_lasti module_name = tb.tb_frame.f_globals.get('__name__', '') module_path = tb.tb_frame.f_code.co_filename line = _DeferredLine(module_path, lineno, tb.tb_frame.f_globals) return cls(module_name, module_path, func_name, lineno, lasti, line=line) def __repr__(self): cn = self.__class__.__name__ args = [getattr(self, s, None) for s in self.__slots__] if not any(args): return super(Callpoint, self).__repr__() else: return '%s(%s)' % (cn, ', '.join([repr(a) for a in args])) def tb_frame_str(self): """Render the Callpoint as it would appear in a standard printed Python traceback. Returns a string with filename, line number, function name, and the actual code line of the error on up to two lines. """ ret = ' File "%s", line %s, in %s\n' % (self.module_path, self.lineno, self.func_name) if self.line: ret += ' %s\n' % (str(self.line).strip(),) return ret class _DeferredLine(object): """The _DeferredLine type allows Callpoints and TracebackInfos to be constructed without potentially hitting the filesystem, as is the normal behavior of the standard Python :mod:`traceback` and :mod:`linecache` modules. Calling :func:`str` fetches and caches the line. Args: filename (str): the path of the file containing the line lineno (int): the number of the line in question module_globals (dict): an optional dict of module globals, used to handle advanced use cases using custom module loaders. """ def __init__(self, filename, lineno, module_globals=None): self.filename = filename self.lineno = lineno # TODO: this is going away when we fix linecache # TODO: (mark) read about loader self.module_globals = {} if module_globals is not None: for k in ('__name__', '__loader__'): v = module_globals.get(k) if v is None: self.module_globals[k] = v def __eq__(self, other): return (self.lineno, self.filename) == (other.lineno, other.filename) def __ne__(self, other): return not self == other def __str__(self): if hasattr(self, '_line'): return self._line try: linecache.checkcache(self.filename) line = linecache.getline(self.filename, self.lineno, self.module_globals) line = line.rstrip() except KeyError: line = '' self._line = line return line def __repr__(self): return repr(str(self)) def __len__(self): return len(str(self)) # TODO: dedup frames, look at __eq__ on _DeferredLine class TracebackInfo(object): """The TracebackInfo class provides a basic representation of a stack trace, be it from an exception being handled or just part of normal execution. It is basically a wrapper around a list of :class:`Callpoint` objects representing frames. Args: frames (list): A list of frame objects in the stack. .. note :: ``TracebackInfo`` can represent both exception tracebacks and non-exception tracebacks (aka stack traces). As a result, there is no ``TracebackInfo.from_current()``, as that would be ambiguous. Instead, call :meth:`TracebackInfo.from_frame` without the *frame* argument for a stack trace, or :meth:`TracebackInfo.from_traceback` without the *tb* argument for an exception traceback. """ callpoint_type = Callpoint def __init__(self, frames): self.frames = frames @classmethod def from_frame(cls, frame=None, level=1, limit=None): """Create a new TracebackInfo *frame* by recurring up in the stack a max of *limit* times. If *frame* is unset, get the frame from :func:`sys._getframe` using *level*. Args: frame (types.FrameType): frame object from :func:`sys._getframe` or elsewhere. Defaults to result of :func:`sys.get_frame`. level (int): If *frame* is unset, the desired frame is this many levels up the stack from the invocation of this method. Default ``1`` (i.e., caller of this method). limit (int): max number of parent frames to extract (defaults to :data:`sys.tracebacklimit`) """ ret = [] if frame is None: frame = sys._getframe(1) if limit is None: limit = getattr(sys, 'tracebacklimit', 1000) n = 0 while frame is not None and n < limit: item = cls.callpoint_type.from_frame(frame) ret.append(item) frame = frame.f_back n += 1 ret.reverse() return cls(ret) @classmethod def from_traceback(cls, tb=None, limit=None): """Create a new TracebackInfo from the traceback *tb* by recurring up in the stack a max of *limit* times. If *tb* is unset, get the traceback from the currently handled exception. If no exception is being handled, raise a :exc:`ValueError`. Args: frame (types.TracebackType): traceback object from :func:`sys.exc_info` or elsewhere. If absent or set to ``None``, defaults to ``sys.exc_info()[2]``, and raises a :exc:`ValueError` if no exception is currently being handled. limit (int): max number of parent frames to extract (defaults to :data:`sys.tracebacklimit`) """ ret = [] if tb is None: tb = sys.exc_info()[2] if tb is None: raise ValueError('no tb set and no exception being handled') if limit is None: limit = getattr(sys, 'tracebacklimit', 1000) n = 0 while tb is not None and n < limit: item = cls.callpoint_type.from_tb(tb) ret.append(item) tb = tb.tb_next n += 1 return cls(ret) @classmethod def from_dict(cls, d): "Complements :meth:`TracebackInfo.to_dict`." # TODO: check this. return cls(d['frames']) def to_dict(self): """Returns a dict with a list of :class:`Callpoint` frames converted to dicts. """ return {'frames': [f.to_dict() for f in self.frames]} def __len__(self): return len(self.frames) def __iter__(self): return iter(self.frames) def __repr__(self): cn = self.__class__.__name__ if self.frames: frame_part = ' last=%r' % (self.frames[-1],) else: frame_part = '' return '<%s frames=%s%s>' % (cn, len(self.frames), frame_part) def __str__(self): return self.get_formatted() def get_formatted(self): """Returns a string as formatted in the traditional Python built-in style observable when an exception is not caught. In other words, mimics :func:`traceback.format_tb` and :func:`traceback.format_stack`. """ ret = 'Traceback (most recent call last):\n' ret += ''.join([f.tb_frame_str() for f in self.frames]) return ret class ExceptionInfo(object): """An ExceptionInfo object ties together three main fields suitable for representing an instance of an exception: The exception type name, a string representation of the exception itself (the exception message), and information about the traceback (stored as a :class:`TracebackInfo` object). These fields line up with :func:`sys.exc_info`, but unlike the values returned by that function, ExceptionInfo does not hold any references to the real exception or traceback. This property makes it suitable for serialization or long-term retention, without worrying about formatting pitfalls, circular references, or leaking memory. Args: exc_type (str): The exception type name. exc_msg (str): String representation of the exception value. tb_info (TracebackInfo): Information about the stack trace of the exception. Like the :class:`TracebackInfo`, ExceptionInfo is most commonly instantiated from one of its classmethods: :meth:`from_exc_info` or :meth:`from_current`. """ #: Override this in inherited types to control the TracebackInfo type used tb_info_type = TracebackInfo def __init__(self, exc_type, exc_msg, tb_info): # TODO: additional fields for SyntaxErrors self.exc_type = exc_type self.exc_msg = exc_msg self.tb_info = tb_info @classmethod def from_exc_info(cls, exc_type, exc_value, traceback): """Create an :class:`ExceptionInfo` object from the exception's type, value, and traceback, as returned by :func:`sys.exc_info`. See also :meth:`from_current`. """ type_str = exc_type.__name__ type_mod = exc_type.__module__ if type_mod not in ("__main__", "__builtin__", "exceptions", "builtins"): type_str = '%s.%s' % (type_mod, type_str) val_str = _some_str(exc_value) tb_info = cls.tb_info_type.from_traceback(traceback) return cls(type_str, val_str, tb_info) @classmethod def from_current(cls): """Create an :class:`ExceptionInfo` object from the current exception being handled, by way of :func:`sys.exc_info`. Will raise an exception if no exception is currently being handled. """ return cls.from_exc_info(*sys.exc_info()) def to_dict(self): """Get a :class:`dict` representation of the ExceptionInfo, suitable for JSON serialization. """ return {'exc_type': self.exc_type, 'exc_msg': self.exc_msg, 'exc_tb': self.tb_info.to_dict()} def __repr__(self): cn = self.__class__.__name__ try: len_frames = len(self.tb_info.frames) last_frame = ', last=%r' % (self.tb_info.frames[-1],) except Exception: len_frames = 0 last_frame = '' args = (cn, self.exc_type, self.exc_msg, len_frames, last_frame) return '<%s [%s: %s] (%s frames%s)>' % args def get_formatted(self): """Returns a string formatted in the traditional Python built-in style observable when an exception is not caught. In other words, mimics :func:`traceback.format_exception`. """ # TODO: add SyntaxError formatting tb_str = self.tb_info.get_formatted() return ''.join([tb_str, '%s: %s' % (self.exc_type, self.exc_msg)]) def get_formatted_exception_only(self): return '%s: %s' % (self.exc_type, self.exc_msg) class ContextualCallpoint(Callpoint): """The ContextualCallpoint is a :class:`Callpoint` subtype with the exact same API and storing two additional values: 1. :func:`repr` outputs for local variables from the Callpoint's scope 2. A number of lines before and after the Callpoint's line of code The ContextualCallpoint is used by the :class:`ContextualTracebackInfo`. """ def __init__(self, *a, **kw): self.local_reprs = kw.pop('local_reprs', {}) self.pre_lines = kw.pop('pre_lines', []) self.post_lines = kw.pop('post_lines', []) super(ContextualCallpoint, self).__init__(*a, **kw) @classmethod def from_frame(cls, frame): "Identical to :meth:`Callpoint.from_frame`" ret = super(ContextualCallpoint, cls).from_frame(frame) ret._populate_local_reprs(frame.f_locals) ret._populate_context_lines() return ret @classmethod def from_tb(cls, tb): "Identical to :meth:`Callpoint.from_tb`" ret = super(ContextualCallpoint, cls).from_tb(tb) ret._populate_local_reprs(tb.tb_frame.f_locals) ret._populate_context_lines() return ret def _populate_context_lines(self, pivot=8): DL, lineno = _DeferredLine, self.lineno try: module_globals = self.line.module_globals except Exception: module_globals = None start_line = max(0, lineno - pivot) pre_lines = [DL(self.module_path, ln, module_globals) for ln in range(start_line, lineno)] self.pre_lines[:] = pre_lines post_lines = [DL(self.module_path, ln, module_globals) for ln in range(lineno + 1, lineno + 1 + pivot)] self.post_lines[:] = post_lines return def _populate_local_reprs(self, f_locals): local_reprs = self.local_reprs for k, v in f_locals.items(): try: local_reprs[k] = repr(v) except Exception: surrogate = '' % type(v).__name__ local_reprs[k] = surrogate return def to_dict(self): """ Same principle as :meth:`Callpoint.to_dict`, but with the added contextual values. With ``ContextualCallpoint.to_dict()``, each frame will now be represented like:: {'func_name': 'print_example', 'lineno': 0, 'module_name': 'example_module', 'module_path': '/home/example/example_module.pyc', 'lasti': 0, 'line': 'print "example"', 'locals': {'variable': '"value"'}, 'pre_lines': ['variable = "value"'], 'post_lines': []} The locals dictionary and line lists are copies and can be mutated freely. """ ret = super(ContextualCallpoint, self).to_dict() ret['locals'] = dict(self.local_reprs) # get the line numbers and textual lines # without assuming DeferredLines start_line = self.lineno - len(self.pre_lines) pre_lines = [{'lineno': start_line + i, 'line': str(l)} for i, l in enumerate(self.pre_lines)] # trim off leading empty lines for i, item in enumerate(pre_lines): if item['line']: break if i: pre_lines = pre_lines[i:] ret['pre_lines'] = pre_lines # now post_lines post_lines = [{'lineno': self.lineno + i, 'line': str(l)} for i, l in enumerate(self.post_lines)] _last = 0 for i, item in enumerate(post_lines): if item['line']: _last = i post_lines = post_lines[:_last + 1] ret['post_lines'] = post_lines return ret class ContextualTracebackInfo(TracebackInfo): """The ContextualTracebackInfo type is a :class:`TracebackInfo` subtype that is used by :class:`ContextualExceptionInfo` and uses the :class:`ContextualCallpoint` as its frame-representing primitive. """ callpoint_type = ContextualCallpoint class ContextualExceptionInfo(ExceptionInfo): """The ContextualTracebackInfo type is a :class:`TracebackInfo` subtype that uses the :class:`ContextualCallpoint` as its frame-representing primitive. It carries with it most of the exception information required to recreate the widely recognizable "500" page for debugging Django applications. """ tb_info_type = ContextualTracebackInfo # TODO: clean up & reimplement -- specifically for syntax errors def format_exception_only(etype, value): """Format the exception part of a traceback. The arguments are the exception type and value such as given by sys.last_type and sys.last_value. The return value is a list of strings, each ending in a newline. Normally, the list contains a single string; however, for SyntaxError exceptions, it contains several lines that (when printed) display detailed information about where the syntax error occurred. The message indicating which exception occurred is always the last string in the list. """ # Gracefully handle (the way Python 2.4 and earlier did) the case of # being called with (None, None). if etype is None: return [_format_final_exc_line(etype, value)] stype = etype.__name__ smod = etype.__module__ if smod not in ("__main__", "builtins", "exceptions"): stype = smod + '.' + stype if not issubclass(etype, SyntaxError): return [_format_final_exc_line(stype, value)] # It was a syntax error; show exactly where the problem was found. lines = [] filename = value.filename or "" lineno = str(value.lineno) or '?' lines.append(' File "%s", line %s\n' % (filename, lineno)) badline = value.text offset = value.offset if badline is not None: lines.append(' %s\n' % badline.strip()) if offset is not None: caretspace = badline.rstrip('\n')[:offset].lstrip() # non-space whitespace (likes tabs) must be kept for alignment caretspace = ((c.isspace() and c or ' ') for c in caretspace) # only three spaces to account for offset1 == pos 0 lines.append(' %s^\n' % ''.join(caretspace)) msg = value.msg or "" lines.append("%s: %s\n" % (stype, msg)) return lines # TODO: use asciify, improved if necessary def _some_str(value): try: return str(value) except Exception: pass try: value = unicode(value) return value.encode("ascii", "backslashreplace") except Exception: pass return '' % type(value).__name__ def _format_final_exc_line(etype, value): valuestr = _some_str(value) if value is None or not valuestr: line = "%s\n" % etype else: line = "%s: %s\n" % (etype, valuestr) return line def print_exception(etype, value, tb, limit=None, file=None): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if traceback is not None, it prints a header "Traceback (most recent call last):"; (2) it prints the exception type and value after the stack trace; (3) if type is SyntaxError and value has the appropriate format, it prints the line where the syntax error occurred with a caret on the next line indicating the approximate position of the error. """ if file is None: file = sys.stderr if tb: tbi = TracebackInfo.from_traceback(tb, limit) print(str(tbi), end='', file=file) for line in format_exception_only(etype, value): print(line, end='', file=file) def fix_print_exception(): """ Sets the default exception hook :func:`sys.excepthook` to the :func:`tbutils.print_exception` that uses all the ``tbutils`` facilities to provide slightly more correct output behavior. """ sys.excepthook = print_exception _frame_re = re.compile(r'^File "(?P.+)", line (?P\d+)' r', in (?P.+)$') _se_frame_re = re.compile(r'^File "(?P.+)", line (?P\d+)') # TODO: ParsedException generator over large bodies of text class ParsedException(object): """Stores a parsed traceback and exception as would be typically output by :func:`sys.excepthook` or :func:`traceback.print_exception`. .. note: Does not currently store SyntaxError details such as column. """ def __init__(self, exc_type_name, exc_msg, frames=None): self.exc_type = exc_type_name self.exc_msg = exc_msg self.frames = list(frames or []) @property def source_file(self): """ The file path of module containing the function that raised the exception, or None if not available. """ try: return self.frames[-1]['filepath'] except IndexError: return None def to_dict(self): "Get a copy as a JSON-serializable :class:`dict`." return {'exc_type': self.exc_type, 'exc_msg': self.exc_msg, 'frames': list(self.frames)} def __repr__(self): cn = self.__class__.__name__ return ('%s(%r, %r, frames=%r)' % (cn, self.exc_type, self.exc_msg, self.frames)) def to_string(self): """Formats the exception and its traceback into the standard format, as returned by the traceback module. ``ParsedException.from_string(text).to_string()`` should yield ``text``. """ lines = [u'Traceback (most recent call last):'] for frame in self.frames: lines.append(u' File "%s", line %s, in %s' % (frame['filepath'], frame['lineno'], frame['funcname'])) source_line = frame.get('source_line') if source_line: lines.append(u' %s' % (source_line,)) if self.exc_msg: lines.append(u'%s: %s' % (self.exc_type, self.exc_msg)) else: lines.append(u'%s' % (self.exc_type,)) return u'\n'.join(lines) @classmethod def from_string(cls, tb_str): """Parse a traceback and exception from the text *tb_str*. This text is expected to have been decoded, otherwise it will be interpreted as UTF-8. This method does not search a larger body of text for tracebacks. If the first line of the text passed does not match one of the known patterns, a :exc:`ValueError` will be raised. This method will ignore trailing text after the end of the first traceback. Args: tb_str (str): The traceback text (:class:`unicode` or UTF-8 bytes) """ if not isinstance(tb_str, text): tb_str = tb_str.decode('utf-8') tb_lines = tb_str.lstrip().splitlines() # First off, handle some ignored exceptions. These can be the # result of exceptions raised by __del__ during garbage # collection while tb_lines: cl = tb_lines[-1] if cl.startswith('Exception ') and cl.endswith('ignored'): tb_lines.pop() else: break if tb_lines and tb_lines[0].strip() == 'Traceback (most recent call last):': start_line = 1 frame_re = _frame_re elif len(tb_lines) > 1 and tb_lines[-2].lstrip().startswith('^'): # This is to handle the slight formatting difference # associated with SyntaxErrors, which also don't really # have tracebacks start_line = 0 frame_re = _se_frame_re else: raise ValueError('unrecognized traceback string format') frames = [] line_no = start_line while True: frame_line = tb_lines[line_no].strip() frame_match = frame_re.match(frame_line) if frame_match: frame_dict = frame_match.groupdict() try: next_line = tb_lines[line_no + 1] except IndexError: # We read what we could next_line = '' next_line_stripped = next_line.strip() if ( frame_re.match(next_line_stripped) or # The exception message will not be indented # This check is to avoid overrunning on eval-like # tracebacks where the last frame doesn't have source # code in the traceback not next_line.startswith(' ') ): frame_dict['source_line'] = '' else: frame_dict['source_line'] = next_line_stripped line_no += 1 else: break line_no += 1 frames.append(frame_dict) try: exc_line = '\n'.join(tb_lines[line_no:]) exc_type, _, exc_msg = exc_line.partition(': ') except Exception: exc_type, exc_msg = '', '' return cls(exc_type, exc_msg, frames) ParsedTB = ParsedException # legacy alias boltons-16.4.1/boltons/tableutils.py0000644000175000017500000004730412727357214015611 0ustar hlehle# -*- coding: utf-8 -*- """If there is one recurring theme in ``boltons``, it is that Python has excellent datastructures that constitute a good foundation for most quick manipulations, as well as building applications. However, Python usage has grown much faster than builtin data structure power. Python has a growing need for more advanced general-purpose data structures which behave intuitively. The :class:`Table` class is one example. When handed one- or two-dimensional data, it can provide useful, if basic, text and HTML renditions of small to medium sized data. It also heuristically handles recursive data of various formats (lists, dicts, namedtuples, objects). For more advanced :class:`Table`-style manipulation check out the `pandas`_ DataFrame. .. _pandas: http://pandas.pydata.org/ """ from __future__ import print_function import cgi import types from itertools import islice from collections import Sequence, Mapping, MutableSequence try: string_types, integer_types = (str, unicode), (int, long) except Exception: # Python 3 compat unicode = str string_types, integer_types = (str, bytes), (int,) try: from typeutils import make_sentinel _MISSING = make_sentinel(var_name='_MISSING') except ImportError: _MISSING = object() """ Some idle feature thoughts: * shift around column order without rearranging data * gotta make it so you can add additional items, not just initialize with * maybe a shortcut would be to allow adding of Tables to other Tables * what's the perf of preallocating lists and overwriting items versus starting from empty? * is it possible to effectively tell the difference between when a Table is from_data()'d with a single row (list) or with a list of lists? * CSS: white-space pre-line or pre-wrap maybe? * Would be nice to support different backends (currently uses lists exclusively). Sometimes large datasets come in list-of-dicts and list-of-tuples format and it's desirable to cut down processing overhead. TODO: make iterable on rows? """ __all__ = ['Table'] def to_text(obj, maxlen=None): try: text = unicode(obj) except Exception: try: text = unicode(repr(obj)) except Exception: text = unicode(object.__repr__(obj)) if maxlen and len(text) > maxlen: text = text[:maxlen - 3] + '...' # TODO: inverse of ljust/rjust/center return text def escape_html(obj, maxlen=None): text = to_text(obj, maxlen=maxlen) return cgi.escape(text, quote=True) _DNR = set((type(None), bool, complex, float, type(NotImplemented), slice, types.FunctionType, types.MethodType, types.BuiltinFunctionType, types.GeneratorType) + string_types + integer_types) class UnsupportedData(TypeError): pass class InputType(object): def __init__(self, *a, **kw): pass def get_entry_seq(self, data_seq, headers): return [self.get_entry(entry, headers) for entry in data_seq] class DictInputType(InputType): def check_type(self, obj): return isinstance(obj, Mapping) def guess_headers(self, obj): return sorted(obj.keys()) def get_entry(self, obj, headers): return [obj.get(h) for h in headers] def get_entry_seq(self, obj, headers): return [[ci.get(h) for h in headers] for ci in obj] class ObjectInputType(InputType): def check_type(self, obj): return type(obj) not in _DNR and hasattr(obj, '__class__') def guess_headers(self, obj): headers = [] for attr in dir(obj): # an object's __dict__ could technically have non-string keys try: val = getattr(obj, attr) except Exception: # seen on greenlet: `run` shows in dir() but raises # AttributeError. Also properties misbehave. continue if callable(val): continue headers.append(attr) return headers def get_entry(self, obj, headers): values = [] for h in headers: try: values.append(getattr(obj, h)) except Exception: values.append(None) return values # might be better to hardcode list support since it's so close to the # core or might be better to make this the copy-style from_* importer # and have the non-copy style be hardcoded in __init__ class ListInputType(InputType): def check_type(self, obj): return isinstance(obj, MutableSequence) def guess_headers(self, obj): return None def get_entry(self, obj, headers): return obj def get_entry_seq(self, obj_seq, headers): return obj_seq class TupleInputType(InputType): def check_type(self, obj): return isinstance(obj, tuple) def guess_headers(self, obj): return None def get_entry(self, obj, headers): return list(obj) def get_entry_seq(self, obj_seq, headers): return [list(t) for t in obj_seq] class NamedTupleInputType(InputType): def check_type(self, obj): return hasattr(obj, '_fields') and isinstance(obj, tuple) def guess_headers(self, obj): return list(obj._fields) def get_entry(self, obj, headers): return [getattr(obj, h, None) for h in headers] def get_entry_seq(self, obj_seq, headers): return [[getattr(obj, h, None) for h in headers] for obj in obj_seq] class Table(object): """ This Table class is meant to be simple, low-overhead, and extensible. Its most common use would be for translation between in-memory data structures and serialization formats, such as HTML and console-ready text. As such, it stores data in list-of-lists format, and *does not* copy lists passed in. It also reserves the right to modify those lists in a "filling" process, whereby short lists are extended to the width of the table (usually determined by number of headers). This greatly reduces overhead and processing/validation that would have to occur otherwise. General description of headers behavior: Headers describe the columns, but are not part of the data, however, if the *headers* argument is omitted, Table tries to infer header names from the data. It is possible to have a table with no headers, just pass in ``headers=None``. Supported inputs: * :class:`list` of :class:`list` objects * :class:`dict` (list/single) * :class:`object` (list/single) * :class:`collections.namedtuple` (list/single) * TODO: DB API cursor? * TODO: json Supported outputs: * HTML * Pretty text (also usable as GF Markdown) * TODO: CSV * TODO: json * TODO: json lines To minimize resident size, the Table data is stored as a list of lists. """ # order definitely matters here _input_types = [DictInputType(), ListInputType(), NamedTupleInputType(), TupleInputType(), ObjectInputType()] _html_tr, _html_tr_close = '', '' _html_th, _html_th_close = '', '' _html_td, _html_td_close = '', '' # _html_thead, _html_thead_close = '', '' # _html_tfoot, _html_tfoot_close = '', '' _html_table_tag, _html_table_tag_close = '', '
' def __init__(self, data=None, headers=_MISSING, metadata=None): if headers is _MISSING: headers = [] if data: headers, data = list(data[0]), islice(data, 1, None) self.headers = headers or [] self.metadata = metadata or {} self._data = [] self._width = 0 self.extend(data) def extend(self, data): """ Append the given data to the end of the Table. """ if not data: return self._data.extend(data) self._set_width() self._fill() def _set_width(self, reset=False): if reset: self._width = 0 if self._width: return if self.headers: self._width = len(self.headers) return self._width = max([len(d) for d in self._data]) def _fill(self): width, filler = self._width, [None] if not width: return for d in self._data: rem = width - len(d) if rem > 0: d.extend(filler * rem) return @classmethod def from_dict(cls, data, headers=_MISSING, max_depth=1, metadata=None): """Create a Table from a :class:`dict`. Operates the same as :meth:`from_data`, but forces interpretation of the data as a Mapping. """ return cls.from_data(data=data, headers=headers, max_depth=max_depth, _data_type=DictInputType(), metadata=metadata) @classmethod def from_list(cls, data, headers=_MISSING, max_depth=1, metadata=None): """Create a Table from a :class:`list`. Operates the same as :meth:`from_data`, but forces the interpretation of the data as a Sequence. """ return cls.from_data(data=data, headers=headers, max_depth=max_depth, _data_type=ListInputType(), metadata=metadata) @classmethod def from_object(cls, data, headers=_MISSING, max_depth=1, metadata=None): """Create a Table from an :class:`object`. Operates the same as :meth:`from_data`, but forces the interpretation of the data as an object. May be useful for some :class:`dict` and :class:`list` subtypes. """ return cls.from_data(data=data, headers=headers, max_depth=max_depth, _data_type=ObjectInputType(), metadata=metadata) @classmethod def from_data(cls, data, headers=_MISSING, max_depth=1, **kwargs): """Create a Table from any supported data, heuristically selecting how to represent the data in Table format. Args: data (object): Any object or iterable with data to be imported to the Table. headers (iterable): An iterable of headers to be matched to the data. If not explicitly passed, headers will be guessed for certain datatypes. max_depth (int): The level to which nested Tables should be created (default: 1). _data_type (InputType subclass): For advanced use cases, do not guess the type of the input data, use this data type instead. """ # TODO: seen/cycle detection/reuse ? # maxdepth follows the same behavior as find command # i.e., it doesn't work if max_depth=0 is passed in metadata = kwargs.pop('metadata', None) _data_type = kwargs.pop('_data_type', None) if max_depth < 1: # return data instead? return cls(headers=headers, metadata=metadata) is_seq = isinstance(data, Sequence) if is_seq: if not data: return cls(headers=headers, metadata=metadata) to_check = data[0] if not _data_type: for it in cls._input_types: if it.check_type(to_check): _data_type = it break else: # not particularly happy about this rewind-y approach is_seq = False to_check = data else: if type(data) in _DNR: # hmm, got scalar data. # raise an exception or make an exception, nahmsayn? return cls([[data]], headers=headers, metadata=metadata) to_check = data if not _data_type: for it in cls._input_types: if it.check_type(to_check): _data_type = it break else: raise UnsupportedData('unsupported data type %r' % type(data)) if headers is _MISSING: headers = _data_type.guess_headers(to_check) if is_seq: entries = _data_type.get_entry_seq(data, headers) else: entries = [_data_type.get_entry(data, headers)] if max_depth > 1: new_max_depth = max_depth - 1 for i, entry in enumerate(entries): for j, cell in enumerate(entry): if type(cell) in _DNR: # optimization to avoid function overhead continue try: entries[i][j] = cls.from_data(cell, max_depth=new_max_depth) except UnsupportedData: continue return cls(entries, headers=headers, metadata=metadata) def __len__(self): return len(self._data) def __getitem__(self, idx): return self._data[idx] def __repr__(self): cn = self.__class__.__name__ if self.headers: return '%s(headers=%r, data=%r)' % (cn, self.headers, self._data) else: return '%s(%r)' % (cn, self._data) def to_html(self, orientation=None, wrapped=True, with_headers=True, with_newlines=True, with_metadata=False, max_depth=1): """Render this Table to HTML. Configure the structure of Table HTML by subclassing and overriding ``_html_*`` class attributes. Args: orientation (str): one of 'auto', 'horizontal', or 'vertical' (or the first letter of any of those). Default 'auto'. wrapped (bool): whether or not to include the wrapping '
' tags. Default ``True``, set to ``False`` if appending multiple Table outputs or an otherwise customized HTML wrapping tag is needed. with_newlines (bool): Set to ``True`` if output should include added newlines to make the HTML more readable. Default ``False``. with_metadata (bool/str): Set to ``True`` if output should be preceded with a Table of preset metadata, if it exists. Set to special value ``'bottom'`` if the metadata Table HTML should come *after* the main HTML output. max_depth (int): Indicate how deeply to nest HTML tables before simply reverting to :func:`repr`-ing the nested data. Returns: A text string of the HTML of the rendered table. """ lines = [] headers = [] if with_metadata and self.metadata: metadata_table = Table.from_data(self.metadata, max_depth=max_depth) metadata_html = metadata_table.to_html(with_headers=True, with_newlines=with_newlines, with_metadata=False, max_depth=max_depth) if with_metadata != 'bottom': lines.append(metadata_html) lines.append('
') if with_headers and self.headers: headers.extend(self.headers) headers.extend([None] * (self._width - len(self.headers))) if wrapped: lines.append(self._html_table_tag) orientation = orientation or 'auto' ol = orientation[0].lower() if ol == 'a': ol = 'h' if len(self) > 1 else 'v' if ol == 'h': self._add_horizontal_html_lines(lines, headers=headers, max_depth=max_depth) elif ol == 'v': self._add_vertical_html_lines(lines, headers=headers, max_depth=max_depth) else: raise ValueError("expected one of 'auto', 'vertical', or" " 'horizontal', not %r" % orientation) if with_metadata and self.metadata and with_metadata == 'bottom': lines.append('
') lines.append(metadata_html) if wrapped: lines.append(self._html_table_tag_close) sep = '\n' if with_newlines else '' return sep.join(lines) def _add_horizontal_html_lines(self, lines, headers, max_depth): esc = escape_html new_depth = max_depth - 1 if max_depth > 1 else max_depth if max_depth > 1: new_depth = max_depth - 1 if headers: _thth = self._html_th_close + self._html_th lines.append(self._html_tr + self._html_th + _thth.join([esc(h) for h in headers]) + self._html_th_close + self._html_tr_close) trtd, _tdtd, _td_tr = (self._html_tr + self._html_td, self._html_td_close + self._html_td, self._html_td_close + self._html_tr_close) for row in self._data: if max_depth > 1: _fill_parts = [] for cell in row: if isinstance(cell, Table): _fill_parts.append(cell.to_html(max_depth=new_depth)) else: _fill_parts.append(esc(cell)) else: _fill_parts = [esc(c) for c in row] lines.append(''.join([trtd, _tdtd.join(_fill_parts), _td_tr])) def _add_vertical_html_lines(self, lines, headers, max_depth): esc = escape_html new_depth = max_depth - 1 if max_depth > 1 else max_depth tr, th, _th = self._html_tr, self._html_th, self._html_th_close td, _tdtd = self._html_td, self._html_td_close + self._html_td _td_tr = self._html_td_close + self._html_tr_close for i in range(self._width): line_parts = [tr] if headers: line_parts.extend([th, esc(headers[i]), _th]) if max_depth > 1: new_depth = max_depth - 1 _fill_parts = [] for row in self._data: cell = row[i] if isinstance(cell, Table): _fill_parts.append(cell.to_html(max_depth=new_depth)) else: _fill_parts.append(esc(row[i])) else: _fill_parts = [esc(row[i]) for row in self._data] line_parts.extend([td, _tdtd.join(_fill_parts), _td_tr]) lines.append(''.join(line_parts)) def to_text(self, with_headers=True, maxlen=None): """Get the Table's textual representation. Only works well for Tables with non-recursive data. Args: with_headers (bool): Whether to include a header row at the top. maxlen (int): Max length of data in each cell. """ # TODO: verify this works for markdown lines = [] widths = [] headers = list(self.headers) text_data = [[to_text(cell, maxlen=maxlen) for cell in row] for row in self._data] for idx in range(self._width): cur_widths = [len(cur) for cur in text_data] if with_headers: cur_widths.append(len(to_text(headers[idx], maxlen=maxlen))) widths.append(max(cur_widths)) if with_headers: lines.append(' | '.join([h.center(widths[i]) for i, h in enumerate(headers)])) lines.append('-+-'.join(['-' * w for w in widths])) for row in text_data: lines.append(' | '.join([cell.center(widths[j]) for j, cell in enumerate(row)])) return '\n'.join(lines) boltons-16.4.1/boltons/deprutils.py0000644000175000017500000000225512727357214015450 0ustar hlehle# -*- coding: utf-8 -*- """ Note that DeprecationWarnings are ignored by default in Python 2.7/3.2+, so be sure to either un-ignore them in your code, or run Python with the -Wd flag. """ import sys from warnings import warn ModuleType = type(sys) # todo: only warn once class DeprecatableModule(ModuleType): def __init__(self, module): name = module.__name__ super(DeprecatableModule, self).__init__(name=name) self.__dict__.update(module.__dict__) def __getattribute__(self, name): get_attribute = super(DeprecatableModule, self).__getattribute__ try: depros = get_attribute('_deprecated_members') except AttributeError: self._deprecated_members = depros = {} ret = get_attribute(name) message = depros.get(name) if message is not None: warn(message, DeprecationWarning, stacklevel=2) return ret def deprecate_module_member(mod_name, name, message): module = sys.modules[mod_name] if not isinstance(module, DeprecatableModule): sys.modules[mod_name] = module = DeprecatableModule(module) module._deprecated_members[name] = message return boltons-16.4.1/boltons/queueutils.py0000644000175000017500000001377512727357214015653 0ustar hlehle# -*- coding: utf-8 -*- """Python comes with a many great data structures, from :class:`dict` to :class:`collections.deque`, and no shortage of serviceable algorithm implementations, from :func:`sorted` to :mod:`bisect`. But priority queues are curiously relegated to an example documented in :mod:`heapq`. Even there, the approach presented is not full-featured and object-oriented. There is a built-in priority queue, :class:`Queue.PriorityQueue`, but in addition to its austere API, it carries the double-edged sword of threadsafety, making it fine for multi-threaded, multi-consumer applications, but high-overhead for cooperative/single-threaded use cases. The ``queueutils`` module currently provides two Queue implementations: :class:`HeapPriorityQueue`, based on a heap, and :class:`SortedPriorityQueue`, based on a sorted list. Both use a unified API based on :class:`BasePriortyQueue` to facilitate testing the slightly different performance characteristics on various application use cases. >>> pq = PriorityQueue() >>> pq.add('low priority task', 0) >>> pq.add('high priority task', 2) >>> pq.add('medium priority task 1', 1) >>> pq.add('medium priority task 2', 1) >>> len(pq) 4 >>> pq.pop() 'high priority task' >>> pq.peek() 'medium priority task 1' >>> len(pq) 3 """ from heapq import heappush, heappop from bisect import insort import itertools try: from typeutils import make_sentinel _REMOVED = make_sentinel(var_name='_REMOVED') except ImportError: _REMOVED = object() try: from listutils import BList # see BarrelList docstring for notes except ImportError: BList = list __all__ = ['PriorityQueue', 'BasePriorityQueue', 'HeapPriorityQueue', 'SortedPriorityQueue'] # TODO: make Base a real abstract class # TODO: add uniqueification class BasePriorityQueue(object): """The abstract base class for the other PriorityQueues in this module. Override the ``_backend_type`` class attribute, as well as the :meth:`_push_entry` and :meth:`_pop_entry` staticmethods for custom subclass behavior. (Don't forget to use :func:`staticmethod`). Args: priority_key (callable): A function that takes *priority* as passed in by :meth:`add` and returns an integer representing the effective priority. """ # negating priority means larger numbers = higher priority _default_priority_key = staticmethod(lambda p: -int(p or 0)) _backend_type = list def __init__(self, **kw): self._pq = self._backend_type() self._entry_map = {} self._counter = itertools.count() self._get_priority = kw.pop('priority_key', self._default_priority_key) if kw: raise TypeError('unexpected keyword arguments: %r' % kw.keys()) @staticmethod def _push_entry(backend, entry): pass # abstract @staticmethod def _pop_entry(backend): pass # abstract def add(self, task, priority=None): """Add a task to the queue, or change the *task*'s priority if *task* is already in the queue. *task* can be any type, and *priority* defaults to ``0``. Higher values representing higher priority, but this behavior can be controlled by setting *priority_key* in the constructor. """ priority = self._get_priority(priority) if task in self._entry_map: self.remove(task) count = next(self._counter) entry = [priority, count, task] self._entry_map[task] = entry self._push_entry(self._pq, entry) def remove(self, task): """Remove a task from the priority queue. Raises :exc:`KeyError` if the *task* is absent. """ entry = self._entry_map.pop(task) entry[-1] = _REMOVED def _cull(self, raise_exc=True): "Remove entries marked as removed by previous :meth:`remove` calls." while self._pq: priority, count, task = self._pq[0] if task is _REMOVED: self._pop_entry(self._pq) continue return if raise_exc: raise IndexError('empty priority queue') def peek(self, default=_REMOVED): """Read the next value in the queue without removing it. Returns *default* on an empty queue, or raises :exc:`KeyError` if *default* is not set. """ try: self._cull() _, _, task = self._pq[0] except IndexError: if default is not _REMOVED: return default raise IndexError('peek on empty queue') return task def pop(self, default=_REMOVED): """Remove and return the next value in the queue. Returns *default* on an empty queue, or raises :exc:`KeyError` if *default* is not set. """ try: self._cull() _, _, task = self._pop_entry(self._pq) del self._entry_map[task] except IndexError: if default is not _REMOVED: return default raise IndexError('pop on empty queue') return task def __len__(self): "Return the number of tasks in the queue." return len(self._entry_map) class HeapPriorityQueue(BasePriorityQueue): """A priority queue inherited from :class:`BasePriorityQueue`, backed by a list and based on the :func:`heapq.heappop` and :func:`heapq.heappush` functions in the built-in :mod:`heapq` module. """ @staticmethod def _pop_entry(backend): return heappop(backend) @staticmethod def _push_entry(backend, entry): heappush(backend, entry) class SortedPriorityQueue(BasePriorityQueue): """A priority queue inherited from :class:`BasePriorityQueue`, based on the :func:`bisect.insort` approach for in-order insertion into a sorted list. """ _backend_type = BList @staticmethod def _pop_entry(backend): return backend.pop(0) @staticmethod def _push_entry(backend, entry): insort(backend, entry) PriorityQueue = SortedPriorityQueue boltons-16.4.1/boltons/socketutils.py0000644000175000017500000006714512727357214016017 0ustar hlehle# -*- coding: utf-8 -*- """At its heart, Python can be viewed as an extension of the C programming language. Springing from the most popular systems programming language has made Python itself a great language for systems programming. One key to success in this domain is Python's very serviceable :mod:`socket` module and its :class:`socket.socket` type. The ``socketutils`` module provides natural next steps to the ``socket`` builtin: straightforward, tested building blocks for higher-level protocols. The :class:`BufferedSocket` wraps an ordinary socket, providing a layer of intuitive buffering for both sending and receiving. This facilitates parsing messages from streams, i.e., all sockets with type ``SOCK_STREAM``. The BufferedSocket enables receiving until the next relevant token, up to a certain size, or until the connection is closed. For all of these, it provides consistent APIs to size limiting, as well as timeouts that are compatible with multiple concurrency paradigms. Use it to parse the next one-off text or binary socket protocol you encounter. This module also provides the :class:`NetstringSocket`, a pure-Python implementation of `the Netstring protocol`_, built on top of the :class:`BufferedSocket`, serving as a ready-made, production-grade example. Special thanks to `Kurt Rose`_ for his original authorship and all his contributions on this module. Also thanks to `Daniel J. Bernstein`_, the original author of `Netstring`_. .. _the Netstring protocol: https://en.wikipedia.org/wiki/Netstring .. _Kurt Rose: https://github.com/doublereedkurt .. _Daniel J. Bernstein: https://cr.yp.to/ .. _Netstring: https://cr.yp.to/proto/netstrings.txt """ import time import socket try: from threading import RLock except Exception: class RLock(object): 'Dummy reentrant lock for builds without threads' def __enter__(self): pass def __exit__(self, exctype, excinst, exctb): pass try: from typeutils import make_sentinel _UNSET = make_sentinel(var_name='_UNSET') except ImportError: _UNSET = object() DEFAULT_TIMEOUT = 10 # 10 seconds DEFAULT_MAXSIZE = 32 * 1024 # 32kb _RECV_LARGE_MAXSIZE = 1024 ** 5 # 1PB class BufferedSocket(object): """Mainly provides recv_until and recv_size. recv, send, sendall, and peek all function as similarly as possible to the built-in socket API. This type has been tested against both the built-in socket type as well as those from gevent and eventlet. It also features support for sockets with timeouts set to 0 (aka nonblocking), provided the caller is prepared to handle the EWOULDBLOCK exceptions. Args: sock (socket): The connected socket to be wrapped. timeout (float): The default timeout for sends and recvs, in seconds. Set to ``None`` for no timeout, and 0 for nonblocking. Defaults to *sock*'s own timeout if already set, and 10 seconds otherwise. maxsize (int): The default maximum number of bytes to be received into the buffer before it is considered full and raises an exception. Defaults to 32 kilobytes. recvsize (int): The number of bytes to recv for every lower-level :meth:`socket.recv` call. Defaults to *maxsize*. *timeout* and *maxsize* can both be overridden on individual socket operations. All ``recv`` methods return bytestrings (:class:`bytes`) and can raise :exc:`socket.error`. :exc:`Timeout`, :exc:`ConnectionClosed`, and :exc:`MessageTooLong` all inherit from :exc:`socket.error` and exist to provide better error messages. Received bytes are always buffered, even if an exception is raised. Use :meth:`BufferedSocket.getrecvbuffer` to retrieve partial recvs. BufferedSocket does not replace the built-in socket by any means. While the overlapping parts of the API are kept parallel to the built-in :class:`socket.socket`, BufferedSocket does not inherit from socket, and most socket functionality is only available on the underlying socket. :meth:`socket.getpeername`, :meth:`socket.getsockname`, :meth:`socket.fileno`, and others are only available on the underlying socket that is wrapped. Use the ``BufferedSocket.sock`` attribute to access it. See the examples for more information on how to use BufferedSockets with built-in sockets. The BufferedSocket is threadsafe, but consider the semantics of your protocol before accessing a single socket from multiple threads. Similarly, once the BufferedSocket is constructed, avoid using the underlying socket directly. Only use it for operations unrelated to messages, e.g., :meth:`socket.getpeername`. """ def __init__(self, sock, timeout=_UNSET, maxsize=DEFAULT_MAXSIZE, recvsize=_UNSET): self.sock = sock self.rbuf = b'' self.sbuf = [] self.maxsize = int(maxsize) if timeout is _UNSET: if self.sock.gettimeout() is None: self.timeout = DEFAULT_TIMEOUT else: self.timeout = self.sock.gettimeout() else: if timeout is None: self.timeout = timeout else: self.timeout = float(timeout) if recvsize is _UNSET: self._recvsize = self.maxsize else: self._recvsize = int(recvsize) self._send_lock = RLock() self._recv_lock = RLock() def settimeout(self, timeout): "Set the default *timeout* for future operations, in seconds." self.timeout = timeout def gettimeout(self): return self.timeout def setblocking(self, blocking): self.timeout = None if blocking else 0.0 def setmaxsize(self, maxsize): """Set the default maximum buffer size *maxsize* for future operations, in bytes. Does not truncate the current buffer. """ self.maxsize = maxsize def getrecvbuffer(self): "Returns the receive buffer bytestring (rbuf)." with self._recv_lock: return self.rbuf def getsendbuffer(self): "Returns a copy of the send buffer list." with self._send_lock: return b''.join(self.sbuf) def recv(self, size, flags=0, timeout=_UNSET): """Returns **up to** *size* bytes, using the internal buffer before performing a single :meth:`socket.recv` operation. Args: size (int): The maximum number of bytes to receive. flags (int): Kept for API compatibility with sockets. Only the default, ``0``, is valid. timeout (float): The timeout for this operation. Can be ``0`` for nonblocking and ``None`` for no timeout. Defaults to the value set in the constructor of BufferedSocket. If the operation does not complete in *timeout* seconds, a :exc:`Timeout` is raised. Much like the built-in :class:`socket.socket`, if this method returns an empty string, then the socket is closed and recv buffer is empty. Further calls to recv will raise :exc:`socket.error`. """ with self._recv_lock: if timeout is _UNSET: timeout = self.timeout if flags: raise ValueError("non-zero flags not supported: %r" % flags) if len(self.rbuf) >= size: data, self.rbuf = self.rbuf[:size], self.rbuf[size:] return data if self.rbuf: ret, self.rbuf = self.rbuf, b'' return ret self.sock.settimeout(timeout) try: data = self.sock.recv(self._recvsize) except socket.timeout: raise Timeout(timeout) # check the rbuf attr for more if len(data) > size: data, self.rbuf = data[:size], data[size:] return data def peek(self, size, timeout=_UNSET): """Returns *size* bytes from the socket and/or internal buffer. Bytes are retained in BufferedSocket's internal recv buffer. To only see bytes in the recv buffer, use :meth:`getrecvbuffer`. Args: size (int): The exact number of bytes to peek at timeout (float): The timeout for this operation. Can be 0 for nonblocking and None for no timeout. Defaults to the value set in the constructor of BufferedSocket. If the appropriate number of bytes cannot be fetched from the buffer and socket before *timeout* expires, then a :exc:`Timeout` will be raised. If the connection is closed, a :exc:`ConnectionClosed` will be raised. """ with self._recv_lock: if len(self.rbuf) >= size: return self.rbuf[:size] data = self.recv_size(size, timeout=timeout) self.rbuf = data + self.rbuf return data def recv_close(self, timeout=_UNSET, maxsize=_UNSET): """Receive until the connection is closed, up to *maxsize* bytes. If more than *maxsize* bytes are received, raises :exc:`MessageTooLong`. """ # recv_close works by using recv_size to request maxsize data, # and ignoring ConnectionClose, returning and clearing the # internal buffer instead. It raises an exception if # ConnectionClosed isn't raised. with self._recv_lock: if maxsize is _UNSET: maxsize = self.maxsize if maxsize is None: maxsize = _RECV_LARGE_MAXSIZE try: recvd = self.recv_size(maxsize + 1, timeout) except ConnectionClosed: ret, self.rbuf = self.rbuf, b'' else: # put extra received bytes (now in rbuf) after recvd self.rbuf = recvd + self.rbuf size_read = min(maxsize, len(self.rbuf)) raise MessageTooLong(size_read) # check receive buffer return ret def recv_until(self, delimiter, timeout=_UNSET, maxsize=_UNSET, with_delimiter=False): """Receive until *delimiter* is found, *maxsize* bytes have been read, or *timeout* is exceeded. Args: delimiter (bytes): One or more bytes to be searched for in the socket stream. timeout (float): The timeout for this operation. Can be 0 for nonblocking and None for no timeout. Defaults to the value set in the constructor of BufferedSocket. maxsize (int): The maximum size for the internal buffer. Defaults to the value set in the constructor. with_delimiter (bool): Whether or not to include the delimiter in the output. ``False`` by default, but ``True`` is useful in cases where one is simply forwarding the messages. ``recv_until`` will raise the following exceptions: * :exc:`Timeout` if more than *timeout* seconds expire. * :exc:`ConnectionClosed` if the underlying socket is closed by the sending end. * :exc:`MessageTooLong` if the delimiter is not found in the first *maxsize* bytes. * :exc:`socket.error` if operating in nonblocking mode (*timeout* equal to 0), or if some unexpected socket error occurs, such as operating on a closed socket. """ with self._recv_lock: if maxsize is _UNSET: maxsize = self.maxsize if maxsize is None: maxsize = _RECV_LARGE_MAXSIZE if timeout is _UNSET: timeout = self.timeout len_delimiter = len(delimiter) sock = self.sock recvd = bytearray(self.rbuf) start = time.time() find_offset_start = 0 # becomes a negative index below if not timeout: # covers None (no timeout) and 0 (nonblocking) sock.settimeout(timeout) try: while 1: offset = recvd.find(delimiter, find_offset_start, maxsize) if offset != -1: # str.find returns -1 when no match found if with_delimiter: # include delimiter in return offset += len_delimiter rbuf_offset = offset else: rbuf_offset = offset + len_delimiter break elif len(recvd) > maxsize: raise MessageTooLong(maxsize, delimiter) # see rbuf if timeout: cur_timeout = timeout - (time.time() - start) if cur_timeout <= 0.0: raise socket.timeout() sock.settimeout(cur_timeout) nxt = sock.recv(self._recvsize) if not nxt: args = (len(recvd), delimiter) msg = ('connection closed after reading %s bytes' ' without finding symbol: %r' % args) raise ConnectionClosed(msg) # check the recv buffer recvd.extend(nxt) find_offset_start = -len(nxt) - len_delimiter + 1 except socket.timeout: self.rbuf = bytes(recvd) msg = ('read %s bytes without finding delimiter: %r' % (len(recvd), delimiter)) raise Timeout(timeout, msg) # check the recv buffer except Exception: self.rbuf = bytes(recvd) raise val, self.rbuf = bytes(recvd[:offset]), bytes(recvd[rbuf_offset:]) return val def recv_size(self, size, timeout=_UNSET): """Read off of the internal buffer, then off the socket, until *size* bytes have been read. Args: size (int): number of bytes to read before returning. timeout (float): The timeout for this operation. Can be 0 for nonblocking and None for no timeout. Defaults to the value set in the constructor of BufferedSocket. If the appropriate number of bytes cannot be fetched from the buffer and socket before *timeout* expires, then a :exc:`Timeout` will be raised. If the connection is closed, a :exc:`ConnectionClosed` will be raised. """ with self._recv_lock: if timeout is _UNSET: timeout = self.timeout chunks = [] total_bytes = 0 try: start = time.time() self.sock.settimeout(timeout) nxt = self.rbuf or self.sock.recv(self._recvsize) while nxt: total_bytes += len(nxt) if total_bytes >= size: break chunks.append(nxt) if timeout: cur_timeout = timeout - (time.time() - start) if cur_timeout <= 0.0: raise socket.timeout() self.sock.settimeout(cur_timeout) nxt = self.sock.recv(self._recvsize) else: msg = ('connection closed after reading %s of %s requested' ' bytes' % (total_bytes, size)) raise ConnectionClosed(msg) # check recv buffer except socket.timeout: self.rbuf = b''.join(chunks) msg = 'read %s of %s bytes' % (total_bytes, size) raise Timeout(timeout, msg) # check recv buffer except Exception: # received data is still buffered in the case of errors self.rbuf = b''.join(chunks) raise extra_bytes = total_bytes - size if extra_bytes: last, self.rbuf = nxt[:-extra_bytes], nxt[-extra_bytes:] else: last, self.rbuf = nxt, b'' chunks.append(last) return b''.join(chunks) def send(self, data, flags=0, timeout=_UNSET): """Send the contents of the internal send buffer, as well as *data*, to the receiving end of the connection. Returns the total number of bytes sent. If no exception is raised, all of *data* was sent and the internal send buffer is empty. Args: data (bytes): The bytes to send. flags (int): Kept for API compatibility with sockets. Only the default 0 is valid. timeout (float): The timeout for this operation. Can be 0 for nonblocking and None for no timeout. Defaults to the value set in the constructor of BufferedSocket. Will raise :exc:`Timeout` if the send operation fails to complete before *timeout*. In the event of an exception, use :meth:`BufferedSocket.getsendbuffer` to see which data was unsent. """ with self._send_lock: if timeout is _UNSET: timeout = self.timeout if flags: raise ValueError("non-zero flags not supported") sbuf = self.sbuf sbuf.append(data) if len(sbuf) > 1: sbuf[:] = [b''.join([s for s in sbuf if s])] self.sock.settimeout(timeout) start, total_sent = time.time(), 0 try: while sbuf[0]: sent = self.sock.send(sbuf[0]) total_sent += sent sbuf[0] = sbuf[0][sent:] if timeout: cur_timeout = timeout - (time.time() - start) if cur_timeout <= 0.0: raise socket.timeout() self.sock.settimeout(cur_timeout) except socket.timeout: raise Timeout(timeout, '%s bytes unsent' % len(sbuf[0])) return total_sent def sendall(self, data, flags=0, timeout=_UNSET): """A passthrough to :meth:`~BufferedSocket.send`, retained for parallelism to the :class:`socket.socket` API. """ return self.send(data, flags, timeout) def flush(self): "Send the contents of the internal send buffer." with self._send_lock: self.send(b'') return def buffer(self, data): "Buffer *data* bytes for the next send operation." with self._send_lock: self.sbuf.append(data) return # # # # # # Passing through some socket basics # # # def getsockname(self): """Convenience function to return the wrapped socket's own address. See :meth:`socket.getsockname` for more details. """ return self.sock.getsockname() def getpeername(self): """Convenience function to return the remote address to which the wrapped socket is connected. See :meth:`socket.getpeername` for more details. """ return self.sock.getpeername() def getsockopt(self, level, optname, buflen=None): """Convenience function passing through to the wrapped socket's :meth:`socket.getsockopt`. """ args = (level, optname) if buflen is not None: args += (buflen,) return self.sock.getsockopt(*args) def setsockopt(self, level, optname, value): """Convenience function passing through to the wrapped socket's :meth:`socket.setsockopt`. """ return self.sock.setsockopt(level, optname, value) @property def type(self): """A passthrough to the wrapped socket's type. Valid usages should only ever see :data:`socket.SOCK_STREAM`. """ return self.sock.type @property def family(self): """A passthrough to the wrapped socket's family. BufferedSocket supports all widely-used families, so this read-only attribute can be one of :data:`socket.AF_INET` for IP, :data:`socket.AF_INET6` for IPv6, and :data:`socket.AF_UNIX` for UDS. """ return self.sock.family @property def proto(self): """A passthrough to the wrapped socket's protocol. The ``proto`` attribute is very rarely used, so it's always 0, meaning "the default" protocol. Pretty much all the practical information is in :attr:`~BufferedSocket.type` and :attr:`~BufferedSocket.family`, so you can go back to never thinking about this. """ return self.sock.proto # # # # # # Now for some more advanced interpretations of the builtin socket # # # def fileno(self): """Returns the file descriptor of the wrapped socket. -1 if it has been closed on this end. Note that this makes the BufferedSocket selectable, i.e., usable for operating system event loops without any external libraries. Keep in mind that the operating system cannot know about data in BufferedSocket's internal buffer. Exercise discipline with calling ``recv*`` functions. """ return self.sock.fileno() def close(self): """Closes the wrapped socket, and empties the internal buffers. The send buffer is not flushed automatically, so if you have been calling :meth:`~BufferedSocket.buffer`, be sure to call :meth:`~BufferedSocket.flush` before calling this method. After calling this method, future socket operations will raise :exc:`socket.error`. """ with self._recv_lock: with self._send_lock: self.rbuf = b'' self.rbuf_unconsumed = self.rbuf self.sbuf[:] = [] self.sock.close() return def shutdown(self, how): """Convenience method which passes through to the wrapped socket's :meth:`~socket.shutdown`. Semantics vary by platform, so no special internal handling is done with the buffers. This method exists to facilitate the most common usage, wherein a full ``shutdown`` is followed by a :meth:`~BufferedSocket.close`. Developers requiring more support, please open `an issue`_. .. _an issue: https://github.com/mahmoud/boltons/issues """ with self._recv_lock: with self._send_lock: self.sock.shutdown(how) return # end BufferedSocket class Error(socket.error): """A subclass of :exc:`socket.error` from which all other ``socketutils`` exceptions inherit. When using :class:`BufferedSocket` and other ``socketutils`` types, generally you want to catch one of the specific exception types below, or :exc:`socket.error`. """ pass class ConnectionClosed(Error): """Raised when receiving and the connection is unexpectedly closed from the sending end. Raised from :class:`BufferedSocket`'s :meth:`~BufferedSocket.peek`, :meth:`~BufferedSocket.recv_until`, and :meth:`~BufferedSocket.recv_size`, and never from its :meth:`~BufferedSocket.recv` or :meth:`~BufferedSocket.recv_close`. """ pass class MessageTooLong(Error): """Raised from :meth:`BufferedSocket.recv_until` and :meth:`BufferedSocket.recv_closed` when more than *maxsize* bytes are read without encountering the delimiter or a closed connection, respectively. """ def __init__(self, bytes_read=None, delimiter=None): msg = 'message exceeded maximum size' if bytes_read is not None: msg += '. %s bytes read' % (bytes_read,) if delimiter is not None: msg += '. Delimiter not found: %r' % (delimiter,) super(MessageTooLong, self).__init__(msg) class Timeout(socket.timeout, Error): """Inheriting from :exc:`socket.timeout`, Timeout is used to indicate when a socket operation did not complete within the time specified. Raised from any of :class:`BufferedSocket`'s ``recv`` methods. """ def __init__(self, timeout, extra=""): msg = 'socket operation timed out' if timeout is not None: msg += ' after %sms.' % (timeout * 1000) if extra: msg += ' ' + extra super(Timeout, self).__init__(msg) class NetstringSocket(object): """ Reads and writes using the netstring protocol. More info: https://en.wikipedia.org/wiki/Netstring Even more info: http://cr.yp.to/proto/netstrings.txt """ def __init__(self, sock, timeout=DEFAULT_TIMEOUT, maxsize=DEFAULT_MAXSIZE): self.bsock = BufferedSocket(sock) self.timeout = timeout self.maxsize = maxsize self._msgsize_maxsize = len(str(maxsize)) + 1 # len(str()) == log10 def fileno(self): return self.bsock.fileno() def settimeout(self, timeout): self.timeout = timeout def setmaxsize(self, maxsize): self.maxsize = maxsize self._msgsize_maxsize = len(str(maxsize)) + 1 # len(str()) == log10 def read_ns(self, timeout=_UNSET, maxsize=_UNSET): if timeout is _UNSET: timeout = self.timeout size_prefix = self.bsock.recv_until(b':', timeout=self.timeout, maxsize=self._msgsize_maxsize) try: size = int(size_prefix) except ValueError: raise NetstringInvalidSize('netstring message size must be valid' ' integer, not %r' % size_prefix) if size > self.maxsize: raise NetstringMessageTooLong(size, self.maxsize) payload = self.bsock.recv_size(size) if self.bsock.recv(1) != b',': raise NetstringProtocolError("expected trailing ',' after message") return payload def write_ns(self, payload): size = len(payload) if size > self.maxsize: raise NetstringMessageTooLong(size, self.maxsize) data = str(size).encode('ascii') + b':' + payload + b',' self.bsock.send(data) class NetstringProtocolError(Error): "Base class for all of socketutils' Netstring exception types." pass class NetstringInvalidSize(NetstringProtocolError): """NetstringInvalidSize is raised when the ``:``-delimited size prefix of the message does not contain a valid integer. Message showing valid size:: 5:hello, Here the ``5`` is the size. Anything in this prefix position that is not parsable as a Python integer (i.e., :class:`int`) will raise this exception. """ def __init__(self, msg): super(NetstringInvalidSize, self).__init__(msg) class NetstringMessageTooLong(NetstringProtocolError): """NetstringMessageTooLong is raised when the size prefix contains a valid integer, but that integer is larger than the :class:`NetstringSocket`'s configured *maxsize*. When this exception is raised, it's recommended to simply close the connection instead of trying to recover. """ def __init__(self, size, maxsize): msg = ('netstring message length exceeds configured maxsize: %s > %s' % (size, maxsize)) super(NetstringMessageTooLong, self).__init__(msg) """ attrs worth adding/passing through: properties: type, proto For its main functionality, BufferedSocket can wrap any object that has the following methods: - gettimeout() - settimeout() - recv(size) - send(data) The following methods are passed through: ... """ # TODO: buffered socket check socket.type == SOCK_STREAM? # TODO: make recv_until support taking a regex # TODO: including the delimiter in the recv_until return is not # necessary, as ConnectionClosed differentiates empty messages # from socket closes. boltons-16.4.1/boltons/fileutils.py0000644000175000017500000005270112727357214015436 0ustar hlehle# -*- coding: utf-8 -*- """Virtually every Python programmer has used Python for wrangling disk contents, and ``fileutils`` collects solutions to some of the most commonly-found gaps in the standard library. """ from __future__ import print_function import os import re import sys import stat import errno import fnmatch from shutil import copy2, copystat, Error __all__ = ['mkdir_p', 'atomic_save', 'AtomicSaver', 'FilePerms', 'iter_find_files', 'copytree'] FULL_PERMS = 511 # 0777 that both Python 2 and 3 can digest RW_PERMS = 438 _SINGLE_FULL_PERM = 7 # or 07 in Python 2 try: basestring except NameError: unicode = str # Python 3 compat basestring = (str, bytes) def mkdir_p(path): """Creates a directory and any parent directories that may need to be created along the way, without raising errors for any existing directories. This function mimics the behavior of the ``mkdir -p`` command available in Linux/BSD environments, but also works on Windows. """ try: os.makedirs(path) except OSError as exc: if exc.errno == errno.EEXIST and os.path.isdir(path): return raise return class FilePerms(object): """The :class:`FilePerms` type is used to represent standard POSIX filesystem permissions: * Read * Write * Execute Across three classes of user: * Owning (u)ser * Owner's (g)roup * Any (o)ther user This class assists with computing new permissions, as well as working with numeric octal ``777``-style and ``rwx``-style permissions. Currently it only considers the bottom 9 permission bits; it does not support sticky bits or more advanced permission systems. Args: user (str): A string in the 'rwx' format, omitting characters for which owning user's permissions are not provided. group (str): A string in the 'rwx' format, omitting characters for which owning group permissions are not provided. other (str): A string in the 'rwx' format, omitting characters for which owning other/world permissions are not provided. There are many ways to use :class:`FilePerms`: >>> FilePerms(user='rwx', group='xrw', other='wxr') # note character order FilePerms(user='rwx', group='rwx', other='rwx') >>> int(FilePerms('r', 'r', '')) 288 >>> oct(288)[-3:] # XXX Py3k '440' See also the :meth:`FilePerms.from_int` and :meth:`FilePerms.from_path` classmethods for useful alternative ways to construct :class:`FilePerms` objects. """ # TODO: consider more than the lower 9 bits class _FilePermProperty(object): _perm_chars = 'rwx' _perm_set = frozenset('rwx') _perm_val = {'r': 4, 'w': 2, 'x': 1} # for sorting def __init__(self, attribute, offset): self.attribute = attribute self.offset = offset def __get__(self, fp_obj, type_=None): if fp_obj is None: return self return getattr(fp_obj, self.attribute) def __set__(self, fp_obj, value): cur = getattr(fp_obj, self.attribute) if cur == value: return try: invalid_chars = set(str(value)) - self._perm_set except TypeError: raise TypeError('expected string, not %r' % value) if invalid_chars: raise ValueError('got invalid chars %r in permission' ' specification %r, expected empty string' ' or one or more of %r' % (invalid_chars, value, self._perm_chars)) sort_key = lambda c: self._perm_val[c] new_value = ''.join(sorted(set(value), key=sort_key, reverse=True)) setattr(fp_obj, self.attribute, new_value) self._update_integer(fp_obj, new_value) def _update_integer(self, fp_obj, value): mode = 0 key = 'xwr' for symbol in value: bit = 2 ** key.index(symbol) mode |= (bit << (self.offset * 3)) fp_obj._integer |= mode def __init__(self, user='', group='', other=''): self._user, self._group, self._other = '', '', '' self._integer = 0 self.user = user self.group = group self.other = other @classmethod def from_int(cls, i): """Create a :class:`FilePerms` object from an integer. >>> FilePerms.from_int(0o644) # note the leading zero-oh for octal FilePerms(user='rw', group='r', other='r') """ i &= FULL_PERMS key = ('', 'x', 'w', 'xw', 'r', 'rx', 'rw', 'rwx') parts = [] while i: parts.append(key[i & _SINGLE_FULL_PERM]) i >>= 3 parts.reverse() return cls(*parts) @classmethod def from_path(cls, path): """Make a new :class:`FilePerms` object based on the permissions assigned to the file or directory at *path*. Args: path (str): Filesystem path of the target file. Here's an example that holds true on most systems: >>> import tempfile >>> 'r' in FilePerms.from_path(tempfile.tempdir).user True """ stat_res = os.stat(path) return cls.from_int(stat.S_IMODE(stat_res.st_mode)) def __int__(self): return self._integer # Sphinx tip: attribute docstrings come after the attribute user = _FilePermProperty('_user', 2) "Stores the ``rwx``-formatted *user* permission." group = _FilePermProperty('_group', 1) "Stores the ``rwx``-formatted *group* permission." other = _FilePermProperty('_other', 0) "Stores the ``rwx``-formatted *other* permission." def __repr__(self): cn = self.__class__.__name__ return ('%s(user=%r, group=%r, other=%r)' % (cn, self.user, self.group, self.other)) #### _TEXT_OPENFLAGS = os.O_RDWR | os.O_CREAT | os.O_EXCL if hasattr(os, 'O_NOINHERIT'): _TEXT_OPENFLAGS |= os.O_NOINHERIT if hasattr(os, 'O_NOFOLLOW'): _TEXT_OPENFLAGS |= os.O_NOFOLLOW _BIN_OPENFLAGS = _TEXT_OPENFLAGS if hasattr(os, 'O_BINARY'): _BIN_OPENFLAGS |= os.O_BINARY try: import fcntl as fcntl except ImportError: def set_cloexec(fd): "Dummy set_cloexec for platforms without fcntl support" pass else: def set_cloexec(fd): """Does a best-effort :func:`fcntl.fcntl` call to set a fd to be automatically closed by any future child processes. Implementation from the :mod:`tempfile` module. """ try: flags = fcntl.fcntl(fd, fcntl.F_GETFD, 0) except IOError: pass else: # flags read successfully, modify flags |= fcntl.FD_CLOEXEC fcntl.fcntl(fd, fcntl.F_SETFD, flags) return def atomic_save(dest_path, **kwargs): """A convenient interface to the :class:`AtomicSaver` type. See the :class:`AtomicSaver` documentation for details. """ return AtomicSaver(dest_path, **kwargs) def path_to_unicode(path): if isinstance(path, unicode): return path encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() return path.decode(encoding) if os.name == 'nt': import ctypes from ctypes import c_wchar_p from ctypes.wintypes import DWORD, LPVOID _ReplaceFile = ctypes.windll.kernel32.ReplaceFile _ReplaceFile.argtypes = [c_wchar_p, c_wchar_p, c_wchar_p, DWORD, LPVOID, LPVOID] def replace(src, dst): # argument names match stdlib docs, docstring below try: # ReplaceFile fails if the dest file does not exist, so # first try to rename it into position os.rename(src, dst) return except WindowsError as we: if we.errno == errno.EEXIST: pass # continue with the ReplaceFile logic below else: raise src = path_to_unicode(src) dst = path_to_unicode(dst) res = _ReplaceFile(c_wchar_p(dst), c_wchar_p(src), None, 0, None, None) if not res: raise OSError('failed to replace %r with %r' % (dst, src)) return def atomic_rename(src, dst, overwrite=False): "Rename *src* to *dst*, replacing *dst* if *overwrite is True" if overwrite: replace(src, dst) else: os.rename(src, dst) return else: # wrapper func for cross compat + docs def replace(src, dst): # os.replace does the same thing on unix return os.rename(src, dst) def atomic_rename(src, dst, overwrite=False): "Rename *src* to *dst*, replacing *dst* if *overwrite is True" if overwrite: os.rename(src, dst) else: os.link(src, dst) os.unlink(dst) return _atomic_rename = atomic_rename # backwards compat replace.__doc__ = """Similar to :func:`os.replace` in Python 3.3+, this function will atomically create or replace the file at path *dst* with the file at path *src*. On Windows, this function uses the ReplaceFile API for maximum possible atomicity on a range of filesystems. """ class AtomicSaver(object): """``AtomicSaver`` is a configurable `context manager`_ that provides a writable :class:`file` which will be moved into place as long as no exceptions are raised within the context manager's block. These "part files" are created in the same directory as the destination path to ensure atomic move operations (i.e., no cross-filesystem moves occur). Args: dest_path (str): The path where the completed file will be written. overwrite (bool): Whether to overwrite the destination file if it exists at completion time. Defaults to ``True``. file_perms (int): Integer representation of file permissions for the newly-created file. Defaults are, when the destination path already exists, to copy the permissions from the previous file, or if the file did not exist, to respect the user's configured `umask`_, usually resulting in octal 0644 or 0664. part_file (str): Name of the temporary *part_file*. Defaults to *dest_path* + ``.part``. Note that this argument is just the filename, and not the full path of the part file. To guarantee atomic saves, part files are always created in the same directory as the destination path. overwrite_part (bool): Whether to overwrite the *part_file*, should it exist at setup time. Defaults to ``False``, which results in an :exc:`OSError` being raised on pre-existing part files. Be careful of setting this to ``True`` in situations when multiple threads or processes could be writing to the same part file. rm_part_on_exc (bool): Remove *part_file* on exception cases. Defaults to ``True``, but ``False`` can be useful for recovery in some cases. Note that resumption is not automatic and by default an :exc:`OSError` is raised if the *part_file* exists. Practically, the AtomicSaver serves a few purposes: * Avoiding overwriting an existing, valid file with a partially written one. * Providing a reasonable guarantee that a part file only has one writer at a time. * Optional recovery of partial data in failure cases. .. _context manager: https://docs.python.org/2/reference/compound_stmts.html#with .. _umask: https://en.wikipedia.org/wiki/Umask """ _default_file_perms = RW_PERMS # TODO: option to abort if target file modify date has changed since start? def __init__(self, dest_path, **kwargs): self.dest_path = dest_path self.overwrite = kwargs.pop('overwrite', True) self.file_perms = kwargs.pop('file_perms', None) self.overwrite_part = kwargs.pop('overwrite_part', False) self.part_filename = kwargs.pop('part_file', None) self.rm_part_on_exc = kwargs.pop('rm_part_on_exc', True) self.text_mode = kwargs.pop('text_mode', False) # for windows self.buffering = kwargs.pop('buffering', -1) if kwargs: raise TypeError('unexpected kwargs: %r' % (kwargs.keys(),)) self.dest_path = os.path.abspath(self.dest_path) self.dest_dir = os.path.dirname(self.dest_path) if not self.part_filename: self.part_path = dest_path + '.part' else: self.part_path = os.path.join(self.dest_dir, self.part_filename) self.mode = 'w+' if self.text_mode else 'w+b' self.open_flags = _TEXT_OPENFLAGS if self.text_mode else _BIN_OPENFLAGS self.part_file = None def _open_part_file(self): do_chmod = True file_perms = self.file_perms if file_perms is None: try: # try to copy from file being replaced stat_res = os.stat(self.dest_path) file_perms = stat.S_IMODE(stat_res.st_mode) except (OSError, IOError): # default if no destination file exists file_perms = self._default_file_perms do_chmod = False # respect the umask fd = os.open(self.part_path, self.open_flags, file_perms) set_cloexec(fd) self.part_file = os.fdopen(fd, self.mode, self.buffering) # if default perms are overridden by the user or previous dest_path # chmod away the effects of the umask if do_chmod: try: os.chmod(self.part_path, file_perms) except (OSError, IOError): self.part_file.close() raise return def setup(self): """Called on context manager entry (the :keyword:`with` statement), the ``setup()`` method creates the temporary file in the same directory as the destination file. ``setup()`` tests for a writable directory with rename permissions early, as the part file may not be written to immediately (not using :func:`os.access` because of the potential issues of effective vs. real privileges). If the caller is not using the :class:`AtomicSaver` as a context manager, this method should be called explicitly before writing. """ if os.path.lexists(self.dest_path): if not self.overwrite: raise OSError(errno.EEXIST, 'Overwrite disabled and file already exists', self.dest_path) if self.overwrite_part and os.path.lexists(self.part_path): os.unlink(self.part_path) self._open_part_file() return def __enter__(self): self.setup() return self.part_file def __exit__(self, exc_type, exc_val, exc_tb): self.part_file.close() if exc_type: if self.rm_part_on_exc: try: os.unlink(self.part_path) except Exception: pass # avoid masking original error return try: atomic_rename(self.part_path, self.dest_path, overwrite=self.overwrite) except OSError: if self.rm_part_on_exc: try: os.unlink(self.part_path) except Exception: pass # avoid masking original error raise # could not save destination file return _CUR_DIR = os.path.dirname(os.path.abspath(__file__)) def iter_find_files(directory, patterns, ignored=None): """Returns a generator that yields file paths under a *directory*, matching *patterns* using `glob`_ syntax (e.g., ``*.txt``). Also supports *ignored* patterns. Args: directory (str): Path that serves as the root of the search. Yielded paths will include this as a prefix. patterns (str or list): A single pattern or list of glob-formatted patterns to find under *directory*. ignored (str or list): A single pattern or list of glob-formatted patterns to ignore. For example, finding Python files in the current directory: >>> filenames = sorted(iter_find_files(_CUR_DIR, '*.py')) >>> os.path.basename(filenames[-1]) 'typeutils.py' Or, Python files while ignoring emacs lockfiles: >>> filenames = iter_find_files(_CUR_DIR, '*.py', ignored='.#*') .. _glob: https://en.wikipedia.org/wiki/Glob_%28programming%29 """ if isinstance(patterns, basestring): patterns = [patterns] pats_re = re.compile('|'.join([fnmatch.translate(p) for p in patterns])) if not ignored: ignored = [] elif isinstance(ignored, basestring): ignored = [ignored] ign_re = re.compile('|'.join([fnmatch.translate(p) for p in ignored])) for root, dirs, files in os.walk(directory): for basename in files: if pats_re.match(basename): if ignored and ign_re.match(basename): continue filename = os.path.join(root, basename) yield filename return def copy_tree(src, dst, symlinks=False, ignore=None): """The ``copy_tree`` function is an exact copy of the built-in :func:`shutil.copytree`, with one key difference: it will not raise an exception if part of the tree already exists. It achieves this by using :func:`mkdir_p`. Args: src (str): Path of the source directory to copy. dst (str): Destination path. Existing directories accepted. symlinks (bool): If ``True``, copy symlinks rather than their contents. ignore (callable): A callable that takes a path and directory listing, returning the files within the listing to be ignored. For more details, check out :func:`shutil.copytree` and :func:`shutil.copy2`. """ names = os.listdir(src) if ignore is not None: ignored_names = ignore(src, names) else: ignored_names = set() mkdir_p(dst) errors = [] for name in names: if name in ignored_names: continue srcname = os.path.join(src, name) dstname = os.path.join(dst, name) try: if symlinks and os.path.islink(srcname): linkto = os.readlink(srcname) os.symlink(linkto, dstname) elif os.path.isdir(srcname): copytree(srcname, dstname, symlinks, ignore) else: # Will raise a SpecialFileError for unsupported file types copy2(srcname, dstname) # catch the Error from the recursive copytree so that we can # continue with other files except Error as e: errors.extend(e.args[0]) except EnvironmentError as why: errors.append((srcname, dstname, str(why))) try: copystat(src, dst) except OSError as why: if WindowsError is not None and isinstance(why, WindowsError): # Copying file access times may fail on Windows pass else: errors.append((src, dst, str(why))) if errors: raise Error(errors) copytree = copy_tree # alias for drop-in replacement of shutil try: file except NameError: file = object # like open(os.devnull) but with even fewer side effects class DummyFile(file): # TODO: raise ValueErrors on closed for all methods? # TODO: enforce read/write def __init__(self, path, mode='r', buffering=None): self.name = path self.mode = mode self.closed = False self.errors = None self.isatty = False self.encoding = None self.newlines = None self.softspace = 0 def close(self): self.closed = True def fileno(self): return -1 def flush(self): if self.closed: raise ValueError('I/O operation on a closed file') return def next(self): raise StopIteration() def read(self, size=0): if self.closed: raise ValueError('I/O operation on a closed file') return '' def readline(self, size=0): if self.closed: raise ValueError('I/O operation on a closed file') return '' def readlines(self, size=0): if self.closed: raise ValueError('I/O operation on a closed file') return [] def seek(self): if self.closed: raise ValueError('I/O operation on a closed file') return def tell(self): if self.closed: raise ValueError('I/O operation on a closed file') return 0 def truncate(self): if self.closed: raise ValueError('I/O operation on a closed file') return def write(self, string): if self.closed: raise ValueError('I/O operation on a closed file') return def writelines(self, list_of_strings): if self.closed: raise ValueError('I/O operation on a closed file') return def __next__(self): raise StopIteration() def __enter__(self): if self.closed: raise ValueError('I/O operation on a closed file') return def __exit__(self, exc_type, exc_val, exc_tb): return if __name__ == '__main__': with atomic_save('/tmp/final.txt') as f: f.write('rofl') f.write('\n') boltons-16.4.1/boltons/gcutils.py0000644000175000017500000001126212727357214015105 0ustar hlehle# -*- coding: utf-8 -*- """The Python Garbage Collector (`GC`_) doesn't usually get too much attention, probably because: - Python's `reference counting`_ effectively handles the vast majority of unused objects - People are slowly learning to avoid implementing `object.__del__()`_ - The collection itself strikes a good balance between simplicity and power (`tunable generation sizes`_) - The collector itself is fast and rarely the cause of long pauses associated with GC in other runtimes Even so, for many applications, the time will come when the developer will need to track down: - Circular references - Misbehaving objects (locks, ``__del__()``) - Memory leaks - Or just ways to shave off a couple percent of execution time Thanks to the :mod:`gc` module, the GC is a well-instrumented entry point for exactly these tasks, and ``gcutils`` aims to facilitate it further. .. _GC: https://docs.python.org/2/glossary.html#term-garbage-collection .. _reference counting: https://docs.python.org/2/glossary.html#term-reference-count .. _object.__del__(): https://docs.python.org/2/glossary.html#term-reference-count .. _tunable generation sizes: https://docs.python.org/2/library/gc.html#gc.set_threshold """ # TODO: type survey from __future__ import print_function import gc import sys __all__ = ['get_all', 'GCToggler', 'toggle_gc', 'toggle_gc_postcollect'] def get_all(type_obj, include_subtypes=True): """Get a list containing all instances of a given type. This will work for the vast majority of types out there. >>> class Ratking(object): pass >>> wiki, hak, sport = Ratking(), Ratking(), Ratking() >>> len(get_all(Ratking)) 3 However, there are some exceptions. For example, ``get_all(bool)`` returns an empty list because ``True`` and ``False`` are themselves built-in and not tracked. >>> get_all(bool) [] Still, it's not hard to see how this functionality can be used to find all instances of a leaking type and track them down further using :func:`gc.get_referrers` and :func:`gc.get_referents`. ``get_all()`` is optimized such that getting instances of user-created types is quite fast. Setting *include_subtypes* to ``False`` will further increase performance in cases where instances of subtypes aren't required. .. note:: There are no guarantees about the state of objects returned by ``get_all()``, especially in concurrent environments. For instance, it is possible for an object to be in the middle of executing its ``__init__()`` and be only partially constructed. """ # TODO: old-style classes if not isinstance(type_obj, type): raise TypeError('expected a type, not %r' % type_obj) try: type_is_tracked = gc.is_tracked(type_obj) except AttributeError: type_is_tracked = False # Python 2.6 and below don't get the speedup if type_is_tracked: to_check = gc.get_referrers(type_obj) else: to_check = gc.get_objects() if include_subtypes: ret = [x for x in to_check if isinstance(x, type_obj)] else: ret = [x for x in to_check if type(x) is type_obj] return ret _IS_PYPY = '__pypy__' in sys.builtin_module_names if _IS_PYPY: # pypy's gc is just different, y'all del get_all class GCToggler(object): """The ``GCToggler`` is a context-manager that allows one to safely take more control of your garbage collection schedule. Anecdotal experience says certain object-creation-heavy tasks see speedups of around 10% by simply doing one explicit collection at the very end, especially if most of the objects will stay resident. Two GCTogglers are already present in the ``gcutils`` module: - :data:`toggle_gc` simply turns off GC at context entrance, and re-enables at exit - :data:`toggle_gc_postcollect` does the same, but triggers an explicit collection after re-enabling. >>> with toggle_gc: ... x = [object() for i in range(1000)] Between those two instances, the ``GCToggler`` type probably won't be used much directly, but is documented for inheritance purposes. """ def __init__(self, postcollect=False): self.postcollect = postcollect def __enter__(self): gc.disable() def __exit__(self, exc_type, exc_val, exc_tb): gc.enable() if self.postcollect: gc.collect() toggle_gc = GCToggler() """A context manager for disabling GC for a code block. See :class:`GCToggler` for more details.""" toggle_gc_postcollect = GCToggler(postcollect=True) """A context manager for disabling GC for a code block, and collecting before re-enabling. See :class:`GCToggler` for more details.""" boltons-16.4.1/boltons/cacheutils.py0000644000175000017500000006627512727357214015575 0ustar hlehle# -*- coding: utf-8 -*- """``cacheutils`` contains consistent implementations of fundamental cache types. Currently there are two to choose from: * :class:`LRI` - Least-recently inserted * :class:`LRU` - Least-recently used Both caches are :class:`dict` subtypes, designed to be as interchangeable as possible, to facilitate experimentation. A key practice with performance enhancement with caching is ensuring that the caching strategy is working. If the cache is constantly missing, it is just adding more overhead and code complexity. The standard statistics are: * ``hit_count`` - the number of times the queried key has been in the cache * ``miss_count`` - the number of times a key has been absent and/or fetched by the cache * ``soft_miss_count`` - the number of times a key has been absent, but a default has been provided by the caller, as with :meth:`dict.get` and :meth:`dict.setdefault`. Soft misses are a subset of misses, so this number is always less than or equal to ``miss_count``. Additionally, ``cacheutils`` provides :class:`ThresholdCounter`, a cache-like bounded counter useful for online statistics collection. Learn more about `caching algorithms on Wikipedia `_. """ # TODO: TimedLRI # TODO: support 0 max_size? import itertools from collections import deque from operator import attrgetter try: from threading import RLock except Exception: class RLock(object): 'Dummy reentrant lock for builds without threads' def __enter__(self): pass def __exit__(self, exctype, excinst, exctb): pass try: from boltons.typeutils import make_sentinel _MISSING = make_sentinel(var_name='_MISSING') _KWARG_MARK = make_sentinel(var_name='_KWARG_MARK') except ImportError: _MISSING = object() _KWARG_MARK = object() try: xrange except NameError: # py3 xrange = range unicode, str, bytes, basestring = str, bytes, bytes, (str, bytes) PREV, NEXT, KEY, VALUE = range(4) # names for the link fields DEFAULT_MAX_SIZE = 128 class LRU(dict): """The ``LRU`` is :class:`dict` subtype implementation of the *Least-Recently Used* caching strategy. Args: max_size (int): Max number of items to cache. Defaults to ``128``. values (iterable): Initial values for the cache. Defaults to ``None``. on_miss (callable): a callable which accepts a single argument, the key not present in the cache, and returns the value to be cached. >>> cap_cache = LRU(max_size=2) >>> cap_cache['a'], cap_cache['b'] = 'A', 'B' >>> from pprint import pprint as pp >>> pp(dict(cap_cache)) {'a': 'A', 'b': 'B'} >>> [cap_cache['b'] for i in range(3)][0] 'B' >>> cap_cache['c'] = 'C' >>> print(cap_cache.get('a')) None This cache is also instrumented with statistics collection. ``hit_count``, ``miss_count``, and ``soft_miss_count`` are all integer members that can be used to introspect the performance of the cache. ("Soft" misses are misses that did not raise :exc:`KeyError`, e.g., ``LRU.get()`` or ``on_miss`` was used to cache a default. >>> cap_cache.hit_count, cap_cache.miss_count, cap_cache.soft_miss_count (3, 1, 1) Other than the size-limiting caching behavior and statistics, ``LRU`` acts like its parent class, the built-in Python :class:`dict`. """ def __init__(self, max_size=DEFAULT_MAX_SIZE, values=None, on_miss=None): if max_size <= 0: raise ValueError('expected max_size > 0, not %r' % max_size) self.hit_count = self.miss_count = self.soft_miss_count = 0 self.max_size = max_size self._lock = RLock() self._init_ll() if on_miss is not None and not callable(on_miss): raise TypeError('expected on_miss to be a callable' ' (or None), not %r' % on_miss) self.on_miss = on_miss if values: self.update(values) # TODO: fromkeys()? # linked list manipulation methods. # # invariants: # 1) 'anchor' is the sentinel node in the doubly linked list. there is # always only one, and its KEY and VALUE are both _MISSING. # 2) the most recently accessed node comes immediately before 'anchor'. # 3) the least recently accessed node comes immediately after 'anchor'. def _init_ll(self): anchor = [] anchor[:] = [anchor, anchor, _MISSING, _MISSING] # a link lookup table for finding linked list links in O(1) # time. self._link_lookup = {} self._anchor = anchor def _print_ll(self): link = self._anchor print('***') while True: print(link[KEY], link[VALUE]) link = link[NEXT] if link is self._anchor: break print('***') return def _get_link_and_move_to_front_of_ll(self, key): # find what will become the newest link. this may raise a # KeyError, which is useful to __getitem__ and __setitem__ newest = self._link_lookup[key] # splice out what will become the newest link. newest[PREV][NEXT] = newest[NEXT] newest[NEXT][PREV] = newest[PREV] # move what will become the newest link immediately before # anchor (invariant 2) anchor = self._anchor second_newest = anchor[PREV] second_newest[NEXT] = anchor[PREV] = newest newest[PREV] = second_newest newest[NEXT] = anchor return newest def _set_key_and_add_to_front_of_ll(self, key, value): # create a new link and place it immediately before anchor # (invariant 2). anchor = self._anchor second_newest = anchor[PREV] newest = [second_newest, anchor, key, value] second_newest[NEXT] = anchor[PREV] = newest self._link_lookup[key] = newest def _set_key_and_evict_last_in_ll(self, key, value): # the link after anchor is the oldest in the linked list # (invariant 3). the current anchor becomes a link that holds # the newest key, and the oldest link becomes the new anchor # (invariant 1). now the newest link comes before anchor # (invariant 2). no links are moved; only their keys # and values are changed. oldanchor = self._anchor oldanchor[KEY] = key oldanchor[VALUE] = value self._anchor = anchor = oldanchor[NEXT] evicted = anchor[KEY] anchor[KEY] = anchor[VALUE] = _MISSING del self._link_lookup[evicted] self._link_lookup[key] = oldanchor return evicted def _remove_from_ll(self, key): # splice a link out of the list and drop it from our lookup # table. link = self._link_lookup.pop(key) link[PREV][NEXT] = link[NEXT] link[NEXT][PREV] = link[PREV] def __setitem__(self, key, value): with self._lock: try: link = self._get_link_and_move_to_front_of_ll(key) except KeyError: if len(self) < self.max_size: self._set_key_and_add_to_front_of_ll(key, value) else: evicted = self._set_key_and_evict_last_in_ll(key, value) super(LRU, self).__delitem__(evicted) super(LRU, self).__setitem__(key, value) else: link[VALUE] = value def __getitem__(self, key): with self._lock: try: link = self._get_link_and_move_to_front_of_ll(key) except KeyError: self.miss_count += 1 if not self.on_miss: raise ret = self[key] = self.on_miss(key) return ret self.hit_count += 1 return link[VALUE] def get(self, key, default=None): try: return self[key] except KeyError: self.soft_miss_count += 1 return default def __delitem__(self, key): with self._lock: super(LRU, self).__delitem__(key) self._remove_from_ll(key) def pop(self, key, default=_MISSING): # NB: hit/miss counts are bypassed for pop() with self._lock: try: ret = super(LRU, self).pop(key) except KeyError: if default is _MISSING: raise ret = default else: self._remove_from_ll(key) return ret def popitem(self): with self._lock: item = super(LRU, self).popitem() self._remove_from_ll(item[0]) return item def clear(self): with self._lock: super(LRU, self).clear() self._init_ll() def copy(self): return self.__class__(max_size=self.max_size, values=self) def setdefault(self, key, default=None): with self._lock: try: return self[key] except KeyError: self.soft_miss_count += 1 self[key] = default return default def update(self, E, **F): # E and F are throwback names to the dict() __doc__ with self._lock: if E is self: return setitem = self.__setitem__ if callable(getattr(E, 'keys', None)): for k in E.keys(): setitem(k, E[k]) else: for k, v in E: setitem(k, v) for k in F: setitem(k, F[k]) return def __eq__(self, other): with self._lock: if self is other: return True if len(other) != len(self): return False if not isinstance(other, LRU): return other == self return super(LRU, self).__eq__(other) def __ne__(self, other): return not (self == other) def __repr__(self): cn = self.__class__.__name__ val_map = super(LRU, self).__repr__() return ('%s(max_size=%r, on_miss=%r, values=%s)' % (cn, self.max_size, self.on_miss, val_map)) class LRI(dict): """The ``LRI`` implements the basic *Least Recently Inserted* strategy to caching. One could also think of this as a ``SizeLimitedDefaultDict``. *on_miss* is a callable that accepts the missing key (as opposed to :class:`collections.defaultdict`'s "default_factory", which accepts no arguments.) Also note that, like the :class:`LRU`, the ``LRI`` is instrumented with statistics tracking. >>> cap_cache = LRI(max_size=2) >>> cap_cache['a'], cap_cache['b'] = 'A', 'B' >>> from pprint import pprint as pp >>> pp(cap_cache) {'a': 'A', 'b': 'B'} >>> [cap_cache['b'] for i in range(3)][0] 'B' >>> cap_cache['c'] = 'C' >>> print(cap_cache.get('a')) None >>> cap_cache.hit_count, cap_cache.miss_count, cap_cache.soft_miss_count (3, 1, 1) """ # In order to support delitem andn .pop() setitem will need to # popleft until it finds a key still in the cache. or, only # support popitems and raise an error on pop. def __init__(self, max_size=DEFAULT_MAX_SIZE, values=None, on_miss=None): super(LRI, self).__init__() self.hit_count = self.miss_count = self.soft_miss_count = 0 self.max_size = max_size self.on_miss = on_miss self._queue = deque() if values: self.update(values) def __setitem__(self, key, value): # TODO: pop support (see above) if len(self) >= self.max_size: old = self._queue.popleft() del self[old] super(LRI, self).__setitem__(key, value) self._queue.append(key) def update(self, E, **F): # E and F are throwback names to the dict() __doc__ if E is self: return setitem = self.__setitem__ if callable(getattr(E, 'keys', None)): for k in E.keys(): setitem(k, E[k]) else: for k, v in E: setitem(k, v) for k in F: setitem(k, F[k]) return def copy(self): return self.__class__(max_size=self.max_size, values=self) def clear(self): self._queue.clear() super(LRI, self).clear() def __getitem__(self, key): try: ret = super(LRI, self).__getitem__(key) except KeyError: self.miss_count += 1 if not self.on_miss: raise ret = self[key] = self.on_miss(key) return ret self.hit_count += 1 return ret def get(self, key, default=None): try: return self[key] except KeyError: self.soft_miss_count += 1 return default def setdefault(self, key, default=None): try: return self[key] except KeyError: self.soft_miss_count += 1 self[key] = default return default ### Cached decorator # Key-making technique adapted from Python 3.4's functools class _HashedKey(list): """The _HashedKey guarantees that hash() will be called no more than once per cached function invocation. """ __slots__ = 'hash_value' def __init__(self, key): self[:] = key self.hash_value = hash(tuple(key)) def __hash__(self): return self.hash_value def __repr__(self): return '%s(%s)' % (self.__class__.__name__, list.__repr__(self)) def make_cache_key(args, kwargs, typed=False, kwarg_mark=_KWARG_MARK, fasttypes=frozenset([int, str, frozenset, type(None)])): """Make a generic key from a function's positional and keyword arguments, suitable for use in caches. Arguments within *args* and *kwargs* must be `hashable`_. If *typed* is ``True``, ``3`` and ``3.0`` will be treated as separate keys. The key is constructed in a way that is flat as possible rather than as a nested structure that would take more memory. If there is only a single argument and its data type is known to cache its hash value, then that argument is returned without a wrapper. This saves space and improves lookup speed. >>> tuple(make_cache_key(('a', 'b'), {'c': ('d')})) ('a', 'b', _KWARG_MARK, ('c', 'd')) .. _hashable: https://docs.python.org/2/glossary.html#term-hashable """ key = list(args) if kwargs: sorted_items = sorted(kwargs.items()) key.append(kwarg_mark) key.extend(sorted_items) if typed: key.extend([type(v) for v in args]) if kwargs: key.extend([type(v) for k, v in sorted_items]) elif len(key) == 1 and type(key[0]) in fasttypes: return key[0] return _HashedKey(key) # for backwards compatibility in case someone was importing it _make_cache_key = make_cache_key class CachedFunction(object): """This type is used by :func:`cached`, below. Instances of this class are used to wrap functions in caching logic. """ def __init__(self, func, cache, typed=False): self.func = func if callable(cache): self.get_cache = cache elif not (callable(getattr(cache, '__getitem__', None)) and callable(getattr(cache, '__setitem__', None))): raise TypeError('expected cache to be a dict-like object,' ' or callable returning a dict-like object, not %r' % cache) else: def _get_cache(): return cache self.get_cache = _get_cache self.typed = typed def __call__(self, *args, **kwargs): cache = self.get_cache() key = make_cache_key(args, kwargs, typed=self.typed) try: ret = cache[key] except KeyError: ret = cache[key] = self.func(*args, **kwargs) return ret def __repr__(self): cn = self.__class__.__name__ if self.typed: return "%s(func=%r, typed=%r)" % (cn, self.func, self.typed) return "%s(func=%r)" % (cn, self.func) class CachedMethod(object): """Similar to :class:`CachedFunction`, this type is used by :func:`cachedmethod` to wrap methods in caching logic. """ def __init__(self, func, cache, typed=False, selfish=True): self.func = func if isinstance(cache, basestring): self.get_cache = attrgetter(cache) elif callable(cache): self.get_cache = cache elif not (callable(getattr(cache, '__getitem__', None)) and callable(getattr(cache, '__setitem__', None))): raise TypeError('expected cache to be an attribute name,' ' dict-like object, or callable returning' ' a dict-like object, not %r' % cache) else: def _get_cache(obj): return cache self.get_cache = _get_cache self.typed = typed self.selfish = selfish self.bound_to = None def __get__(self, obj, objtype=None): if obj is None: return self cls = self.__class__ ret = cls(self.func, self.get_cache, typed=self.typed, selfish=self.selfish) ret.bound_to = obj return ret def __call__(self, *args, **kwargs): obj = args[0] if self.bound_to is None else self.bound_to cache = self.get_cache(obj) key_args = (self.bound_to,) + args if self.selfish else args key = make_cache_key(key_args, kwargs, typed=self.typed) try: ret = cache[key] except KeyError: if self.bound_to is not None: args = (self.bound_to,) + args ret = cache[key] = self.func(*args, **kwargs) return ret def __repr__(self): cn = self.__class__.__name__ args = (cn, self.func, self.typed, self.selfish) if self.bound_to is not None: args += (self.bound_to,) return ('<%s func=%r typed=%r selfish=%r bound_to=%r>' % args) return ("%s(func=%r, typed=%r, selfish=%r)" % args) def cached(cache, typed=False): """Cache any function with the cache object of your choosing. Note that the function wrapped should take only `hashable`_ arguments. Args: cache (Mapping): Any :class:`dict`-like object suitable for use as a cache. Instances of the :class:`LRU` and :class:`LRI` are good choices, but a plain :class:`dict` can work in some cases, as well. This argument can also be a callable which accepts no arguments and returns a mapping. typed (bool): Whether to factor argument types into the cache check. Default ``False``, setting to ``True`` causes the cache keys for ``3`` and ``3.0`` to be considered unequal. >>> my_cache = LRU() >>> @cached(my_cache) ... def cached_lower(x): ... return x.lower() ... >>> cached_lower("CaChInG's FuN AgAiN!") "caching's fun again!" >>> len(my_cache) 1 .. _hashable: https://docs.python.org/2/glossary.html#term-hashable """ def cached_func_decorator(func): return CachedFunction(func, cache, typed=typed) return cached_func_decorator def cachedmethod(cache, typed=False, selfish=True): """Similar to :func:`cached`, ``cachedmethod`` is used to cache methods based on their arguments, using any :class:`dict`-like *cache* object. Args: cache (str/Mapping/callable): Can be the name of an attribute on the instance, any Mapping/:class:`dict`-like object, or a callable which returns a Mapping. typed (bool): Whether to factor argument types into the cache check. Default ``False``, setting to ``True`` causes the cache keys for ``3`` and ``3.0`` to be considered unequal. selfish (bool): Whether an instance's ``self`` argument should be considered in the cache key. Use this to share cache objects across instances. >>> class Lowerer(object): ... def __init__(self): ... self.cache = LRI() ... ... @cachedmethod('cache') ... def lower(self, text): ... return text.lower() ... >>> lowerer = Lowerer() >>> lowerer.lower('WOW WHO COULD GUESS CACHING COULD BE SO NEAT') 'wow who could guess caching could be so neat' >>> len(lowerer.cache) 1 """ def cached_method_decorator(func): return CachedMethod(func, cache, typed=typed, selfish=selfish) return cached_method_decorator class cachedproperty(object): """The ``cachedproperty`` is used similar to :class:`property`, except that the wrapped method is only called once. This is commonly used to implement lazy attributes. After the property has been accessed, the value is stored on the instance itself, using the same name as the cachedproperty. This allows the cache to be cleared with :func:`delattr`, or through manipulating the object's ``__dict__``. """ def __init__(self, func): self.func = func def __get__(self, obj, objtype=None): if obj is None: return self value = self.func(obj) setattr(obj, self.func.__name__, value) return value def __repr__(self): cn = self.__class__.__name__ return '<%s func=%s>' % (cn, self.func) class ThresholdCounter(object): """A **bounded** dict-like Mapping from keys to counts. The ThresholdCounter automatically compacts after every (1 / *threshold*) additions, maintaining exact counts for any keys whose count represents at least a *threshold* ratio of the total data. In other words, if a particular key is not present in the ThresholdCounter, its count represents less than *threshold* of the total data. >>> tc = ThresholdCounter(threshold=0.1) >>> tc.add(1) >>> tc.items() [(1, 1)] >>> tc.update([2] * 10) >>> tc.get(1) 0 >>> tc.add(5) >>> 5 in tc True >>> len(list(tc.elements())) 11 As you can see above, the API is kept similar to :class:`collections.Counter`. The most notable feature omissions being that counted items cannot be set directly, uncounted, or removed, as this would disrupt the math. Use the ThresholdCounter when you need best-effort long-lived counts for dynamically-keyed data. Without a bounded datastructure such as this one, the dynamic keys often represent a memory leak and can impact application reliability. The ThresholdCounter's item replacement strategy is fully deterministic and can be thought of as *Amortized Least Relevant*. The absolute upper bound of keys it will store is *(2/threshold)*, but realistically *(1/threshold)* is expected for uniformly random datastreams, and one or two orders of magnitude better for real-world data. This algorithm is an implementation of the Lossy Counting algorithm described in "Approximate Frequency Counts over Data Streams" by Manku & Motwani. Hat tip to Kurt Rose for discovery and initial implementation. """ # TODO: hit_count/miss_count? def __init__(self, threshold=0.001): if not 0 < threshold < 1: raise ValueError('expected threshold between 0 and 1, not: %r' % threshold) self.total = 0 self._count_map = {} self._threshold = threshold self._thresh_count = int(1 / threshold) self._cur_bucket = 1 @property def threshold(self): return self._threshold def add(self, key): """Increment the count of *key* by 1, automatically adding it if it does not exist. Cache compaction is triggered every *1/threshold* additions. """ self.total += 1 try: self._count_map[key][0] += 1 except KeyError: self._count_map[key] = [1, self._cur_bucket - 1] if self.total % self._thresh_count == 0: self._count_map = dict([(k, v) for k, v in self._count_map.items() if sum(v) > self._cur_bucket]) self._cur_bucket += 1 return def elements(self): """Return an iterator of all the common elements tracked by the counter. Yields each key as many times as it has been seen. """ repeaters = itertools.starmap(itertools.repeat, self.iteritems()) return itertools.chain.from_iterable(repeaters) def most_common(self, n=None): """Get the top *n* keys and counts as tuples. If *n* is omitted, returns all the pairs. """ if n <= 0: return [] ret = sorted(self.iteritems(), key=lambda x: x[1][0], reverse=True) if n is None or n >= len(ret): return ret return ret[:n] def get_common_count(self): """Get the sum of counts for keys exceeding the configured data threshold. """ return sum([count for count, _ in self._count_map.itervalues()]) def get_uncommon_count(self): """Get the sum of counts for keys that were culled because the associated counts represented less than the configured threshold. The long-tail counts. """ return self.total - self.get_common_count() def get_commonality(self): """Get a float representation of the effective count accuracy. The higher the number, the less uniform the keys being added, and the higher accuracy and efficiency of the ThresholdCounter. If a stronger measure of data cardinality is required, consider using hyperloglog. """ return float(self.get_common_count()) / self.total def __getitem__(self, key): return self._count_map[key][0] def __len__(self): return len(self._count_map) def __contains__(self, key): return key in self._count_map def iterkeys(self): return iter(self._count_map) def keys(self): return list(self.iterkeys()) def itervalues(self): count_map = self._count_map for k in count_map: yield count_map[k][0] def values(self): return list(self.itervalues()) def iteritems(self): count_map = self._count_map for k in count_map: yield (k, count_map[k][0]) def items(self): return list(self.iteritems()) def get(self, key, default=0): "Get count for *key*, defaulting to 0." try: return self[key] except KeyError: return default def update(self, iterable, **kwargs): """Like dict.update() but add counts instead of replacing them, used to add multiple items in one call. Source can be an iterable of keys to add, or a mapping of keys to integer counts. """ if iterable is not None: if callable(getattr(iterable, 'iteritems', None)): for key, count in iterable.iteritems(): for i in xrange(count): self.add(key) else: for key in iterable: self.add(key) if kwargs: self.update(kwargs) # end cacheutils.py boltons-16.4.1/boltons/__init__.py0000644000175000017500000000000012727357214015156 0ustar hlehleboltons-16.4.1/boltons/jsonutils.py0000644000175000017500000001763012727357214015472 0ustar hlehle# -*- coding: utf-8 -*- """``jsonutils`` aims to provide various helpers for working with JSON. Currently it focuses on providing a reliable and intuitive means of working with `JSON Lines`_-formatted files. .. _JSON Lines: http://jsonlines.org/ """ from __future__ import print_function import os import json DEFAULT_BLOCKSIZE = 4096 # reverse iter lines algorithm: # # - if it ends in a newline, add an empty string to the line list # - if there's one item, then prepend it to the buffer, continue # - if there's more than one item, pop the last item and prepend it # to the buffer, yielding it # - yield all remaining items in reverse, except for the first # - first item becomes the new buffer # # - when the outer loop completes, yield the buffer __all__ = ['JSONLIterator', 'reverse_iter_lines'] def reverse_iter_lines(file_obj, blocksize=DEFAULT_BLOCKSIZE, preseek=True): """Returns an iterator over the lines from a file object, in reverse order, i.e., last line first, first line last. Uses the :meth:`file.seek` method of file objects, and is tested compatible with :class:`file` objects, as well as :class:`StringIO.StringIO`. Args: file_obj (file): An open file object. Note that ``reverse_iter_lines`` mutably reads from the file and other functions should not mutably interact with the file object. blocksize (int): The block size to pass to :meth:`file.read()` preseek (bool): Tells the function whether or not to automatically seek to the end of the file. Defaults to ``True``. ``preseek=False`` is useful in cases when the file cursor is already in position, either at the end of the file or in the middle for relative reverse line generation. """ if preseek: file_obj.seek(0, os.SEEK_END) cur_pos = file_obj.tell() buff = '' while 0 < cur_pos: read_size = min(blocksize, cur_pos) cur_pos -= read_size file_obj.seek(cur_pos, os.SEEK_SET) cur = file_obj.read(read_size) lines = cur.splitlines() if cur[-1] == '\n': lines.append('') if len(lines) == 1: buff = lines[0] + buff continue last = lines.pop() yield last + buff for line in lines[:0:-1]: yield line buff = lines[0] if buff: # TODO: test this, does an empty buffer always mean don't yield? yield buff """ TODO: allow passthroughs for: json.load(fp[, encoding[, cls[, object_hook[, parse_float[, parse_int[, parse_constant[, object_pairs_hook[, **kw]]]]]]]]) """ class JSONLIterator(object): """The ``JSONLIterator`` is used to iterate over JSON-encoded objects stored in the `JSON Lines format`_ (one object per line). Most notably it has the ability to efficiently read from the bottom of files, making it very effective for reading in simple append-only JSONL use cases. It also has the ability to start from anywhere in the file and ignore corrupted lines. Args: file_obj (file): An open file object. ignore_errors (bool): Whether to skip over lines that raise an error on deserialization (:func:`json.loads`). reverse (bool): Controls the direction of the iteration. Defaults to ``False``. If set to ``True`` and *rel_seek* is unset, seeks to the end of the file before iteration begins. rel_seek (float): Used to preseek the start position of iteration. Set to 0.0 for the start of the file, 1.0 for the end, and anything in between. .. _JSON Lines format: http://jsonlines.org/ """ def __init__(self, file_obj, ignore_errors=False, reverse=False, rel_seek=None): self._reverse = bool(reverse) self._file_obj = file_obj self.ignore_errors = ignore_errors if rel_seek is None: if reverse: rel_seek = 1.0 elif not -1.0 < rel_seek < 1.0: raise ValueError("'rel_seek' expected a float between" " -1.0 and 1.0, not %r" % rel_seek) elif rel_seek < 0: rel_seek = 1.0 - rel_seek self._rel_seek = rel_seek self._blocksize = 4096 if rel_seek is not None: self._init_rel_seek() if self._reverse: self._line_iter = reverse_iter_lines(self._file_obj, blocksize=self._blocksize, preseek=False) else: self._line_iter = iter(self._file_obj) @property def cur_byte_pos(self): "A property representing where in the file the iterator is reading." return self._file_obj.tell() def _align_to_newline(self): "Aligns the file object's position to the next newline." fo, bsize = self._file_obj, self._blocksize cur, total_read = '', 0 cur_pos = fo.tell() while '\n' not in cur: cur = fo.read(bsize) total_read += bsize try: newline_offset = cur.index('\n') + total_read - bsize except ValueError: raise # TODO: seek to end? fo.seek(cur_pos + newline_offset) def _init_rel_seek(self): "Sets the file object's position to the relative location set above." rs, fo = self._rel_seek, self._file_obj if rs == 0.0: fo.seek(0, os.SEEK_SET) else: fo.seek(0, os.SEEK_END) size = fo.tell() if rs == 1.0: self._cur_pos = size else: target = int(size * rs) fo.seek(target, os.SEEK_SET) self._align_to_newline() self._cur_pos = fo.tell() def __iter__(self): return self def next(self): """Yields one :class:`dict` loaded with :func:`json.loads`, advancing the file object by one line. Raises :exc:`StopIteration` upon reaching the end of the file (or beginning, if ``reverse`` was set to ``True``. """ while 1: line = next(self._line_iter).lstrip() if not line: continue try: obj = json.loads(line) except Exception: if not self.ignore_errors: raise continue return obj __next__ = next if __name__ == '__main__': def _main(): import sys if '-h' in sys.argv or '--help' in sys.argv: print('loads one or more JSON Line files for basic validation.') return verbose = False if '-v' in sys.argv or '--verbose' in sys.argv: verbose = True file_count, obj_count = 0, 0 filenames = sys.argv[1:] for filename in filenames: if filename in ('-h', '--help', '-v', '--verbose'): continue file_count += 1 with open(filename, 'rb') as file_obj: iterator = JSONLIterator(file_obj) cur_obj_count = 0 while 1: try: next(iterator) except ValueError: print('error reading object #%s around byte %s in %s' % (cur_obj_count + 1, iterator.cur_byte_pos, filename)) return except StopIteration: break obj_count += 1 cur_obj_count += 1 if verbose and obj_count and obj_count % 100 == 0: sys.stdout.write('.') if obj_count % 10000: sys.stdout.write('%s\n' % obj_count) if verbose: print('files checked: %s' % file_count) print('objects loaded: %s' % obj_count) return _main() boltons-16.4.1/boltons/funcutils.py0000644000175000017500000004661012727357214015454 0ustar hlehle# -*- coding: utf-8 -*- """Python's built-in :mod:`functools` module builds several useful utilities on top of Python's first-class function support. ``funcutils`` generally stays in the same vein, adding to and correcting Python's standard metaprogramming facilities. """ from __future__ import print_function import sys import re import inspect import functools import itertools from types import MethodType, FunctionType try: xrange make_method = MethodType except NameError: # Python 3 make_method = lambda desc, obj, obj_type: MethodType(desc, obj) basestring = (str, bytes) # Python 3 compat _IS_PY2 = False else: _IS_PY2 = True def get_module_callables(mod, ignore=None): """Returns two maps of (*types*, *funcs*) from *mod*, optionally ignoring based on the :class:`bool` return value of the *ignore* callable. *mod* can be a string name of a module in :data:`sys.modules` or the module instance itself. """ if isinstance(mod, basestring): mod = sys.modules[mod] types, funcs = {}, {} for attr_name in dir(mod): if ignore and ignore(attr_name): continue try: attr = getattr(mod, attr_name) except Exception: continue try: attr_mod_name = attr.__module__ except AttributeError: continue if attr_mod_name != mod.__name__: continue if isinstance(attr, type): types[attr_name] = attr elif callable(attr): funcs[attr_name] = attr return types, funcs def mro_items(type_obj): """Takes a type and returns an iterator over all class variables throughout the type hierarchy (respecting the MRO). >>> sorted(set([k for k, v in mro_items(int) if not k.startswith('__') and not callable(v)])) ['denominator', 'imag', 'numerator', 'real'] """ # TODO: handle slots? return itertools.chain.from_iterable(ct.__dict__.items() for ct in type_obj.__mro__) def dir_dict(obj, raise_exc=False): """Return a dictionary of attribute names to values for a given object. Unlike ``obj.__dict__``, this function returns all attributes on the object, including ones on parent classes. """ # TODO: separate function for handling descriptors on types? ret = {} for k in dir(obj): try: ret[k] = getattr(obj, k) except Exception: if raise_exc: raise return ret def copy_function(orig, copy_dict=True): """Returns a shallow copy of the function, including code object, globals, closure, etc. >>> func = lambda: func >>> func() is func True >>> func_copy = copy_function(func) >>> func_copy() is func True >>> func_copy is not func True Args: orig (function): The function to be copied. Must be a function, not just any method or callable. copy_dict (bool): Also copy any attributes set on the function instance. Defaults to ``True``. """ ret = FunctionType(orig.__code__, orig.__globals__, name=orig.__name__, argdefs=getattr(orig, "__defaults__", None), closure=getattr(orig, "__closure__", None)) if copy_dict: ret.__dict__.update(orig.__dict__) return ret def partial_ordering(cls): """Class decorator, similar to :func:`functools.total_ordering`, except it is used to define `partial orderings`_ (i.e., it is possible that *x* is neither greater than, equal to, or less than *y*). It assumes the presence of the ``__le__()`` and ``__ge__()`` method, but nothing else. It will not override any existing additional comparison methods. .. _partial orderings: https://en.wikipedia.org/wiki/Partially_ordered_set >>> @partial_ordering ... class MySet(set): ... def __le__(self, other): ... return self.issubset(other) ... def __ge__(self, other): ... return self.issuperset(other) ... >>> a = MySet([1,2,3]) >>> b = MySet([1,2]) >>> c = MySet([1,2,4]) >>> b < a True >>> b > a False >>> b < c True >>> a < c False >>> c > a False """ def __lt__(self, other): return self <= other and not self >= other def __gt__(self, other): return self >= other and not self <= other def __eq__(self, other): return self >= other and self <= other if not hasattr(cls, '__lt__'): cls.__lt__ = __lt__ if not hasattr(cls, '__gt__'): cls.__gt__ = __gt__ if not hasattr(cls, '__eq__'): cls.__eq__ = __eq__ return cls class InstancePartial(functools.partial): """:class:`functools.partial` is a huge convenience for anyone working with Python's great first-class functions. It allows developers to curry arguments and incrementally create simpler callables for a variety of use cases. Unfortunately there's one big gap in its usefulness: methods. Partials just don't get bound as methods and automatically handed a reference to ``self``. The ``InstancePartial`` type remedies this by inheriting from :class:`functools.partial` and implementing the necessary descriptor protocol. There are no other differences in implementation or usage. :class:`CachedInstancePartial`, below, has the same ability, but is slightly more efficient. """ def __get__(self, obj, obj_type): return make_method(self, obj, obj_type) class CachedInstancePartial(functools.partial): """The ``CachedInstancePartial`` is virtually the same as :class:`InstancePartial`, adding support for method-usage to :class:`functools.partial`, except that upon first access, it caches the bound method on the associated object, speeding it up for future accesses, and bringing the method call overhead to about the same as non-``partial`` methods. See the :class:`InstancePartial` docstring for more details. """ def __get__(self, obj, obj_type): # These assignments could've been in __init__, but there was # no simple way to do it without breaking one of PyPy or Py3. self.__name__ = None self.__doc__ = self.func.__doc__ self.__module__ = self.func.__module__ name = self.__name__ if name is None: for k, v in mro_items(obj_type): if v is self: self.__name__ = name = k if obj is None: return make_method(self, obj, obj_type) try: # since this is a data descriptor, this block # is probably only hit once (per object) return obj.__dict__[name] except KeyError: obj.__dict__[name] = ret = make_method(self, obj, obj_type) return ret partial = CachedInstancePartial # # # # # # Function builder # # # def wraps(func, injected=None, **kw): """Modeled after the built-in :func:`functools.wraps`, this function is used to make your decorator's wrapper functions reflect the wrapped function's: * Name * Documentation * Module * Signature The built-in :func:`functools.wraps` copies the first three, but does not copy the signature. This version of ``wraps`` can copy the inner function's signature exactly, allowing seamless usage and :mod:`introspection `. Usage is identical to the built-in version:: >>> from boltons.funcutils import wraps >>> >>> def print_return(func): ... @wraps(func) ... def wrapper(*args, **kwargs): ... ret = func(*args, **kwargs) ... print(ret) ... return ret ... return wrapper ... >>> @print_return ... def example(): ... '''docstring''' ... return 'example return value' >>> >>> val = example() example return value >>> example.__name__ 'example' >>> example.__doc__ 'docstring' In addition, the boltons version of wraps supports modifying the outer signature based on the inner signature. By passing a list of *injected* argument names, those arguments will be removed from the outer wrapper's signature, allowing your decorator to provide arguments that aren't passed in. Args: func (function): The callable whose attributes are to be copied. injected (list): An optional list of argument names which should not appear in the new wrapper's signature. update_dict(bool): Whether to copy other, non-standard attributes of *func* over to the wrapper. Defaults to True. For more in-depth wrapping of functions, see the :class:`FunctionBuilder` type, on which wraps was built. """ if injected is None: injected = [] elif isinstance(injected, basestring): injected = [injected] update_dict = kw.pop('update_dict', True) if kw: raise TypeError('unexpected kwargs: %r' % kw.keys()) fb = FunctionBuilder.from_func(func) for arg in injected: fb.remove_arg(arg) fb.body = 'return _call(%s)' % fb.get_invocation_str() def wrapper_wrapper(wrapper_func): execdict = dict(_call=wrapper_func, _func=func) fully_wrapped = fb.get_func(execdict, with_dict=update_dict) return fully_wrapped return wrapper_wrapper class FunctionBuilder(object): """The FunctionBuilder type provides an interface for programmatically creating new functions, either based on existing functions or from scratch. Values are passed in at construction or set as attributes on the instance. For creating a new function based of an existing one, see the :meth:`~FunctionBuilder.from_func` classmethod. At any point, :meth:`~FunctionBuilder.get_func` can be called to get a newly compiled function, based on the values configured. >>> fb = FunctionBuilder('return_five', doc='returns the integer 5', ... body='return 5') >>> f = fb.get_func() >>> f() 5 >>> fb.varkw = 'kw' >>> f_kw = fb.get_func() >>> f_kw(ignored_arg='ignored_val') 5 Note that function signatures themselves changed quite a bit in Python 3, so several arguments are only applicable to FunctionBuilder in Python 3. Except for *name*, all arguments to the constructor are keyword arguments. Args: name (str): Name of the function. doc (str): `Docstring`_ for the function, defaults to empty. module (str): Name of the module from which this function was imported. Defaults to None. body (str): String version of the code representing the body of the function. Defaults to ``'pass'``, which will result in a function which does nothing and returns ``None``. args (list): List of argument names, defaults to empty list, denoting no arguments. varargs (str): Name of the catch-all variable for positional arguments. E.g., "args" if the resultant function is to have ``*args`` in the signature. Defaults to None. varkw (str): Name of the catch-all variable for keyword arguments. E.g., "kwargs" if the resultant function is to have ``**kwargs`` in the signature. Defaults to None. defaults (dict): A mapping of argument names to default values. kwonlyargs (list): Argument names which are only valid as keyword arguments. **Python 3 only.** kwonlydefaults (dict): A mapping, same as normal *defaults*, but only for the *kwonlyargs*. **Python 3 only.** annotations (dict): Mapping of type hints and so forth. **Python 3 only.** indent (int): Number of spaces with which to indent the function *body*. Values less than 1 will result in an error. dict (dict): Any other attributes which should be added to the functions compiled with this FunctionBuilder. All of these arguments are also made available as attributes which can be mutated as necessary. .. _Docstring: https://en.wikipedia.org/wiki/Docstring#Python """ if _IS_PY2: _argspec_defaults = {'args': list, 'varargs': lambda: None, 'varkw': lambda: None, 'defaults': lambda: None} @classmethod def _argspec_to_dict(cls, f): args, varargs, varkw, defaults = inspect.getargspec(f) return {'args': args, 'varargs': varargs, 'varkw': varkw, 'defaults': defaults} else: _argspec_defaults = {'args': list, 'varargs': lambda: None, 'varkw': lambda: None, 'defaults': lambda: None, 'kwonlyargs': list, 'kwonlydefaults': dict, 'annotations': dict} @classmethod def _argspec_to_dict(cls, f): argspec = inspect.getfullargspec(f) return dict((attr, getattr(argspec, attr)) for attr in cls._argspec_defaults) _defaults = {'doc': str, 'dict': dict, 'module': lambda: None, 'body': lambda: 'pass', 'indent': lambda: 4} _defaults.update(_argspec_defaults) _compile_count = itertools.count() def __init__(self, name, **kw): self.name = name for a, default_factory in self._defaults.items(): val = kw.pop(a, None) if val is None: val = default_factory() setattr(self, a, val) if kw: raise TypeError('unexpected kwargs: %r' % kw.keys()) return # def get_argspec(self): # TODO if _IS_PY2: def get_sig_str(self): return inspect.formatargspec(self.args, self.varargs, self.varkw, []) def get_invocation_str(self): return inspect.formatargspec(self.args, self.varargs, self.varkw, [])[1:-1] else: def get_sig_str(self): return inspect.formatargspec(self.args, self.varargs, self.varkw, [], self.kwonlyargs, {}, self.annotations) _KWONLY_MARKER = re.compile(r""" \* # a star \s* # followed by any amount of whitespace , # followed by a comma \s* # followed by any amount of whitespace """, re.VERBOSE) def get_invocation_str(self): kwonly_pairs = None formatters = {} if self.kwonlyargs: kwonly_pairs = dict((arg, arg) for arg in self.kwonlyargs) formatters['formatvalue'] = lambda value: '=' + value sig = inspect.formatargspec(self.args, self.varargs, self.varkw, [], kwonly_pairs, kwonly_pairs, {}, **formatters) sig = self._KWONLY_MARKER.sub('', sig) return sig[1:-1] @classmethod def from_func(cls, func): """Create a new FunctionBuilder instance based on an existing function. The original function will not be stored or modified. """ # TODO: copy_body? gonna need a good signature regex. # TODO: might worry about __closure__? kwargs = {'name': func.__name__, 'doc': func.__doc__, 'module': func.__module__, 'dict': getattr(func, '__dict__', {})} kwargs.update(cls._argspec_to_dict(func)) return cls(**kwargs) def get_func(self, execdict=None, add_source=True, with_dict=True): """Compile and return a new function based on the current values of the FunctionBuilder. Args: execdict (dict): The dictionary representing the scope in which the compilation should take place. Defaults to an empty dict. add_source (bool): Whether to add the source used to a special ``__source__`` attribute on the resulting function. Defaults to True. with_dict (bool): Add any custom attributes, if applicable. Defaults to True. To see an example of usage, see the implementation of :func:`~boltons.funcutils.wraps`. """ execdict = execdict or {} body = self.body or self._default_body tmpl = 'def {name}{sig_str}:' if self.doc: tmpl += '\n """{doc}"""' tmpl += '\n{body}' body = _indent(self.body, ' ' * self.indent) name = self.name.replace('<', '_').replace('>', '_') # lambdas src = tmpl.format(name=name, sig_str=self.get_sig_str(), doc=self.doc, body=body) self._compile(src, execdict) func = execdict[name] func.__name__ = self.name func.__doc__ = self.doc func.__defaults__ = self.defaults if not _IS_PY2: func.__kwdefaults__ = self.kwonlydefaults if with_dict: func.__dict__.update(self.dict) func.__module__ = self.module # TODO: caller module fallback? if add_source: func.__source__ = src return func def get_defaults_dict(self): """Get a dictionary of function arguments with defaults and the respective values. """ ret = dict(reversed(list(zip(reversed(self.args), reversed(self.defaults or []))))) return ret def remove_arg(self, arg_name): """Remove an argument from this FunctionBuilder's argument list. The resulting function will have one less argument per call to this function. Args: arg_name (str): The name of the argument to remove. Raises a :exc:`ValueError` if the argument is not present. """ d_dict = self.get_defaults_dict() try: self.args.remove(arg_name) except ValueError: raise ValueError('arg %r not found in %s argument list: %r' % (arg_name, self.name, self.args)) d_dict.pop(arg_name, None) self.defaults = tuple([d_dict[a] for a in self.args if a in d_dict]) return def _compile(self, src, execdict): filename = ('' % (next(self._compile_count),)) try: code = compile(src, filename, 'single') exec(code, execdict) except Exception: raise return execdict def _indent(text, margin, newline='\n', key=bool): "based on boltons.strutils.indent" indented_lines = [(margin + line if key(line) else line) for line in text.splitlines()] return newline.join(indented_lines) # end funcutils.py boltons-16.4.1/boltons/mboxutils.py0000644000175000017500000001054112727357214015460 0ustar hlehle# -*- coding: utf-8 -*- """Useful utilities for working with the `mbox`_-formatted mailboxes. Credit to Mark Williams for these. .. _mbox: https://en.wikipedia.org/wiki/Mbox """ import mailbox import tempfile DEFAULT_MAXMEM = 4 * 1024 * 1024 # 4MB class mbox_readonlydir(mailbox.mbox): """A subclass of :class:`mailbox.mbox` suitable for use with mboxs insides a read-only mail directory, e.g., ``/var/mail``. Otherwise the API is exactly the same as the built-in mbox. Deletes messages via truncation, in the manner of `Heirloom mailx`_. Args: path (str): Path to the mbox file. factory (type): Message type (defaults to :class:`rfc822.Message`) create (bool): Create mailbox if it does not exist. (defaults to ``True``) maxmem (int): Specifies, in bytes, the largest sized mailbox to attempt to copy into memory. Larger mailboxes will be copied incrementally which is more hazardous. (defaults to 4MB) .. note:: Because this truncates and rewrites parts of the mbox file, this class can corrupt your mailbox. Only use this if you know the built-in :class:`mailbox.mbox` does not work for your use case. .. _Heirloom mailx: http://heirloom.sourceforge.net/mailx.html """ def __init__(self, path, factory=None, create=True, maxmem=1024 * 1024): mailbox.mbox.__init__(self, path, factory, create) self.maxmem = maxmem def flush(self): """Write any pending changes to disk. This is called on mailbox close and is usually not called explicitly. .. note:: This deletes messages via truncation. Interruptions may corrupt your mailbox. """ # Appending and basic assertions are the same as in mailbox.mbox.flush. if not self._pending: if self._pending_sync: # Messages have only been added, so syncing the file # is enough. mailbox._sync_flush(self._file) self._pending_sync = False return # In order to be writing anything out at all, self._toc must # already have been generated (and presumably has been modified # by adding or deleting an item). assert self._toc is not None # Check length of self._file; if it's changed, some other process # has modified the mailbox since we scanned it. self._file.seek(0, 2) cur_len = self._file.tell() if cur_len != self._file_length: raise mailbox.ExternalClashError('Size of mailbox file changed ' '(expected %i, found %i)' % (self._file_length, cur_len)) self._file.seek(0) # Truncation logic begins here. Mostly the same except we # can use tempfile because we're not doing rename(2). with tempfile.TemporaryFile() as new_file: new_toc = {} self._pre_mailbox_hook(new_file) for key in sorted(self._toc.keys()): start, stop = self._toc[key] self._file.seek(start) self._pre_message_hook(new_file) new_start = new_file.tell() while True: buffer = self._file.read(min(4096, stop - self._file.tell())) if buffer == '': break new_file.write(buffer) new_toc[key] = (new_start, new_file.tell()) self._post_message_hook(new_file) self._file_length = new_file.tell() self._file.seek(0) new_file.seek(0) # Copy back our messages if self._file_length <= self.maxmem: self._file.write(new_file.read()) else: while True: buffer = new_file.read(4096) if not buffer: break self._file.write(buffer) # Delete the rest. self._file.truncate() # Same wrap up. self._toc = new_toc self._pending = False self._pending_sync = False if self._locked: mailbox._lock_file(self._file, dotlock=False) boltons-16.4.1/boltons/strutils.py0000644000175000017500000010030312727357214015317 0ustar hlehle# -*- coding: utf-8 -*- """So much practical programming involves string manipulation, which Python readily accomodates. Still, there are dozens of basic and common capabilities missing from the standard library, several of them provided by ``strutils``. """ from __future__ import print_function import re import sys import uuid import zlib import string import unicodedata import collections try: unicode, str, bytes, basestring = unicode, str, str, basestring from HTMLParser import HTMLParser import htmlentitydefs except NameError: # basestring not defined in Python 3 unicode, str, bytes, basestring = str, bytes, bytes, (str, bytes) unichr = chr from html.parser import HTMLParser from html import entities as htmlentitydefs __all__ = ['camel2under', 'under2camel', 'slugify', 'split_punct_ws', 'unit_len', 'ordinalize', 'cardinalize', 'pluralize', 'singularize', 'asciify', 'is_ascii', 'is_uuid', 'html2text', 'strip_ansi', 'bytes2human', 'find_hashtags', 'a10n', 'gunzip_bytes', 'iter_splitlines', 'indent', 'escape_shell_args', 'args2cmd', 'args2sh', 'parse_int_list', 'format_int_list'] _punct_ws_str = string.punctuation + string.whitespace _punct_re = re.compile('[' + _punct_ws_str + ']+') _camel2under_re = re.compile('((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))') def camel2under(camel_string): """Converts a camelcased string to underscores. Useful for turning a class name into a function name. >>> camel2under('BasicParseTest') 'basic_parse_test' """ return _camel2under_re.sub(r'_\1', camel_string).lower() def under2camel(under_string): """Converts an underscored string to camelcased. Useful for turning a function name into a class name. >>> under2camel('complex_tokenizer') 'ComplexTokenizer' """ return ''.join(w.capitalize() or '_' for w in under_string.split('_')) def slugify(text, delim='_', lower=True, ascii=False): """ A basic function that turns text full of scary characters (i.e., punctuation and whitespace), into a relatively safe lowercased string separated only by the delimiter specified by *delim*, which defaults to ``_``. The *ascii* convenience flag will :func:`asciify` the slug if you require ascii-only slugs. >>> slugify('First post! Hi!!!!~1 ') 'first_post_hi_1' >>> slugify("Kurt Gödel's pretty cool.", ascii=True) == \ b'kurt_goedel_s_pretty_cool' True """ ret = delim.join(split_punct_ws(text)) if ascii: ret = asciify(ret) if lower: ret = ret.lower() return ret def split_punct_ws(text): """While :meth:`str.split` will split on whitespace, :func:`split_punct_ws` will split on punctuation and whitespace. This used internally by :func:`slugify`, above. >>> split_punct_ws('First post! Hi!!!!~1 ') ['First', 'post', 'Hi', '1'] """ return [w for w in _punct_re.split(text) if w] def unit_len(sized_iterable, unit_noun='item'): # TODO: len_units()/unitize()? """Returns a plain-English description of an iterable's :func:`len()`, conditionally pluralized with :func:`cardinalize`, detailed below. >>> print(unit_len(range(10), 'number')) 10 numbers >>> print(unit_len('aeiou', 'vowel')) 5 vowels >>> print(unit_len([], 'worry')) No worries """ count = len(sized_iterable) units = cardinalize(unit_noun, count) if count: return u'%s %s' % (count, units) return u'No %s' % (units,) _ORDINAL_MAP = {'1': 'st', '2': 'nd', '3': 'rd'} # 'th' is the default def ordinalize(number, ext_only=False): """Turns *number* into its cardinal form, i.e., 1st, 2nd, 3rd, 4th, etc. If the last character isn't a digit, it returns the string value unchanged. Args: number (int or str): Number to be cardinalized. ext_only (bool): Whether to return only the suffix. Default ``False``. >>> print(ordinalize(1)) 1st >>> print(ordinalize(3694839230)) 3694839230th >>> print(ordinalize('hi')) hi >>> print(ordinalize(1515)) 1515th """ numstr, ext = unicode(number), '' if numstr and numstr[-1] in string.digits: try: # first check for teens if numstr[-2] == '1': ext = 'th' else: # all other cases ext = _ORDINAL_MAP.get(numstr[-1], 'th') except IndexError: # single digit numbers (will reach here based on [-2] above) ext = _ORDINAL_MAP.get(numstr[-1], 'th') if ext_only: return ext else: return numstr + ext def cardinalize(unit_noun, count): """Conditionally pluralizes a singular word *unit_noun* if *count* is not one, preserving case when possible. >>> vowels = 'aeiou' >>> print(len(vowels), cardinalize('vowel', len(vowels))) 5 vowels >>> print(3, cardinalize('Wish', 3)) 3 Wishes """ if count == 1: return unit_noun return pluralize(unit_noun) def singularize(word): """Semi-intelligently converts an English plural *word* to its singular form, preserving case pattern. >>> singularize('records') 'record' >>> singularize('FEET') 'FOOT' """ orig_word, word = word, word.strip().lower() if not word or word in _IRR_S2P: return orig_word irr_singular = _IRR_P2S.get(word) if irr_singular: singular = irr_singular elif not word.endswith('s'): return orig_word elif len(word) == 2: singular = word[:-1] # or just return word? elif word.endswith('ies') and word[-5:-4] not in 'aeiou': singular = word[:-3] + 'y' elif word.endswith('es'): singular = word[:-2] else: singular = word[:-1] return _match_case(orig_word, singular) def pluralize(word): """Semi-intelligently converts an English *word* from singular form to plural, preserving case pattern. >>> pluralize('friend') 'friends' >>> pluralize('enemy') 'enemies' >>> pluralize('Sheep') 'Sheep' """ orig_word, word = word, word.strip().lower() if not word or word in _IRR_P2S: return orig_word irr_plural = _IRR_S2P.get(word) if irr_plural: plural = irr_plural elif word.endswith('y') and word[-2:-1] not in 'aeiou': plural = word[:-1] + 'ies' elif word[-1] == 's' or word.endswith('ch') or word.endswith('sh'): plural = word if word.endswith('es') else word + 'es' else: plural = word + 's' return _match_case(orig_word, plural) def _match_case(master, disciple): if not master.strip(): return disciple if master.lower() == master: return disciple.lower() elif master.upper() == master: return disciple.upper() elif master.capitalize() == master: return disciple.capitalize() return disciple # Singular to plural map of irregular pluralizations _IRR_S2P = {'addendum': 'addenda', 'alga': 'algae', 'alumna': 'alumnae', 'alumnus': 'alumni', 'analysis': 'analyses', 'antenna': 'antennae', 'appendix': 'appendices', 'axis': 'axes', 'bacillus': 'bacilli', 'bacterium': 'bacteria', 'basis': 'bases', 'beau': 'beaux', 'bison': 'bison', 'bureau': 'bureaus', 'cactus': 'cacti', 'calf': 'calves', 'child': 'children', 'corps': 'corps', 'corpus': 'corpora', 'crisis': 'crises', 'criterion': 'criteria', 'curriculum': 'curricula', 'datum': 'data', 'deer': 'deer', 'diagnosis': 'diagnoses', 'die': 'dice', 'dwarf': 'dwarves', 'echo': 'echoes', 'elf': 'elves', 'ellipsis': 'ellipses', 'embargo': 'embargoes', 'emphasis': 'emphases', 'erratum': 'errata', 'fireman': 'firemen', 'fish': 'fish', 'focus': 'foci', 'foot': 'feet', 'formula': 'formulae', 'formula': 'formulas', 'fungus': 'fungi', 'genus': 'genera', 'goose': 'geese', 'half': 'halves', 'hero': 'heroes', 'hippopotamus': 'hippopotami', 'hoof': 'hooves', 'hypothesis': 'hypotheses', 'index': 'indices', 'knife': 'knives', 'leaf': 'leaves', 'life': 'lives', 'loaf': 'loaves', 'louse': 'lice', 'man': 'men', 'matrix': 'matrices', 'means': 'means', 'medium': 'media', 'memorandum': 'memoranda', 'millennium': 'milennia', 'moose': 'moose', 'mosquito': 'mosquitoes', 'mouse': 'mice', 'nebula': 'nebulae', 'neurosis': 'neuroses', 'nucleus': 'nuclei', 'oasis': 'oases', 'octopus': 'octopi', 'offspring': 'offspring', 'ovum': 'ova', 'ox': 'oxen', 'paralysis': 'paralyses', 'parenthesis': 'parentheses', 'person': 'people', 'phenomenon': 'phenomena', 'potato': 'potatoes', 'radius': 'radii', 'scarf': 'scarves', 'scissors': 'scissors', 'self': 'selves', 'series': 'series', 'sheep': 'sheep', 'shelf': 'shelves', 'species': 'species', 'stimulus': 'stimuli', 'stratum': 'strata', 'syllabus': 'syllabi', 'symposium': 'symposia', 'synopsis': 'synopses', 'synthesis': 'syntheses', 'tableau': 'tableaux', 'that': 'those', 'thesis': 'theses', 'thief': 'thieves', 'this': 'these', 'tomato': 'tomatoes', 'tooth': 'teeth', 'torpedo': 'torpedoes', 'vertebra': 'vertebrae', 'veto': 'vetoes', 'vita': 'vitae', 'watch': 'watches', 'wife': 'wives', 'wolf': 'wolves', 'woman': 'women'} # Reverse index of the above _IRR_P2S = dict([(v, k) for k, v in _IRR_S2P.items()]) HASHTAG_RE = re.compile(r"(?:^|\s)[＃#]{1}(\w+)", re.UNICODE) def find_hashtags(string): """Finds and returns all hashtags in a string, with the hashmark removed. Supports full-width hashmarks for Asian languages and does not false-positive on URL anchors. >>> find_hashtags('#atag http://asite/#ananchor') ['atag'] ``find_hashtags`` also works with unicode hashtags. """ # the following works, doctest just struggles with it # >>> find_hashtags(u"can't get enough of that dignity chicken #肯德基 woo") # [u'\u80af\u5fb7\u57fa'] return HASHTAG_RE.findall(string) def a10n(string): """That thing where "internationalization" becomes "i18n", what's it called? Abbreviation? Oh wait, no: ``a10n``. (It's actually a form of `numeronym`_.) >>> a10n('abbreviation') 'a10n' >>> a10n('internationalization') 'i18n' >>> a10n('') '' .. _numeronym: http://en.wikipedia.org/wiki/Numeronym """ if len(string) < 3: return string return '%s%s%s' % (string[0], len(string[1:-1]), string[-1]) class StringBuffer(object): """ This is meant to be a better file-like string buffer. Faster than StringIO, better encoding handling than cStringIO. This one is for unicode text strings. Look for ByteBuffer if you want to handle byte strings. (NOTE: not quite done yet) """ def __init__(self, default_encoding=None, errors='strict'): self.data = collections.deque() self.default_encoding = default_encoding or 'utf-8' self.errors = errors def write(self, s): if not isinstance(s, unicode): enc = self.default_encoding errs = self.errors try: s = s.decode(enc, errs) except AttributeError: raise ValueError('write() expected a unicode or byte string') self.data.append(s) def truncate(self): self.data = collections.deque() self.write = self.data.append def getvalue(self): return unicode().join(self.data) ANSI_ESCAPE_BEGIN = '\x1b[' ANSI_TERMINATORS = ('H', 'f', 'A', 'B', 'C', 'D', 'R', 's', 'u', 'J', 'K', 'h', 'l', 'p', 'm') def strip_ansi(text): """Strips ANSI escape codes from *text*. Useful for the occasional time when a log or redirected output accidentally captures console color codes and the like. >>> strip_ansi('\x1b[0m\x1b[1;36mart\x1b[46;34m\xdc') 'art' The test above is an excerpt from ANSI art on `sixteencolors.net`_. This function does not interpret or render ANSI art, but you can do so with `ansi2img`_ or `escapes.js`_. .. _sixteencolors.net: http://sixteencolors.net .. _ansi2img: http://www.bedroomlan.org/projects/ansi2img .. _escapes.js: https://github.com/atdt/escapes.js """ # TODO: move to cliutils.py nansi, keep, i, text_len = [], True, 0, len(text) while i < text_len: if not keep and text[i] in ANSI_TERMINATORS: keep = True elif keep: keep_end_i = text.find(ANSI_ESCAPE_BEGIN, i) if keep_end_i < 0: break else: nansi.append(text[i:keep_end_i]) i, keep = keep_end_i, False i += 1 if not nansi: return text return type(text)().join(nansi) # attempted unicode + str support def asciify(text, ignore=False): """Converts a unicode or bytestring, *text*, into a bytestring with just ascii characters. Performs basic deaccenting for all you Europhiles out there. Also, a gentle reminder that this is a **utility**, primarily meant for slugification. Whenever possible, make your application work **with** unicode, not against it. Args: text (str or unicode): The string to be asciified. ignore (bool): Configures final encoding to ignore remaining unasciified unicode instead of replacing it. >>> asciify('Beyoncé') == b'Beyonce' True """ try: try: return text.encode('ascii') except UnicodeDecodeError: # this usually means you passed in a non-unicode string text = text.decode('utf-8') return text.encode('ascii') except UnicodeEncodeError: mode = 'replace' if ignore: mode = 'ignore' transd = unicodedata.normalize('NFKD', text.translate(DEACCENT_MAP)) ret = transd.encode('ascii', mode) return ret def is_ascii(text): """Check if a unicode or bytestring, *text*, is composed of ascii characters only. Raises :exc:`ValueError` if argument is not text. Args: text (str or unicode): The string to be checked. >>> is_ascii('Beyoncé') False >>> is_ascii('Beyonce') True """ if isinstance(text, unicode): try: text.encode('ascii') except UnicodeEncodeError: return False elif isinstance(text, bytes): try: text.decode('ascii') except UnicodeDecodeError: return False else: raise ValueError('expected text or bytes, not %r' % type(text)) return True class DeaccenterDict(dict): "A small caching dictionary for deaccenting." def __missing__(self, key): ch = self.get(key) if ch is not None: return ch try: de = unicodedata.decomposition(unichr(key)) p1, _, p2 = de.rpartition(' ') if int(p2, 16) == 0x308: ch = self.get(key) else: ch = int(p1, 16) except (IndexError, ValueError): ch = self.get(key, key) self[key] = ch return ch try: from collections import defaultdict except ImportError: # no defaultdict means that __missing__ isn't supported in # this version of python, so we define __getitem__ def __getitem__(self, key): try: return super(DeaccenterDict, self).__getitem__(key) except KeyError: return self.__missing__(key) else: del defaultdict # http://chmullig.com/2009/12/python-unicode-ascii-ifier/ # For something more complete, investigate the unidecode # or isounidecode packages, which are capable of performing # crude transliteration. _BASE_DEACCENT_MAP = { 0xc6: u"AE", # Æ LATIN CAPITAL LETTER AE 0xd0: u"D", # Ð LATIN CAPITAL LETTER ETH 0xd8: u"OE", # Ø LATIN CAPITAL LETTER O WITH STROKE 0xde: u"Th", # Þ LATIN CAPITAL LETTER THORN 0xc4: u'Ae', # Ä LATIN CAPITAL LETTER A WITH DIAERESIS 0xd6: u'Oe', # Ö LATIN CAPITAL LETTER O WITH DIAERESIS 0xdc: u'Ue', # Ü LATIN CAPITAL LETTER U WITH DIAERESIS 0xc0: u"A", # À LATIN CAPITAL LETTER A WITH GRAVE 0xc1: u"A", # Á LATIN CAPITAL LETTER A WITH ACUTE 0xc3: u"A", # Ã LATIN CAPITAL LETTER A WITH TILDE 0xc7: u"C", # Ç LATIN CAPITAL LETTER C WITH CEDILLA 0xc8: u"E", # È LATIN CAPITAL LETTER E WITH GRAVE 0xc9: u"E", # É LATIN CAPITAL LETTER E WITH ACUTE 0xca: u"E", # Ê LATIN CAPITAL LETTER E WITH CIRCUMFLEX 0xcc: u"I", # Ì LATIN CAPITAL LETTER I WITH GRAVE 0xcd: u"I", # Í LATIN CAPITAL LETTER I WITH ACUTE 0xd2: u"O", # Ò LATIN CAPITAL LETTER O WITH GRAVE 0xd3: u"O", # Ó LATIN CAPITAL LETTER O WITH ACUTE 0xd5: u"O", # Õ LATIN CAPITAL LETTER O WITH TILDE 0xd9: u"U", # Ù LATIN CAPITAL LETTER U WITH GRAVE 0xda: u"U", # Ú LATIN CAPITAL LETTER U WITH ACUTE 0xdf: u"ss", # ß LATIN SMALL LETTER SHARP S 0xe6: u"ae", # æ LATIN SMALL LETTER AE 0xf0: u"d", # ð LATIN SMALL LETTER ETH 0xf8: u"oe", # ø LATIN SMALL LETTER O WITH STROKE 0xfe: u"th", # þ LATIN SMALL LETTER THORN, 0xe4: u'ae', # ä LATIN SMALL LETTER A WITH DIAERESIS 0xf6: u'oe', # ö LATIN SMALL LETTER O WITH DIAERESIS 0xfc: u'ue', # ü LATIN SMALL LETTER U WITH DIAERESIS 0xe0: u"a", # à LATIN SMALL LETTER A WITH GRAVE 0xe1: u"a", # á LATIN SMALL LETTER A WITH ACUTE 0xe3: u"a", # ã LATIN SMALL LETTER A WITH TILDE 0xe7: u"c", # ç LATIN SMALL LETTER C WITH CEDILLA 0xe8: u"e", # è LATIN SMALL LETTER E WITH GRAVE 0xe9: u"e", # é LATIN SMALL LETTER E WITH ACUTE 0xea: u"e", # ê LATIN SMALL LETTER E WITH CIRCUMFLEX 0xec: u"i", # ì LATIN SMALL LETTER I WITH GRAVE 0xed: u"i", # í LATIN SMALL LETTER I WITH ACUTE 0xf2: u"o", # ò LATIN SMALL LETTER O WITH GRAVE 0xf3: u"o", # ó LATIN SMALL LETTER O WITH ACUTE 0xf5: u"o", # õ LATIN SMALL LETTER O WITH TILDE 0xf9: u"u", # ù LATIN SMALL LETTER U WITH GRAVE 0xfa: u"u", # ú LATIN SMALL LETTER U WITH ACUTE 0x2018: u"'", # ‘ LEFT SINGLE QUOTATION MARK 0x2019: u"'", # ’ RIGHT SINGLE QUOTATION MARK 0x201c: u'"', # “ LEFT DOUBLE QUOTATION MARK 0x201d: u'"', # ” RIGHT DOUBLE QUOTATION MARK } DEACCENT_MAP = DeaccenterDict(_BASE_DEACCENT_MAP) _SIZE_SYMBOLS = ('B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') _SIZE_BOUNDS = [(1024 ** i, sym) for i, sym in enumerate(_SIZE_SYMBOLS)] _SIZE_RANGES = list(zip(_SIZE_BOUNDS, _SIZE_BOUNDS[1:])) def bytes2human(nbytes, ndigits=0): """Turns an integer value of *nbytes* into a human readable format. Set *ndigits* to control how many digits after the decimal point should be shown (default ``0``). >>> bytes2human(128991) '126K' >>> bytes2human(100001221) '95M' >>> bytes2human(0, 2) '0.00B' """ abs_bytes = abs(nbytes) for (size, symbol), (next_size, next_symbol) in _SIZE_RANGES: if abs_bytes <= next_size: break hnbytes = float(nbytes) / size return '{hnbytes:.{ndigits}f}{symbol}'.format(hnbytes=hnbytes, ndigits=ndigits, symbol=symbol) class HTMLTextExtractor(HTMLParser): def __init__(self): self.reset() self.strict = False self.convert_charrefs = True self.result = [] def handle_data(self, d): self.result.append(d) def handle_charref(self, number): if number[0] == u'x' or number[0] == u'X': codepoint = int(number[1:], 16) else: codepoint = int(number) self.result.append(unichr(codepoint)) def handle_entityref(self, name): try: codepoint = htmlentitydefs.name2codepoint[name] except KeyError: self.result.append(u'&' + name + u';') else: self.result.append(unichr(codepoint)) def get_text(self): return u''.join(self.result) def html2text(html): """Strips tags from HTML text, returning markup-free text. Also, does a best effort replacement of entities like " " >>> r = html2text(u'Test &(\u0394ημώ)') >>> r == u'Test &(\u0394\u03b7\u03bc\u03ce)' True """ # based on answers to http://stackoverflow.com/questions/753052/ s = HTMLTextExtractor() s.feed(html) return s.get_text() _EMPTY_GZIP_BYTES = b'\x1f\x8b\x08\x089\xf3\xb9U\x00\x03empty\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00' _NON_EMPTY_GZIP_BYTES = b'\x1f\x8b\x08\x08\xbc\xf7\xb9U\x00\x03not_empty\x00K\xaa,I-N\xcc\xc8\xafT\xe4\x02\x00\xf3nb\xbf\x0b\x00\x00\x00' def gunzip_bytes(bytestring): """The :mod:`gzip` module is great if you have a file or file-like object, but what if you just have bytes. StringIO is one possibility, but it's often faster, easier, and simpler to just use this one-liner. Use this tried-and-true utility function to decompress gzip from bytes. >>> gunzip_bytes(_EMPTY_GZIP_BYTES) == b'' True >>> gunzip_bytes(_NON_EMPTY_GZIP_BYTES).rstrip() == b'bytesahoy!' True """ return zlib.decompress(bytestring, 16 + zlib.MAX_WBITS) _line_ending_re = re.compile(r'(\r\n|\n|\x0b|\f|\r|\x85|\x2028|\x2029)', re.UNICODE) def iter_splitlines(text): r"""Like :meth:`str.splitlines`, but returns an iterator of lines instead of a list. Also similar to :meth:`file.next`, as that also lazily reads and yields lines from a file. This function works with a variety of line endings, but as always, be careful when mixing line endings within a file. >>> list(iter_splitlines('\nhi\nbye\n')) ['', 'hi', 'bye', ''] >>> list(iter_splitlines('\r\nhi\rbye\r\n')) ['', 'hi', 'bye', ''] >>> list(iter_splitlines('')) [] """ prev_end, len_text = 0, len(text) # print('last: %r' % last_idx) # start, end = None, None for match in _line_ending_re.finditer(text): start, end = match.start(1), match.end(1) # print(start, end) if prev_end <= start: yield text[prev_end:start] if end == len_text: yield '' prev_end = end tail = text[prev_end:] if tail: yield tail return def indent(text, margin, newline='\n', key=bool): """The missing counterpart to the built-in :func:`textwrap.dedent`. Args: text (str): The text to indent. margin (str): The string to prepend to each line. newline (str): The newline used to rejoin the lines (default: ``\\n``) key (callable): Called on each line to determine whether to indent it. Default: :class:`bool`, to ensure that empty lines do not get whitespace added. """ indented_lines = [(margin + line if key(line) else line) for line in iter_splitlines(text)] return newline.join(indented_lines) def is_uuid(obj, version=4): """Check the argument is either a valid UUID object or string. Args: obj (object): The test target. Strings and UUID objects supported. version (int): The target UUID version, set to 0 to skip version check. >>> is_uuid('e682ccca-5a4c-4ef2-9711-73f9ad1e15ea') True >>> is_uuid('0221f0d9-d4b9-11e5-a478-10ddb1c2feb9') False >>> is_uuid('0221f0d9-d4b9-11e5-a478-10ddb1c2feb9', version=1) True """ if not isinstance(obj, uuid.UUID): try: obj = uuid.UUID(obj) except (TypeError, ValueError, AttributeError): return False if version and obj.version != int(version): return False return True def escape_shell_args(args, sep=' ', style=None): """Returns an escaped version of each string in *args*, according to *style*. Args: args (list): A list of arguments to escape and join together sep (str): The separator used to join the escaped arguments. style (str): The style of escaping to use. Can be one of ``cmd`` or ``sh``, geared toward Windows and Linux/BSD/etc., respectively. If *style* is ``None``, then it is picked according to the system platform. See :func:`args2cmd` and :func:`args2sh` for details and example output for each style. """ if not style: style = 'cmd' if sys.platform == 'win32' else 'sh' if style == 'sh': return args2sh(args, sep=sep) elif style == 'cmd': return args2cmd(args, sep=sep) raise ValueError("style expected one of 'cmd' or 'sh', not %r" % style) _find_sh_unsafe = re.compile(r'[^a-zA-Z0-9_@%+=:,./-]').search def args2sh(args, sep=' '): """Return a shell-escaped string version of *args*, separated by *sep*, based on the rules of sh, bash, and other shells in the Linux/BSD/MacOS ecosystem. >>> print(args2sh(['aa', '[bb]', "cc'cc", 'dd"dd'])) aa '[bb]' 'cc'"'"'cc' 'dd"dd' As you can see, arguments with no special characters are not escaped, arguments with special characters are quoted with single quotes, and single quotes themselves are quoted with double quotes. Double quotes are handled like any other special character. Based on code from the :mod:`pipes`/:mod:`shlex` modules. Also note that :mod:`shlex` and :mod:`argparse` have functions to split and parse strings escaped in this manner. """ ret_list = [] for arg in args: if not arg: ret_list.append("''") continue if _find_sh_unsafe(arg) is None: ret_list.append(arg) continue # use single quotes, and put single quotes into double quotes # the string $'b is then quoted as '$'"'"'b' ret_list.append("'" + arg.replace("'", "'\"'\"'") + "'") return ' '.join(ret_list) def args2cmd(args, sep=' '): r"""Return a shell-escaped string version of *args*, separated by *sep*, using the same rules as the Microsoft C runtime. >>> print(args2cmd(['aa', '[bb]', "cc'cc", 'dd"dd'])) aa [bb] cc'cc dd\"dd As you can see, escaping is through backslashing and not quoting, and double quotes are the only special character. See the comment in the code for more details. Based on internal code from the :mod:`subprocess` module. """ # technique description from subprocess below """ 1) Arguments are delimited by white space, which is either a space or a tab. 2) A string surrounded by double quotation marks is interpreted as a single argument, regardless of white space contained within. A quoted string can be embedded in an argument. 3) A double quotation mark preceded by a backslash is interpreted as a literal double quotation mark. 4) Backslashes are interpreted literally, unless they immediately precede a double quotation mark. 5) If backslashes immediately precede a double quotation mark, every pair of backslashes is interpreted as a literal backslash. If the number of backslashes is odd, the last backslash escapes the next double quotation mark as described in rule 3. See http://msdn.microsoft.com/en-us/library/17w5ykft.aspx or search http://msdn.microsoft.com for "Parsing C++ Command-Line Arguments" """ result = [] needquote = False for arg in args: bs_buf = [] # Add a space to separate this argument from the others if result: result.append(' ') needquote = (" " in arg) or ("\t" in arg) or not arg if needquote: result.append('"') for c in arg: if c == '\\': # Don't know if we need to double yet. bs_buf.append(c) elif c == '"': # Double backslashes. result.append('\\' * len(bs_buf)*2) bs_buf = [] result.append('\\"') else: # Normal char if bs_buf: result.extend(bs_buf) bs_buf = [] result.append(c) # Add remaining backslashes, if any. if bs_buf: result.extend(bs_buf) if needquote: result.extend(bs_buf) result.append('"') return ''.join(result) def parse_int_list(range_string, delim=',', range_delim='-'): """Returns a sorted list of positive integers based on *range_string*. Reverse of :func:`format_int_list`. Args: range_string (str): String of comma separated positive integers or ranges (e.g. '1,2,4-6,8'). Typical of a custom page range string used in printer dialogs. delim (char): Defaults to ','. Separates integers and contiguous ranges of integers. range_delim (char): Defaults to '-'. Indicates a contiguous range of integers. >>> parse_int_list('1,3,5-8,10-11,15') [1, 3, 5, 6, 7, 8, 10, 11, 15] """ output = [] for x in range_string.strip().split(delim): # Range if range_delim in x: range_limits = list(map(int, x.split(range_delim))) output += list(range(min(range_limits), max(range_limits)+1)) # Empty String elif not x: continue # Integer else: output.append(int(x)) return sorted(output) def format_int_list(int_list, delim=',', range_delim='-', delim_space=False): """Returns a sorted range string from a list of positive integers (*int_list*). Contiguous ranges of integers are collapsed to min and max values. Reverse of :func:`parse_int_list`. Args: int_list (list): List of positive integers to be converted into a range string (e.g. [1,2,4,5,6,8]). delim (char): Defaults to ','. Separates integers and contiguous ranges of integers. range_delim (char): Defaults to '-'. Indicates a contiguous range of integers. delim_space (bool): Defaults to ``False``. If ``True``, adds a space after all *delim* characters. >>> format_int_list([1,3,5,6,7,8,10,11,15]) '1,3,5-8,10-11,15' """ output = [] contig_range = collections.deque() for x in sorted(int_list): # Handle current (and first) value. if len(contig_range) < 1: contig_range.append(x) # Handle current value, given multiple previous values are contiguous. elif len(contig_range) > 1: delta = x - contig_range[-1] # Current value is contiguous. if delta == 1: contig_range.append(x) # Current value is non-contiguous. elif delta > 1: range_substr = '{0:d}{1}{2:d}'.format(min(contig_range), range_delim, max(contig_range)) output.append(range_substr) contig_range.clear() contig_range.append(x) # Current value repeated. else: continue # Handle current value, given no previous contiguous integers else: delta = x - contig_range[0] # Current value is contiguous. if delta == 1: contig_range.append(x) # Current value is non-contiguous. elif delta > 1: output.append('{0:d}'.format(contig_range.popleft())) contig_range.append(x) # Current value repeated. else: continue # Handle the last value. else: # Last value is non-contiguous. if len(contig_range) == 1: output.append('{0:d}'.format(contig_range.popleft())) contig_range.clear() # Last value is part of contiguous range. elif len(contig_range) > 1: range_substr = '{0:d}{1}{2:d}'.format(min(contig_range), range_delim, max(contig_range)) output.append(range_substr) contig_range.clear() if delim_space: output_str = (delim+' ').join(output) else: output_str = delim.join(output) return output_str boltons-16.4.1/setup.cfg0000644000175000017500000000002612727357214013216 0ustar hlehle[wheel] universal = 1 boltons-16.4.1/.travis.yml0000644000175000017500000000036112727357214013510 0ustar hlehlelanguage: python sudo: false cache: pip python: - "2.6" - "2.7" - "3.4" - "3.5" - "pypy" install: "pip install -r requirements-test.txt" script: "py.test --doctest-modules boltons tests" branches: except: - function_builder boltons-16.4.1/CHANGELOG.md0000644000175000017500000011462112727357214013215 0ustar hlehleboltons Changelog ================= Since February 20, 2013 there have been 28 releases and 950 commits for an average of one 34-commit release every 6.4 weeks. 16.4.1 ------ *(June 14, 2016)* This release primarily contains several [statsutils][statsutils] updates. * The biggest change was the addition of [Stats.format_histogram][statsutils.Stats.format_histogram] complete with Freedman bin selection and other useful options. * Added inter-quartile range (iqr) to [statsutils.Stats][statsutils.Stats] * Adding mad (median absolute deviation) to [Stats.describe][statsutils.Stats.describe], since median and std_dev were already there. 16.4.0 ------ *(June 8, 2016)* another significant release, thanks to the addition of funcutils.wraps and funcutils.FunctionBuilder. also iterutils.chunked speedup, and tbutils.ParsedException.to_string. * [funcutils.wraps][funcutils.wraps]: Just like functools.wraps, but can preserve the function signature as well. * [funcutils.FunctionBuilder][funcutils.FunctionBuilder]: The basis for [funcutils.wraps][funcutils.wraps], this full-featured type enables programmatically creating functions, from scratch or from existing functions. Supports all Python 2 and 3 function features. * [ecoutils][ecoutils]: Python 2.4 and 2.5 support. * [iterutils][iterutils]: optimize [iterutils.chunked_iter][iterutils.chunked_iter] (2-5x faster depending on runtime). [See #79][i79]. * [tbutils][tbutils]: add the [ParsedException.to_string][tbutils.ParsedException.to_string] method, to convert parsed exceptions back into strings, possibly after manipulation * switch FunctionBuilder on Python 2 to be congruent with Python 3 (keywords attribute renamed to varkw, preview users might have to adjust) 16.3.1 ------ *(May 24, 2016)* Just a couple of [ecoutils][ecoutils] follow-ons, removing uuid dependency and adding the ability to scrub identifiable data. 16.3.0 ------ *(May 23, 2016)* Big, big update. Lots of additions, a few bugfixes. * [ecoutils][ecoutils] - Python runtime/environment profile generator * [timeutils.strpdate][timeutils.strpdate] - like datetime.datetime.strpdate but for date * [timeutils.daterange][timeutils.daterange] - like range() but for datetime.date objects * [strutils.parse_int_list][strutils.parse_int_list] and [strutils.format_int_list][strutils.format_int_list] * [cacheutils][cacheutils] * [cachedproperty][cacheutils.cachedproperty] * [cacheutils.cachedmethod][cacheutils.cachedmethod] * [cacheutils.cached][cacheutils.cached] now accepts a callable, as well. * ``cacheutils.make_cache_key`` is now public, should others need it * [statsutils.Stats][statsutils.Stats] update, several new methods, including [Stats.describe][statsutils.Stats.describe] * A few [socketutils][socketutils] platform tweaks * `debugutils.wrap_trace` preview 16.2.2 ------ *(May 3, 2016)* many small tweaks to socketutils.BufferedSocket, including optional inclusion of the delimiter in recv_until. also undid the enabling of bak files with AtomicSaver on windows * Small [socketutils.BufferedSocket][socketutils.BufferedSocket] tweaks * make recv_until conditionally return the delimiter (by default it does not). also fix a NetstringException inheritance typo * [socketutils][socketutils]: rename BufferedSocket.recv_lock to _recv_lock, and same for send_lock. * add a bunch of simple passthrough methods to better fill out socket's API * add .fileno/.close/.shutdown to [socketutils.BufferedSocket][socketutils.BufferedSocket] * added type/family/proto [socketutils.BufferedSocket][socketutils.BufferedSocket] passthrough properties * BufferedSocket: also lock on .shutdown() * adding an rbuf_unconsumed attribute for post-close debugging, per @doublereedkurt's request * `getsendbuffer()` returns a bytestring and `recv_size()` uses the proper `._recvsize` on the first socket fetch * [fileutils.AtomicSaver][fileutils.AtomicSaver]: revert bak file as it was causing confusion, per [nvie/pip-tools#351](https://github.com/nvie/pip-tools/issues/351) 16.2.1 ------ *(April 29, 2016)* This version sees the soft debut of [socketutils][socketutils], which includes wrappers and tools for working with the built-in socket. A lot of [socketutils.BufferedSocket][socketutils.BufferedSocket] changes. * [BufferedSocket.recv_until][socketutils.BufferedSocket.recv_until] now supports multibyte delimiters and also includes the delimiter in its returns. * Better BufferedSocket timeout discipline throughout. * Various BufferedSocket argument name changes, *maxbytes* became *maxsize*, *n* became *size*, *marker* became *delimiter*, etc. * [BufferedSocket][socketutils.BufferedSocket] BufferedSocket became threadsafe * [BufferedSocket.recv][socketutils.BufferedSocket.recv] now always returns the contents of the internal buffer before doing a socket call. * [BufferedSocket.recv_close][socketutils.BufferedSocket.recv_close] now exists to receive until the sending end closes the connection. * Can now pass *recvsize* to [BufferedSocket][socketutils.BufferedSocket] constructor to tune the size passed to the lower-level recv call. * [socketutils][socketutils] got documented and tested. 16.2.0 ------ *(April 18, 2016)* adding shell args escaper-joiners to strutils (escape_shell_args, args2cmd, args2sh) as well as a rare breaking fix to [iterutils.pairwise][iterutils.pairwise]. * Argument joiners, functions to join command line arguments in context-appropriate ways: * [strutils.escape_shell_args][strutils.escape_shell_args] * [strutils.args2cmd][strutils.args2cmd] * [strutils.args2sh][strutils.args2sh] * BREAKING: finally fixing [iterutils.pairwise][iterutils.pairwise]. pairwise used to call to `chunked`, now it calls to `windowed`. `pairwise([1, 2, 3, 4])` no longer returns `[(1, 2), (3, 4)]`. Instead, it returns `[(1, 2), (2, 3), (3, 4)]`, which is what I always mean when I say pairwise, but not what the original contributor implemented. * Adding a universal wheel distribution option! 16.1.1 ------ *(March 6, 2016)* Added [iterutils.same][iterutils.same], improvement of Windows [fileutils.AtomicSaver][fileutils.AtomicSaver] behavior for old filesystems, bugfix on [strutils.is_uuid][strutils.is_uuid], expansion of [strutils.pluralize][strutils.pluralize], new trove classifiers and docs improvements! * [fileutils.replace][fileutils.replace]: use bak file option for win32 ReplaceFile for slightly better corner case coverage on less featureful filesystems * [strutils.pluralize][strutils.pluralize]: Add more irregular plurals * [strutils.is_uuid][strutils.is_uuid]: Catch un-parsable UUIDs. * [iterutils.same][iterutils.same]: Return `True` when all values in iterable are the same. 16.1.0 ------ *(February 24, 2016)* The centerpiece of this release is highly improved Windows support for [fileutils.atomic_save][fileutils.atomic_save] via [ReplaceFile](https://msdn.microsoft.com/en-us/library/windows/desktop/aa365512%28v=vs.85%29.aspx) system call. This functionality is also made available directly via [fileutils.replace][fileutils.replace], which is akin to Python 3.3+'s [os.replace][os.replace], except that `os.replace`'s approach has [arguably poorer behavior and atomicity](http://stupidpythonideas.blogspot.com/2014/07/getting-atomic-writes-right.html) compared to `fileutils.replace`. Also, a couple new strutils, and [iterutils.backoff][iterutils.backoff] grew a jitter argument. * [iterutils.backoff][iterutils.backoff] now supports start=0 * More comprehensive [iterutils.backoff][iterutils.backoff] argument checking/validation * [fileutils.replace][fileutils.replace] and [fileutils.atomic_rename][fileutils.atomic_rename] are now public functions in [fileutils][fileutils] with cross-platform implementations ([discussion here](https://github.com/mahmoud/boltons/issues/60)) * [tableutils.Table][tableutils.Table]s have a metadata argument and attribute for miscellaneous metadata. * [strutils.is_ascii][strutils.is_ascii] and [strutils.is_uuid][strutils.is_uuid]: About as straightforward as they are handy. * Tox testing improvements 16.0.1 ------ *(January 24, 2016)* DummyFile, Table.metadata, better exception handling, and in-progress iterutils.get_path * Small format fix in [iterutils.one][iterutils.one] for None * Initial implementation of [fileutils.DummyFile][fileutils.DummyFile], which allows for easy no-op file handling without restructuring code. Sort of like a dummy RLock for systems without threading, if you've seen those. * avoid catching BaseException in all boltons * better error handling in iterutils.get_path 16.0.0 ------ One important fix and one small but handy string function. * Fixed an [LRU][cacheutils.LRU] bug related to the 15.1.1 refactor. Also enhanced LRU testing with doubly-linked list invariant enforcement. * Added [strutils.indent][strutils.indent], the counterpart to [textwrap.dedent](https://docs.python.org/2/library/textwrap.html#textwrap.dedent). 15.1.1 ------ *(November 18, 2015)* A lot of bugfixes and docfixes in 15.1.1. updated AtomicSaver for better permissions handling, update BufferedSocket message sending, beta version of iterutils.get_path, several docs fixes, Stats zscore and cache bugfix, and an LRU refactor with significantly improved behavior and code factoring. * Updated [fileutils.AtomicSaver][fileutils.AtomicSaver] handling of filesystem permissions to be simpler and more secure. This also merges `dest_perms` and `part_perms` arguments to AtomicSaver and atomic_save. * Fix large message sending with [socketutils.BufferedSocket][socketutils.BufferedSocket] * [strutils.iter_splitlines][strutils.iter_splitlines] is now in the docs. * [cacheutils][cacheutils]: now imports RLock from the right place for python 2 * [statsutils][statsutils]: Only `delattr` when `hasattr` in [Stats.clear_cache][statsutils.Stats.clear_cache] * [statsutils.Stats][statsutils.Stats]: Add [Stats.get_zscore][statsutils.Stats.get_zscore] to support calculating the [z-score][zscore] (see also: t-statistic) * [cacheutils.LRU][cacheutils.LRU]: Massive refactor of the backing linked list for better handling of duplicate data in the cache. More aggressive locking and better `__eq__` 15.1.0 ------ *(September 23, 2015)* Reached the first release version of [iterutils.remap][iterutils.remap](), fully tested and documented. Also a couple of tweaks to expose the [iterutils.unique][iterutils.unique] docs. 15.0.2 ------ *(September 9, 2015)* a couple [dictutils.OMD][dictutils.OMD] fixes to [dictutils.OMD.pop][dictutils.OMD.pop] and [dictutils.OMD.popall][dictutils.OMD.popall] to make them consistent with the docstrings. and the accompanying tests of course. * fix [dictutils.OMD.setdefault][dictutils.OMD.setdefault] to default to None and not empty list, per documentation (and add a test to the same effect) 15.0.1 ------ *(August 27, 2015)* * Added [OrderedMultiDict.sortedvalues()][OrderedMultiDict.sortedvalues], which returns a copy of the OMD with sublists within a keyspace sorted. * Fixing a bug in [dictutils.OrderedMultiDict][dictutils.OrderedMultiDict]'s addlist method that caused values to be added multiple times. * Fixing a [iterutils.backoff][iterutils.backoff] string identity check [OrderedMultiDict.sortedvalues]: http://boltons.readthedocs.org/en/latest/dictutils.html#boltons.dictutils.OrderedMultiDict.sortedvalues 15.0.0 ------ *(August 19, 2015)* Finally the 15.0 major release. All passing PRs and feature requests from the first wave addressed and closed. tzutils merged into timeutils. AtomicSaver approach and API much improved. Several other features added: * [iterutils.backoff][iterutils.backoff] and [iterutils.backoff_iter][iterutils.backoff_iter] for exponential backoff * [iterutils.frange][iterutils.frange] and [iterutils.xfrange][iterutils.xfrange] for floating point range generation * Slightly more permissive [jsonutils.JSONLIterator][jsonutils.JSONLIterator] blank line ignoring * [strutils.iter_splitlines][strutils.iter_splitlines] for lazily getting lines from a larger string * [timeutils.dt_to_timestamp][timeutils.dt_to_timestamp], per the long-lived PR [#13][i13]. * Merged tzutils into timeutils * [fileutils.AtomicSaver][fileutils.AtomicSaver] rewrite and redoc * -teens support for [strutils.ordinalize][strutils.ordinalize] * made [iterutils.one][iterutils.one] consistent with [iterutils.first][iterutils.first] 0.6.6 ----- *(July 31, 2015)* Fix atomic saving open-file issue for Windows. * Patch for AtomicSaver on Windows. Full rewrite comes in 15.0.0. * [strutils.gunzip_bytes][strutils.gunzip_bytes] for decompressing a gzip bytestring 0.6.5 ----- *(July 30, 2015)* BufferedSocket work, html2text, pairwise shortcut, is_container, plural typo fix, [timeutils.isoparse][timeutils.isoparse], [cacheutils.ThresholdCounter][cacheutils.ThresholdCounter], and lots of testing * Add [iterutils.first][iterutils.first] function * Add [cacheutils.ThresholdCounter][cacheutils.ThresholdCounter] * Add JSONL verification to jsonutils * Add [timeutils.isoparse][timeutils.isoparse] * Add [strutils.html2text][strutils.html2text] and [strutils.HTMLTextExtractor][strutils.HTMLTextExtractor] * Fix [strutils.pluralize][strutils.pluralize] (indeces -> indices, per [#41][i41]) * Add [iterutils.is_container][iterutils.is_container] function * Fixed a small formatting bug in [tbutils.ExceptionInfo][tbutils.ExceptionInfo] that added an extra 'builtins.' for builtin exceptions under python 3 * Added tests for many modules * Create [iterutils.pairwise][iterutils.pairwise] shortcuts for pairwise chunks since pairs (key/val) are common * Additional 2.6 compatibility and tests * Fixed CachedInstancePartial to be Python 3 friendly without breaking PyPy. * Made formatutils Python 3 compatible * Rename sockutils to socketutils and other changes 0.6.4 ----- *(May 10, 2015)* Fixed multiline exception message handling in ParsedException. added mathutils. adding a tentative version of socketutils. fix LRU.popitem. fix OMD.__eq__. * Fix a bug where [dictutils.OrderedMultiDict][dictutils.OrderedMultiDict]'s __eq__ would fail with non-iterable objects of comparison * Fixed `LRU.popitem` to return a key value pair * Added mathutils with [mathutils.ceil][mathutils.ceil] and [mathutils.floor][mathutils.floor] implementations that can search a fixed set of choices using the bisect module. * Fix [excutils.ParsedException][excutils.ParsedException] so exception message would not start with whitespace * Fix multiline exception messages * Adding [socketutils.BufferedSocket][socketutils.BufferedSocket] and [socketutils.NetstringSocket][socketutils.NetstringSocket] 0.6.3 ----- *(April 20, 2015)* Add typeutils, remove compat.py, make ParsedException work with eval()ed code * Properly parse tracebacks with missing source. Resolves [#30][i30] * Tweak the docs for [typeutils.get_all_subclasses][typeutils.get_all_subclasses] * Moved [typeutils.make_sentinel][typeutils.make_sentinel] into typeutils and removed the confusing compat module * Add in typeutils with modifications per the caveats of [#15][i15] * Added function [iterutils.one][iterutils.one] 0.6.2 ----- *(April 11, 2015)* Add partial_ordering, fix LRU repr and addition behavior * Add [funcutils.partial_ordering][funcutils.partial_ordering](), decorator similar to functools.total_ordering() * Fixed [cacheutils.LRU][cacheutils.LRU]'s behavior per [#21][i21] * Fix [cacheutils.LRU][cacheutils.LRU] repr reversal, fixes [#20][i20] 0.6.0 ----- *(April 10, 2015)* Python 3 support and several community bugfixes. Docs clarifications, too. * Make boltons Python 3 compatible without any external dependencies. All modules are independent and work in Python 2.6, 2.7, 3.4, and PyPy. * clarify TracebackInfo.from_current() method gap, per user 'dl__' here: http://www.reddit.com/r/Python/comments/321d3o/boltons_over_100_python_utilities/ * Fix the [cacheutils.cached][cacheutils.cached] decorator, adding a sanity test, fixes [#12][i12] * Fix bytes2human when builtin zip returns iterators * Simplified logic of [iterutils.chunked][iterutils.chunked] 0.5.1 ----- *(April 10, 2015)* A lot of bugfixes and Python 2.6 and PyPy compatibility changes thanks to community contributions and encouragement. * Corrected cases where OMD was not exactly a dropin for OrderedDict * conditional availability of [gcutils.get_all][gcutils.get_all] based on pypy or cpython, also [gcutils.is_tracked][gcutils.is_tracked] was added in 2.7, so making [gcutils.get_all][gcutils.get_all] work with 2.6 * Made namedutils backwards compatibilty for python 2.6 best effort * Fix invalid part_path usage in [fileutils.AtomicSaver][fileutils.AtomicSaver] 0.5.0 ----- *(April 9, 2015)* First publicly released version. The major focus of this release was docs, docstrings, and Read The Docs. * Cleared out __init__ module for maximum independence * making [statsutils.median][statsutils.median] use _get_quantile and add [statsutils.trimean][statsutils.trimean] * Switching the [statsutils.Stats][statsutils.Stats] type to be more sorted-data oriented, since it's only for offline analysis of unordered data * Made consistent multi-line string formats, as well as usage of the term 'builtin' vs 'built-in' (noun vs adjective) * Instrumented LRI with stats tracking * Made [timeutils.decimal_relative_time][timeutils.decimal_relative_time] cardinalization optional * Removed timeutils dependency on strutils * Made [tbutils.TracebackInfo][tbutils.TracebackInfo] classmethods work with no arguments. * Renamed ParsedTB to [tbutils.ParsedException][tbutils.ParsedException] * Made [dictutils.OMD][dictutils.OMD] .get()/.getlist() semantics more consistent. * finalizing .todict() and adding .sorted() to the [dictutils.OMD][dictutils.OMD] * Removed osutils and adding a note about utils in general * Made cacheutils more consistent between LRU and LRI, adding some cacheutils docs * Deprecate osutils, moving its contents into fileutils * Adding in-process statsutils2, with new DataAnalyzer and get_pearson_type (not merged yet) 0.4.2 ----- *(March 8, 2015)* Mostly a dictutils API update (addlist), but also gcutils. * [dictutils.OMD][dictutils.OMD]: split out addlist() from add(), removing the multi kwarg * adding gcutils with [gcutils.GCToggler][gcutils.GCToggler] and gc.get_all 0.4.1 ----- *(February 26, 2015)* adding mboxutils * adding mboxutils for handy dandy /var/mail integrations like cronfed 0.4.0 ----- *(February 23, 2015)* updated tbutils, JSONL support, initial cacheutils, atomic writer, hashtags * tbutils: Changed the way exceptions are parsed out of tracebacks * tbutils: Guard against potential linecache issues * Defined/implemented [iterutils.windowed_iter][iterutils.windowed_iter] corner case behavior * Added from_path to [fileutils.FilePerms][fileutils.FilePerms] * Adding [strutils.find_hashtags][strutils.find_hashtags] * Add ignore patterns to [fileutils.iter_find_files][fileutils.iter_find_files] 0.3.0 ----- *(October 19, 2014)* First alpha release. Practically, everything not mentioned above was added in this release. * tbutils: add ContextualTracebackInfo and ContextualExceptionInfo * cacheutils: a few minor changes to the BasicCache * update tbutils with some critical bits. still need to add convenience method for ExceptionInfo -> default exception print, also need to add more docstrings. * adding initial jsonutils with JSONL support * added [cacheutils.LRU][cacheutils.LRU] * added [timeutils.parse_timedelta][timeutils.parse_timedelta] * fixing iteritems with multi=False in the [dictutils.OrderedMultiDict][dictutils.OrderedMultiDict] (should return first key, but last value not first) * debugutils: add pdb excepthook and [debugutils.pdb_on_signal][debugutils.pdb_on_signal] * add [fileutils.mkdir_p][fileutils.mkdir_p] * tableutils: add maxlen to table text stuff * fix date citation for gob's * adding pure-Python implementation of Gob's algorithm * fix object header guessing * namedutils: add namedtuple support * fix a headers bug in tableutils * tableutils: add a couple more do-not-recurse types, add UnsupportedData exception for better recursion, insert recursive entries in-line, improve 'Object' strategy heuristic * wrap up html tag customization and fix a related bug * make html configurable via class attributes * strengthen the max_depth check * InputType classes are just shorter and clearer, imo * new from_dict, from_list, from_object, removing old ones. almost used a MetaClass for this * starting new table * only update the integer for the field being updated * fixing a verbalization/pronunciation issue * no regex for better error messages * being opinionated about some FilePerms things * adding initial version of fileutils/FilePerms * update formatutils * fix a slightly nasty namedlist bug * make OrderedMultiDict.get()'s default allow singulars * sync over ExceptionInfo * add from_current() classmethod with depth option to Callpoint class for easier instantiation * it's called a numeronym * add a repr to ParsedTB. A bit verbose, but better than nothing. * add ParsedTB, which provides the ability to parse tracebacks dumped out to logs, the command line, etc. * improve test output and make assertion that new except hook is the same as the builtin. * update tbutils to use the more-powerful Callpoint type. * copy_function * partially clean up partial stuff * first version of the namedlist * fixing up namedtuple, groundwork for namedlist. humorously named module. * embed value in cell for incremental iteration boost on OMD * reorganize code, add 'default' argument to poplast() * make key argument optional to OMD.poplast * rectifying inconsistent names and adjusting respective tests accordingly. using smashcase because that's what builtin dict() uses. * fix reverse; missing yield from! * add initial table biz * add get_counts * add [dictutils.OrderedMultiDict.get_inverted][dictutils.OrderedMultiDict.get_inverted]() for those handy reverse lookups * break out skip list to FastIter OMD + bench * add [strutils.a10n][strutils.a10n]() * fix a bug in [dictutils.OrderedMultiDict][dictutils.OrderedMultiDict]'s .add() * adding initial reimplementation of OMD * adding some tests to dictutils * update boltons formatutils to match what's goin on in lithoxyl * remove infer_pos_args() from strutils (already in formatutils) * add formatutils to boltons * fix a potential infinite recursion in LocalTZ * use more explicit names for Local/Constant tzinfo types * add a basic but handy file finder * add infer_positional_args() to strutils (from lithoxyl) * split BasicCache out of dictutils into cacheutils * update median calculation slightly * add appropriate stacklevel to deprutils warning * add an initial version of deprutils (basic utils for facilitating deprecation) * add bytes2human * first version of some basic timezone utils which came in handy for a train scheduling application I wrote (etavta) * reorder imports for pep8 * redo plain-english relative_time() to have a decimal rounding factor and handle future dates * swap the order of cardinalize()'s arguments after intuiting the wrong order a couple times. gotta be consistent, this isn't PHP. * a weird little relative time approach * add total_seconds() implementation for Python <2.7, rename relative_datetime to relative_time * add a relative datetime function in a new module: timeutils * a little more transparency with orderedmultidict's maphistory * add a test for BasicCache * add the super simple BasicCache, a size-limited defaultdict-like thing * add a cheesy little splay list construct that can be used for splay- like manual reordering for eventual optimization * traceback utils, first draft * add strip_ansi() (need to make a cliutils or something) * add ansi strip task * mess with list tuning * add ordinalize() * add __all__ to statsutils * add more stats docs and doctests * add some stats functions * add unit_len() * add pluralize/singularize/cardinalize to strutils * add __all__s all around, clean up imports a bit * adding license * add sorted queue type, make it the default * fix little bug in insert * inheriting from list necessitates overriding the deprecated __get-, __set-, and __del- slice methods * hacky refactor to have a BasePriorityQueue to make room for SortedPriorityQueue with peek_n, etc. * add a little docstring and update sort method in BarrelList * add HeapPriorityQueue * tidy up listutils comments and imports * move treeutils out of boltons since I don't really think a pure python version actually adds much. i'll make an academic one-off repo for less practical data structure experiments like that. * inherit from list * add reverse() to blist * add index() to blist * cheesy __setitem__() for blist * add __delitem__() to BarrelList * change the way the in-place sort works with just one list * tune the list size a bit * add slicing to BarrelList * add initial version of BarrelList, a pure-python b-list-y thing to approximate O(log(n)) behavior by multiplexing the fast O(n) list operations * switch to new dead index interval approach; the IndexedSet is about half the speed of a list in the ultra-pathological case of random popping on the low end of the IndexedSet * made BisectTree's get() defaulting work a bit more like a dict's * added get_adjacent and miscellaneous to BisectTree * added a default name and always-falsy __nonzero__ to Sentinel * add pop() for BisectTree and export the generic Tree * make a bisect tree, because O(n) is still pretttttty fast up to about 100k items * add a little hack to chunked/chunked_iter to make it work nicely with strings * tidy up ki_vs_vi_lrh usage just a titch * revamp indices behavior (key_size and value_size) * switch to new multi-key mode * pretty much done porting insert, delete, balance (the basic operations) * switch to negative refs, arbitrary length nodes * add sentinel utility thing * add .index() for list compat, updated exception messages, and added a silly test to show off slicing and indexing * add slicing support and .clear() * remove ifilter dependency (using generator expression) * add .reverse and .sort() to IndexedSet, fix bisection related bug exposing MISSING (insort requested index, not real_index) * pretty much all fundy IndexedSet bugs hit and fixed, looks like * IndexedSet getting much closer * initial rough draft of IndexedSet with a short docstring and a bunch of fixes already (still not workin tho) * add dictutils (OrderedMultiDict) * it was a long time coming, but I'm finally halfway happy with this wrapped exception * add uniqueification capabilities * go back to using __new__ and positional arguments * exception wrapping green path mostly working * working on a wrapping exception mixin thing for less lossy nonraising. * add asciify and update slugify with ascii flag * add basic docs and doctests to strutils * scratch that, regexes still faster * add under2camel, camel2under, and slugify (and split_punct_ws, which is much faster than re-based punctuation splitting) * python3-compatible doctest for bucketize_bool * bucketize and bucketize_bool, with docs * add examples to chunked and chunked_iter * update split() docs with more examples. * [iterutils.chunked_iter][iterutils.chunked_iter] and [iterutils.chunked][iterutils.chunked] * [iterutils.split][iterutils.split] and [iterutils.split_iter][iterutils.split_iter] work [os.replace]: https://docs.python.org/3/library/os.html#os.replace [cacheutils]: http://boltons.readthedocs.org/en/latest/cacheutils.html [cacheutils.LRU]: http://boltons.readthedocs.org/en/latest/cacheutils.html#boltons.cacheutils.LRU [cacheutils.ThresholdCounter]: http://boltons.readthedocs.org/en/latest/cacheutils.html#boltons.cacheutils.ThresholdCounter [cacheutils.cached]: http://boltons.readthedocs.org/en/latest/cacheutils.html#boltons.cacheutils.cached [cacheutils.cachedmethod]: http://boltons.readthedocs.org/en/latest/cacheutils.html#boltons.cacheutils.cachedmethod [cacheutils.cachedproperty]: http://boltons.readthedocs.org/en/latest/cacheutils.html#boltons.cacheutils.cachedproperty [debugutils.pdb_on_signal]: http://boltons.readthedocs.org/en/latest/debugutils.html#boltons.debugutils.pdb_on_signal [dictutils.OMD]: http://boltons.readthedocs.org/en/latest/dictutils.html#boltons.dictutils.OMD [dictutils.OMD.pop]: http://boltons.readthedocs.org/en/latest/dictutils.html#boltons.dictutils.OrderedMultiDict.pop [dictutils.OMD.popall]: http://boltons.readthedocs.org/en/latest/dictutils.html#boltons.dictutils.OrderedMultiDict.popall [dictutils.OMD.setdefault]: http://boltons.readthedocs.org/en/latest/dictutils.html#boltons.dictutils.OrderedMultiDict.setdefault [dictutils.OrderedMultiDict]: http://boltons.readthedocs.org/en/latest/dictutils.html#boltons.dictutils.OrderedMultiDict [dictutils.OrderedMultiDict.get_inverted]: http://boltons.readthedocs.org/en/latest/dictutils.html#boltons.dictutils.OrderedMultiDict.get_inverted [ecoutils]: http://boltons.readthedocs.org/en/latest/ecoutils.html [excutils.ParsedException]: http://boltons.readthedocs.org/en/latest/excutils.html#boltons.excutils.ParsedException [fileutils]: http://boltons.readthedocs.org/en/latest/fileutils.html [fileutils.replace]: http://boltons.readthedocs.org/en/latest/fileutils.html#boltons.fileutils.replace [fileutils.atomic_rename]: http://boltons.readthedocs.org/en/latest/fileutils.html#boltons.fileutils.atomic_rename [fileutils.atomic_save]: http://boltons.readthedocs.org/en/latest/fileutils.html#boltons.fileutils.atomic_save [fileutils.AtomicSaver]: http://boltons.readthedocs.org/en/latest/fileutils.html#boltons.fileutils.AtomicSaver [fileutils.FilePerms]: http://boltons.readthedocs.org/en/latest/fileutils.html#boltons.fileutils.FilePerms [fileutils.iter_find_files]: http://boltons.readthedocs.org/en/latest/fileutils.html#boltons.fileutils.iter_find_files [fileutils.mkdir_p]: http://boltons.readthedocs.org/en/latest/fileutils.html#boltons.fileutils.mkdir_p [fileutils.DummyFile]: http://boltons.readthedocs.org/en/latest/fileutils.html#boltons.fileutils.DummyFile [funcutils.FunctionBuilder]: http://boltons.readthedocs.org/en/latest/funcutils.html#boltons.funcutils.FunctionBuilder [funcutils.partial_ordering]: http://boltons.readthedocs.org/en/latest/funcutils.html#boltons.funcutils.partial_ordering [funcutils.wraps]: http://boltons.readthedocs.org/en/latest/funcutils.html#boltons.funcutils.wraps [gcutils.GCToggler]: http://boltons.readthedocs.org/en/latest/gcutils.html#boltons.gcutils.GCToggler [gcutils.get_all]: http://boltons.readthedocs.org/en/latest/gcutils.html#boltons.gcutils.get_all [gcutils.is_tracked]: http://boltons.readthedocs.org/en/latest/gcutils.html#boltons.gcutils.is_tracked [i12]: https://github.com/mahmoud/boltons/issues/12 [i13]: https://github.com/mahmoud/boltons/issues/13 [i15]: https://github.com/mahmoud/boltons/issues/15 [i20]: https://github.com/mahmoud/boltons/issues/20 [i21]: https://github.com/mahmoud/boltons/issues/21 [i30]: https://github.com/mahmoud/boltons/issues/30 [i41]: https://github.com/mahmoud/boltons/issues/41 [i79]: https://github.com/mahmoud/boltons/pull/79 [iterutils]: http://boltons.readthedocs.org/en/latest/iterutils.html [iterutils.backoff]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.backoff [iterutils.backoff_iter]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.backoff_iter [iterutils.chunked]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.chunked [iterutils.chunked_iter]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.chunked_iter [iterutils.first]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.first [iterutils.backoff]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.backoff [iterutils.frange]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.frange [iterutils.is_container]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.is_container [iterutils.one]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.one [iterutils.pairwise]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.pairwise [iterutils.same]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.same [iterutils.remap]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.remap [iterutils.split]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.split [iterutils.split_iter]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.split_iter [iterutils.unique]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.unique [iterutils.windowed_iter]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.windowed_iter [iterutils.xfrange]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.xfrange [jsonutils.JSONLIterator]: http://boltons.readthedocs.org/en/latest/jsonutils.html#boltons.jsonutils.JSONLIterator [mathutils.ceil]: http://boltons.readthedocs.org/en/latest/mathutils.html#boltons.mathutils.ceil [mathutils.floor]: http://boltons.readthedocs.org/en/latest/mathutils.html#boltons.mathutils.floor [socketutils]: http://boltons.readthedocs.org/en/latest/socketutils.html [socketutils.BufferedSocket]: http://boltons.readthedocs.org/en/latest/socketutils.html#boltons.socketutils.BufferedSocket [socketutils.BufferedSocket.recv]: http://boltons.readthedocs.org/en/latest/socketutils.html#boltons.socketutils.BufferedSocket.recv [socketutils.BufferedSocket.recv_until]: http://boltons.readthedocs.org/en/latest/socketutils.html#boltons.socketutils.BufferedSocket.recv_until [socketutils.BufferedSocket.recv_close]: http://boltons.readthedocs.org/en/latest/socketutils.html#boltons.socketutils.BufferedSocket.recv_close [socketutils.NetstringSocket]: http://boltons.readthedocs.org/en/latest/socketutils.html#boltons.socketutils.NetstringSocket [statsutils]: http://boltons.readthedocs.org/en/latest/statsutils.html [statsutils.Stats]: http://boltons.readthedocs.org/en/latest/statsutils.html#boltons.statsutils.Stats [statsutils.Stats.clear_cache]: http://boltons.readthedocs.org/en/latest/statsutils.html#boltons.statsutils.Stats.clear_cache [statsutils.Stats.describe]: http://boltons.readthedocs.org/en/latest/statsutils.html#boltons.statsutils.Stats.describe [statsutils.Stats.format_histogram]: http://boltons.readthedocs.org/en/latest/statsutils.html#boltons.statsutils.Stats.format_histogram [statsutils.Stats.get_zscore]: http://boltons.readthedocs.org/en/latest/statsutils.html#boltons.statsutils.Stats.get_zscore [statsutils.median]: http://boltons.readthedocs.org/en/latest/statsutils.html#boltons.statsutils.median [statsutils.trimean]: http://boltons.readthedocs.org/en/latest/statsutils.html#boltons.statsutils.trimean [strutils.HTMLTextExtractor]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.HTMLTextExtractor [strutils.a10n]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.a10n [strutils.args2cmd]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.args2cmd [strutils.args2sh]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.args2sh [strutils.escape_shell_args]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.escape_shell_args [strutils.find_hashtags]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.find_hashtags [strutils.gunzip_bytes]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.gunzip_bytes [strutils.html2text]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.html2text [strutils.indent]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.indent [strutils.iter_splitlines]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.iter_splitlines [strutils.ordinalize]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.ordinalize [strutils.pluralize]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.pluralize [strutils.is_ascii]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.is_ascii [strutils.is_uuid]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.is_uuid [strutils.parse_int_list]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.parse_int_list [strutils.format_int_list]: http://boltons.readthedocs.org/en/latest/strutils.html#boltons.strutils.format_int_list [tableutils]: http://boltons.readthedocs.org/en/latest/tableutils.html [tableutils.Table]: http://boltons.readthedocs.org/en/latest/tableutils.html#boltons.tableutils.Table [tbutils]: http://boltons.readthedocs.org/en/latest/tbutils.html [tbutils.ExceptionInfo]: http://boltons.readthedocs.org/en/latest/tbutils.html#boltons.tbutils.ExceptionInfo [tbutils.ParsedException]: http://boltons.readthedocs.org/en/latest/tbutils.html#boltons.tbutils.ParsedException [tbutils.ParsedException.to_string]: http://boltons.readthedocs.org/en/latest/tbutils.html#boltons.tbutils.ParsedException.to_string [tbutils.TracebackInfo]: http://boltons.readthedocs.org/en/latest/tbutils.html#boltons.tbutils.TracebackInfo [timeutils.daterange]: http://boltons.readthedocs.org/en/latest/timeutils.html#boltons.timeutils.daterange [timeutils.decimal_relative_time]: http://boltons.readthedocs.org/en/latest/timeutils.html#boltons.timeutils.decimal_relative_time [timeutils.dt_to_timestamp]: http://boltons.readthedocs.org/en/latest/timeutils.html#boltons.timeutils.dt_to_timestamp [timeutils.isoparse]: http://boltons.readthedocs.org/en/latest/timeutils.html#boltons.timeutils.isoparse [timeutils.parse_timedelta]: http://boltons.readthedocs.org/en/latest/timeutils.html#boltons.timeutils.parse_timedelta [timeutils.strpdate]: http://boltons.readthedocs.org/en/latest/timeutils.html#boltons.timeutils.strpdate [typeutils.get_all_subclasses]: http://boltons.readthedocs.org/en/latest/typeutils.html#boltons.typeutils.get_all_subclasses [typeutils.make_sentinel]: http://boltons.readthedocs.org/en/latest/typeutils.html#boltons.typeutils.make_sentinel [zscore]: https://en.wikipedia.org/wiki/Standard_score boltons-16.4.1/tox.ini0000644000175000017500000000026212727357214012712 0ustar hlehle[tox] envlist = py26,py27,py34,pypy [testenv] changedir = .tox deps = -rrequirements-test.txt commands = py.test --doctest-modules {envsitepackagesdir}/boltons {toxinidir}/tests boltons-16.4.1/TODO.rst0000644000175000017500000000251312727357214012677 0ustar hlehleTODO ==== - dummy context manager - dummy file cacheutils ---------- - counting dictionary (with sys.modules example) dictutils --------- - autoindexing list for dictionaries. As records get added, uses a basic proportion-based heuristic to create subdictionaries as indexes over the same data. Maybe automatically does a full-scan option too. - non-overwriting version of dict.update() jsonutils --------- * jsonl ignore blank lines * jsonl add line number to error message misc? ----- - wrap_trace debug utility. Takes an object, looks at its dir, wraps everything callable, with a hook. Needs an enable/disable flag. - get/set/call/return/exception - __slots__ - Tracking proxy. An object that always succeeds for all operations, saving the call history. - Top/Bottom singletons (greater than and less than everything) cliutils -------- - progress bar - confirmation prompt (e.g., "Question? (Y/n)") tbutils ------- - fold repeated frames (recursive calls) statsutils ---------- - dirty bit auto clears cache on property access - geometric mean (2 ** sum(log(a, b, ...)) funcutils --------- #class FunctionDef(object): # """ # general blocker: accept a bunch of fine-grained arguments, or just # accept a whole ArgSpec? or a whole signature? # """ # def __init__(self, name, code, doc=None): # pass boltons-16.4.1/docs/0000755000175000017500000000000012727357214012327 5ustar hlehleboltons-16.4.1/docs/mathutils.rst0000644000175000017500000000124712727357214015077 0ustar hlehle``mathutils`` - Mathematical functions ====================================== .. automodule:: boltons.mathutils Alternative Ceiling/Floor Functions ----------------------------------- .. autofunction:: boltons.mathutils.ceil .. autofunction:: boltons.mathutils.floor Note: :func:`ceil` and :func:`floor` functions are based on `this example`_ using from the :mod:`bisect` module in the standard library. Refer to this `StackOverflow Answer`_ for further information regarding the performance impact of this approach. .. _this example: https://docs.python.org/3/library/bisect.html#searching-sorted-lists .. _StackOverflow Answer: http://stackoverflow.com/a/12141511/811740 boltons-16.4.1/docs/ecoutils.rst0000644000175000017500000000016512727357214014712 0ustar hlehle``ecoutils`` - Ecosystem analytics ================================== .. automodule:: boltons.ecoutils :members: boltons-16.4.1/docs/make.bat0000644000175000017500000001611612727357214013741 0ustar hlehle@ECHO OFF REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set BUILDDIR=_build set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . set I18NSPHINXOPTS=%SPHINXOPTS% . if NOT "%PAPER%" == "" ( set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% ) if "%1" == "" goto help if "%1" == "help" ( :help echo.Please use `make ^` where ^ is one of echo. html to make standalone HTML files echo. dirhtml to make HTML files named index.html in directories echo. singlehtml to make a single large HTML file echo. pickle to make pickle files echo. json to make JSON files echo. htmlhelp to make HTML files and a HTML help project echo. qthelp to make HTML files and a qthelp project echo. devhelp to make HTML files and a Devhelp project echo. epub to make an epub echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter echo. text to make text files echo. man to make manual pages echo. texinfo to make Texinfo files echo. gettext to make PO message catalogs echo. changes to make an overview over all changed/added/deprecated items echo. xml to make Docutils-native XML files echo. pseudoxml to make pseudoxml-XML files for display purposes echo. linkcheck to check all external links for integrity echo. doctest to run all doctests embedded in the documentation if enabled echo. coverage to run coverage check of the documentation if enabled goto end ) if "%1" == "clean" ( for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i del /q /s %BUILDDIR%\* goto end ) REM Check if sphinx-build is available and fallback to Python version if any %SPHINXBUILD% 2> nul if errorlevel 9009 goto sphinx_python goto sphinx_ok :sphinx_python set SPHINXBUILD=python -m sphinx.__init__ %SPHINXBUILD% 2> nul if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.http://sphinx-doc.org/ exit /b 1 ) :sphinx_ok if "%1" == "html" ( %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/html. goto end ) if "%1" == "dirhtml" ( %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. goto end ) if "%1" == "singlehtml" ( %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. goto end ) if "%1" == "pickle" ( %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the pickle files. goto end ) if "%1" == "json" ( %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the JSON files. goto end ) if "%1" == "htmlhelp" ( %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run HTML Help Workshop with the ^ .hhp project file in %BUILDDIR%/htmlhelp. goto end ) if "%1" == "qthelp" ( %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run "qcollectiongenerator" with the ^ .qhcp project file in %BUILDDIR%/qthelp, like this: echo.^> qcollectiongenerator %BUILDDIR%\qthelp\boltons.qhcp echo.To view the help file: echo.^> assistant -collectionFile %BUILDDIR%\qthelp\boltons.ghc goto end ) if "%1" == "devhelp" ( %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp if errorlevel 1 exit /b 1 echo. echo.Build finished. goto end ) if "%1" == "epub" ( %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub if errorlevel 1 exit /b 1 echo. echo.Build finished. The epub file is in %BUILDDIR%/epub. goto end ) if "%1" == "latex" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex if errorlevel 1 exit /b 1 echo. echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. goto end ) if "%1" == "latexpdf" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex cd %BUILDDIR%/latex make all-pdf cd %~dp0 echo. echo.Build finished; the PDF files are in %BUILDDIR%/latex. goto end ) if "%1" == "latexpdfja" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex cd %BUILDDIR%/latex make all-pdf-ja cd %~dp0 echo. echo.Build finished; the PDF files are in %BUILDDIR%/latex. goto end ) if "%1" == "text" ( %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text if errorlevel 1 exit /b 1 echo. echo.Build finished. The text files are in %BUILDDIR%/text. goto end ) if "%1" == "man" ( %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man if errorlevel 1 exit /b 1 echo. echo.Build finished. The manual pages are in %BUILDDIR%/man. goto end ) if "%1" == "texinfo" ( %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo if errorlevel 1 exit /b 1 echo. echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. goto end ) if "%1" == "gettext" ( %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale if errorlevel 1 exit /b 1 echo. echo.Build finished. The message catalogs are in %BUILDDIR%/locale. goto end ) if "%1" == "changes" ( %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes if errorlevel 1 exit /b 1 echo. echo.The overview file is in %BUILDDIR%/changes. goto end ) if "%1" == "linkcheck" ( %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck if errorlevel 1 exit /b 1 echo. echo.Link check complete; look for any errors in the above output ^ or in %BUILDDIR%/linkcheck/output.txt. goto end ) if "%1" == "doctest" ( %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest if errorlevel 1 exit /b 1 echo. echo.Testing of doctests in the sources finished, look at the ^ results in %BUILDDIR%/doctest/output.txt. goto end ) if "%1" == "coverage" ( %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage if errorlevel 1 exit /b 1 echo. echo.Testing of coverage in the sources finished, look at the ^ results in %BUILDDIR%/coverage/python.txt. goto end ) if "%1" == "xml" ( %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml if errorlevel 1 exit /b 1 echo. echo.Build finished. The XML files are in %BUILDDIR%/xml. goto end ) if "%1" == "pseudoxml" ( %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml if errorlevel 1 exit /b 1 echo. echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. goto end ) :end boltons-16.4.1/docs/Makefile0000644000175000017500000001636512727357214014002 0ustar hlehle# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " applehelp to make an Apple Help Book" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" @echo " coverage to run coverage check of the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/boltons.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/boltons.qhc" applehelp: $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp @echo @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." @echo "N.B. You won't be able to view it unless you put it in" \ "~/Library/Documentation/Help or install it in your application" \ "bundle." devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/boltons" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/boltons" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." coverage: $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage @echo "Testing of coverage in the sources finished, look at the " \ "results in $(BUILDDIR)/coverage/python.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." boltons-16.4.1/docs/architecture.rst0000644000175000017500000000660412727357214015551 0ustar hlehleArchitecture ============ ``boltons`` has a minimalist architecture: remain as consistent, and self-contained as possible, with an eye toward maintaining its range of use cases and usage patterns as wide as possible. .. _arch_integration: Integration ----------- Utility libraries are often used extensively within a project, and because they are not often fundamental to the architecture of the application, simplicity and stability may take precedence over version recency. In these cases, developers can: 1. Copy the whole ``boltons`` package into a project. 2. Copy just the ``utils.py`` file that a project requires. Boltons take this into account by design. The ``boltons`` package depends on no packages, making it easy for inclusion into a project. Furthermore, virtually all individual modules have been written to be as self-contained as possible, allowing cherrypicking of functionality into projects. Design of a ``bolton`` ---------------------- ``boltons`` aims to be a living library, an ever-expanding collection of tested and true utilities. For a bolton to be a bolton, it should: 1. Be pure-Python and as self-contained as possible. 2. Perform a common task or fulfill a common role. 3. Demonstrate and mitigate some insufficiency in the standard library. 4. Strive for the standard set forth by the standard library by striking a balance between best practice and "good enough", correctness and common sense. When in doubt, ask, "what would the standard library do?" 5. Have approachable documentation with at least one helpful :mod:`doctest`, links to relevant standard library functionality, as well as any 3rd-party packages that provide further capabilities. Finally, boltons should be substantial implementations of commonly trivialized stumbling blocks and not the other way around. The larger the problem solved, the less likely the functionality is suitable for inclusion in boltons; boltons are fundamental and self-contained, not sweeping and architecture-defining. Themes of ``boltons`` --------------------- ``boltons`` has had a wide variety of inspirations over the years, but a definite set of themes have emerged: 1. From the Python docs: 1. :mod:`~boltons.queueutils` - `heapq docs`_ 2. :mod:`~boltons.iterutils` - `itertools docs`_ 3. :mod:`~boltons.timeutils` - `datetime docs`_ 2. Reimplementations and tweaks of the standard library: 1. :func:`boltons.fileutils.copytree` - :func:`shutil.copytree` 2. :class:`boltons.namedutils.namedtuple` - :func:`collections.namedtuple` 3. One-off implementations discovered in multiple other libraries' ``utils.py`` or equivalent 1. :func:`boltons.strutils.slugify` 2. :func:`boltons.strutils.bytes2human` 3. :func:`boltons.timeutils.relative_time` 4. More powerful multi-purpose data structures 1. :class:`boltons.dictutils.OrderedMultiDict` 2. :class:`boltons.setutils.IndexedSet` 3. :class:`boltons.listutils.BList` 4. :class:`boltons.namedutils.namedlist` 5. :class:`boltons.tableutils.Table` 5. Personal practice and experience 1. :mod:`boltons.debugutils` 2. :mod:`boltons.gcutils` 3. :mod:`boltons.tbutils` .. _heapq docs: https://docs.python.org/2/library/heapq.html#priority-queue-implementation-notes .. _itertools docs: https://docs.python.org/2/library/itertools.html#recipes .. _datetime docs: https://docs.python.org/2/library/datetime.html#tzinfo-objects boltons-16.4.1/docs/_templates/0000755000175000017500000000000012727357214014464 5ustar hlehleboltons-16.4.1/docs/_templates/page.html0000644000175000017500000000044612727357214016272 0ustar hlehle{% extends "!page.html" %} {% block menu %} {{ super() }} {% endblock %} boltons-16.4.1/docs/statsutils.rst0000644000175000017500000000022612727357214015300 0ustar hlehle``statsutils`` - Statistics fundamentals ======================================== .. automodule:: boltons.statsutils :members: :undoc-members: boltons-16.4.1/docs/gcutils.rst0000644000175000017500000000021712727357214014533 0ustar hlehle``gcutils`` - Garbage collecting tools ====================================== .. automodule:: boltons.gcutils :members: :undoc-members: boltons-16.4.1/docs/mboxutils.rst0000644000175000017500000000022112727357214015102 0ustar hlehle``mboxutils`` - Unix mailbox utilities ====================================== .. automodule:: boltons.mboxutils :members: :undoc-members: boltons-16.4.1/docs/queueutils.rst0000644000175000017500000000020612727357214015264 0ustar hlehle``queueutils`` - Priority queues ================================ .. automodule:: boltons.queueutils :members: :undoc-members: boltons-16.4.1/docs/conf.py0000644000175000017500000002310512727357214013627 0ustar hlehle# -*- coding: utf-8 -*- # # boltons documentation build configuration file, created by # sphinx-quickstart on Sat Mar 21 00:34:18 2015. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import os import sys import sphinx from pprint import pprint # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. CUR_PATH = os.path.dirname(os.path.abspath(__file__)) PROJECT_PATH = os.path.abspath(CUR_PATH + '/../') PACKAGE_PATH = os.path.abspath(CUR_PATH + '/../boltons/') sys.path.insert(0, PROJECT_PATH) sys.path.insert(0, PACKAGE_PATH) pprint(os.environ) def get_mod_stats(): # TODO: docstring percentage. import pkgutil from boltons.funcutils import get_module_callables mod_count = 0 tot_type_count = 0 tot_func_count = 0 ignore = lambda attr_name: attr_name.startswith('_') for _, mod_name, _ in pkgutil.iter_modules([PACKAGE_PATH]): if not mod_name.endswith('utils'): continue mod = __import__(mod_name) types, funcs = get_module_callables(mod, ignore=ignore) if not len(types) and not len(funcs): continue mod_count += 1 tot_type_count += len(types) tot_func_count += len(funcs) ret = (mod_count, tot_type_count, tot_func_count) print ('==== %s modules ==== %s types ==== %s funcs ====' % ret) return ret B_MOD_COUNT, B_TYPE_COUNT, B_FUNC_COUNT = get_mod_stats() rst_epilog = """ .. |b_mod_count| replace:: {mod_count} .. |b_type_count| replace:: {type_count} .. |b_func_count| replace:: {func_count} """.format(mod_count=B_MOD_COUNT, type_count=B_TYPE_COUNT, func_count=B_FUNC_COUNT) # -- General configuration ------------------------------------------------ autosummary_generate = True # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', ] # Read the Docs is version 1.2 as of writing if sphinx.version_info[:2] < (1, 3): extensions.append('sphinxcontrib.napoleon') else: extensions.append('sphinx.ext.napoleon') # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The master toctree document. master_doc = 'index' # General information about the project. project = u'boltons' copyright = u'2015, Mahmoud Hashemi' author = u'Mahmoud Hashemi' version = '16.3' release = '16.3.1' if os.name != 'nt': today_fmt = '%B %e, %Y' exclude_patterns = ['_build'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'python': ('https://docs.python.org/2.7', None)} # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. on_rtd = os.environ.get('READTHEDOCS', None) == 'True' if on_rtd: html_theme = 'default' else: # only import and set the theme if we're building docs locally import sphinx_rtd_theme html_theme = 'sphinx_rtd_theme' html_theme_path = ['_themes', sphinx_rtd_theme.get_html_theme_path()] # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] # TEMP: see https://github.com/rtfd/readthedocs.org/issues/1692 # Add RTD Theme Path. #if 'html_theme_path' in globals(): # html_theme_path.append('/home/docs/checkouts/readthedocs.org/readthedocs/templates/sphinx') #else: # html_theme_path = ['_themes', '/home/docs/checkouts/readthedocs.org/readthedocs/templates/sphinx'] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. #html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' #html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # Now only 'ja' uses this config value #html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. #html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. htmlhelp_basename = 'boltonsdoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', # Latex figure (float) alignment #'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'boltons.tex', u'boltons Documentation', u'Mahmoud Hashemi', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ (master_doc, 'boltons', u'boltons Documentation', [author], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'boltons', u'boltons Documentation', author, 'boltons', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False boltons-16.4.1/docs/socketutils.rst0000644000175000017500000000164412727357214015437 0ustar hlehle``socketutils`` - ``socket`` wrappers ===================================== .. automodule:: boltons.socketutils BufferedSocket -------------- .. autoclass:: boltons.socketutils.BufferedSocket :members: Exceptions ^^^^^^^^^^ These are a few exceptions that derive from :exc:`socket.error` and provide clearer code and better error messages. .. autoexception:: boltons.socketutils.Error .. autoexception:: boltons.socketutils.Timeout .. autoexception:: boltons.socketutils.ConnectionClosed .. autoexception:: boltons.socketutils.MessageTooLong Netstring --------- .. autoclass:: boltons.socketutils.NetstringSocket :members: Nestring Exceptions ^^^^^^^^^^^^^^^^^^^ These are a few higher-level exceptions for Netstring connections. .. autoexception:: boltons.socketutils.NetstringProtocolError .. autoexception:: boltons.socketutils.NetstringInvalidSize .. autoexception:: boltons.socketutils.NetstringMessageTooLong boltons-16.4.1/docs/setutils.rst0000644000175000017500000000016512727357214014737 0ustar hlehle``setutils`` - ``IndexedSet`` type ================================== .. automodule:: boltons.setutils :members: boltons-16.4.1/docs/dictutils.rst0000644000175000017500000000021312727357214015061 0ustar hlehle``dictutils`` - Mapping types (OMD) =================================== .. automodule:: boltons.dictutils :members: :undoc-members: boltons-16.4.1/docs/namedutils.rst0000644000175000017500000000022412727357214015224 0ustar hlehle``namedutils`` - Lightweight containers ======================================= .. automodule:: boltons.namedutils :members: :undoc-members: boltons-16.4.1/docs/tableutils.rst0000644000175000017500000000021212727357214015224 0ustar hlehle``tableutils`` - 2D data structure ================================== .. automodule:: boltons.tableutils :members: :undoc-members: boltons-16.4.1/docs/jsonutils.rst0000644000175000017500000000020712727357214015112 0ustar hlehle``jsonutils`` - JSON interactions ================================= .. automodule:: boltons.jsonutils :members: :undoc-members: boltons-16.4.1/docs/timeutils.rst0000644000175000017500000000276112727357214015106 0ustar hlehle``timeutils`` - ``datetime`` additions ====================================== .. automodule:: boltons.timeutils .. autofunction:: daterange .. autofunction:: isoparse .. autofunction:: parse_timedelta .. autofunction:: strpdate .. autofunction:: total_seconds .. autofunction:: dt_to_timestamp .. autofunction:: relative_time .. autofunction:: decimal_relative_time General timezones ----------------- By default, :class:`datetime.datetime` objects are "naïve", meaning they lack attached timezone information. These objects can be useful for many operations, but many operations require timezone-aware datetimes. The two most important timezones in programming are Coordinated Universal Time (`UTC`_) and the local timezone of the host running your code. Boltons provides two :class:`datetime.tzinfo` subtypes for working with them: .. _UTC: https://en.wikipedia.org/wiki/Coordinated_Universal_Time .. autoattribute:: boltons.timeutils.UTC .. autodata:: boltons.timeutils.LocalTZ .. autoclass:: boltons.timeutils.ConstantTZInfo US timezones ------------ These four US timezones were implemented in the :mod:`datetime` documentation and have been reproduced here in boltons for convenience. More in-depth support is provided by `pytz`_. .. _pytz: https://pypi.python.org/pypi/pytz .. autoattribute:: boltons.timeutils.Eastern .. autoattribute:: boltons.timeutils.Central .. autoattribute:: boltons.timeutils.Mountain .. autoattribute:: boltons.timeutils.Pacific .. autoclass:: boltons.timeutils.USTimeZone boltons-16.4.1/docs/cacheutils.rst0000644000175000017500000000341512727357214015210 0ustar hlehle``cacheutils`` - Caches and caching =================================== .. automodule:: boltons.cacheutils Least-Recently Inserted (LRI) ----------------------------- The :class:`LRI` is the simpler cache, implementing a very simple first-in, first-out (FIFO) approach to cache eviction. If the use case calls for simple, very-low overhead caching, such as somewhat expensive local operations (e.g., string operations), then the LRI is likely the right choice. .. autoclass:: boltons.cacheutils.LRI :members: Least-Recently Used (LRU) ------------------------- The :class:`LRU` is the more advanced cache, but it's still quite simple. When it reaches capacity, a new insertion replaces the least-recently used item. This strategy makes the LRU a more effective cache than the LRI for a wide variety of applications, but also entails more operations for all of its APIs, especially reads. Unlike the :class:`LRI`, the LRU has threadsafety built in. .. autoclass:: boltons.cacheutils.LRU :members: Automatic function caching -------------------------- Continuing in the theme of cache tunability and experimentation, ``cacheutils`` also offers a pluggable way to cache function return values: the :func:`cached` function decorator and the :func:`cachedmethod` method decorator. .. autofunction:: boltons.cacheutils.cached .. autofunction:: boltons.cacheutils.cachedmethod Similar functionality can be found in Python 3.4's :func:`functools.lru_cache` decorator, but the functools approach does not support the same cache strategy modification, nor does it support sharing the cache object across multiple functions. .. autofunction:: boltons.cacheutils.cachedproperty Threshold-bounded Counting -------------------------- .. autoclass:: boltons.cacheutils.ThresholdCounter :members: boltons-16.4.1/docs/formatutils.rst0000644000175000017500000000023312727357214015430 0ustar hlehle``formatutils`` - ``str.format()`` toolbox ========================================== .. automodule:: boltons.formatutils :members: :undoc-members: boltons-16.4.1/docs/debugutils.rst0000644000175000017500000000021612727357214015227 0ustar hlehle``debugutils`` - Debugging utilities ==================================== .. automodule:: boltons.debugutils :members: :undoc-members: boltons-16.4.1/docs/tbutils.rst0000644000175000017500000000020012727357214014537 0ustar hlehle``tbutils`` - Tracebacks and call stacks ======================================== .. automodule:: boltons.tbutils :members: boltons-16.4.1/docs/iterutils.rst0000644000175000017500000000430412727357214015106 0ustar hlehle``iterutils`` - ``itertools`` improvements ========================================== .. automodule:: boltons.iterutils .. _iteration: Iteration --------- These are generators and convenient :class:`list`-producing counterparts comprising several common patterns of iteration not present in the standard library. .. autofunction:: split .. autofunction:: split_iter .. autofunction:: chunked .. autofunction:: chunked_iter .. autofunction:: pairwise .. autofunction:: pairwise_iter .. autofunction:: windowed .. autofunction:: windowed_iter .. autofunction:: unique .. autofunction:: unique_iter Nested ------ Nested data structures are common. Yet virtually all of Python's compact iteration tools work with flat data: list comprehensions, map/filter, generator expressions, itertools, even other iterutils. The functions below make working with nested iterables and other containers as succinct and powerful as Python itself. .. autofunction:: remap Numeric ------- Number sequences are an obvious target of Python iteration, such as the built-in :func:`range`, :func:`xrange`, and :func:`itertools.count`. Like the :ref:`iteration` members above, these return iterators and lists, but take numeric inputs instead of iterables. .. autofunction:: backoff .. autofunction:: backoff_iter .. autofunction:: frange .. autofunction:: xfrange Categorization -------------- These functions operate on iterables, dividing into groups based on a given condition. .. autofunction:: bucketize .. autofunction:: partition Reduction --------- :func:`reduce` is a powerful function, but it is also very open-ended and not always the most readable. The standard library recognized this with the addition of :func:`sum`, :func:`all`, and :func:`any`. All these functions take a basic operator (``+``, ``and``, and ``or``) and use the operator to turn an iterable into a single value. Functions in this category follow that same spirit, turning iterables like lists into single values: .. autofunction:: one .. autofunction:: first .. autofunction:: same Type Checks ----------- In the same vein as the feature-checking builtin, :func:`callable`. .. autofunction:: is_iterable .. autofunction:: is_scalar .. autofunction:: is_collection boltons-16.4.1/docs/funcutils.rst0000644000175000017500000000173012727357214015076 0ustar hlehle``funcutils`` - ``functools`` fixes =================================== .. automodule:: boltons.funcutils Decoration ---------- `Decorators`_ are among Python's most elegant and succinct language features, and boltons adds one special function to make them even more powerful. .. _Decorators: https://en.wikipedia.org/wiki/Python_syntax_and_semantics#Decorators .. autofunction:: wraps Function construction --------------------- Functions are so key to programming in Python that there will even arise times where Python functions must be constructed in Python. Thankfully, Python is a dynamic enough to make this possible. Boltons makes it easy. .. autoclass:: FunctionBuilder :members: Improved ``partial`` -------------------- .. autoclass:: partial .. autoclass:: InstancePartial .. autoclass:: CachedInstancePartial Miscellaneous metaprogramming ----------------------------- .. autofunction:: copy_function .. autofunction:: dir_dict .. autofunction:: mro_items boltons-16.4.1/docs/strutils.rst0000644000175000017500000000020512727357214014747 0ustar hlehle``strutils`` - Text manipulation ================================= .. automodule:: boltons.strutils :members: :undoc-members: boltons-16.4.1/docs/typeutils.rst0000644000175000017500000000015412727357214015123 0ustar hlehle``typeutils`` - Type handling ============================= .. automodule:: boltons.typeutils :members: boltons-16.4.1/docs/fileutils.rst0000644000175000017500000000263012727357214015062 0ustar hlehle``fileutils`` - Filesystem helpers ================================== .. automodule:: boltons.fileutils Creating, Finding, and Copying ------------------------------ Python's :mod:`os`, :mod:`os.path`, and :mod:`shutil` modules provide good coverage of file wrangling fundaments, and these functions help close a few remaining gaps. .. autofunction:: boltons.fileutils.mkdir_p .. autofunction:: boltons.fileutils.iter_find_files .. autofunction:: boltons.fileutils.copytree Atomic File Saving ------------------ Ideally, the road to success should never put current progress at risk. That's why overwriting a file should only happen after the task at hand has completed. And that's exactly what :func:`atomic_save` and :class:`AtomicSaver` do. .. autofunction:: boltons.fileutils.atomic_save .. autoclass:: boltons.fileutils.AtomicSaver .. autofunction:: boltons.fileutils.atomic_rename .. autofunction:: boltons.fileutils.replace File Permissions ---------------- Linux, BSD, Mac OS, and other Unix-like operating systems all share a simple, foundational file permission structure that is commonly complicit in accidental access denial, as well as file leakage. :class:`FilePerms` was built to increase clarity and cut down on permission-related accidents when working with files from Python code. .. autoclass:: boltons.fileutils.FilePerms Miscellaneous ------------- .. autoclass:: boltons.fileutils.DummyFile boltons-16.4.1/docs/listutils.rst0000644000175000017500000000021512727357214015113 0ustar hlehle``listutils`` - ``list`` derivatives ==================================== .. automodule:: boltons.listutils :members: :undoc-members: boltons-16.4.1/docs/index.rst0000644000175000017500000000701612727357214014174 0ustar hlehle.. boltons documentation master file, created on Sat Mar 21 00:34:18 2015. boltons ======= *boltons should be builtins.* **Boltons** is a set of pure-Python utilities in the same spirit as — and yet conspicuously missing from — the `the standard library`_, including: * :func:`Atomic file saving `, bolted on with :mod:`~boltons.fileutils` * A highly-optimized :class:`~boltons.dictutils.OrderedMultiDict`, in :mod:`~boltons.dictutils` * Two types of :class:`~boltons.queueutils.PriorityQueue`, in :mod:`~boltons.queueutils` * :func:`Chunked ` and :func:`windowed ` iteration, in :mod:`~boltons.iterutils` * A full-featured :class:`~boltons.tbutils.TracebackInfo` type, for representing stack traces, in :mod:`~boltons.tbutils` * A lightweight :class:`UTC timezone ` available in :mod:`~boltons.timeutils`. * Recursive mapping for nested data transforms, with :func:`remap ` And that's just a small selection. As of |today|, ``boltons`` is |b_type_count| types and |b_func_count| functions, spread across |b_mod_count| modules. See what's new by `checking the CHANGELOG`_. .. counts are appx 50, 75, and 23, respectively, as of initial docs writing .. in mid 2016, the counts are now 62, 112, and 25, respectively .. _the standard library: https://docs.python.org/2.7/library/index.html .. _checking the CHANGELOG: https://github.com/mahmoud/boltons/blob/master/CHANGELOG.md Installation and Integration ---------------------------- Boltons can be added to a project in a few ways. There's the obvious one:: pip install boltons Then dozens of boltons are just an import away:: from boltons.cacheutils import LRU lru_cache = LRU() lru_cache['result'] = 'success' Due to the nature of utilities, application developers might want to consider other integration options. See the :ref:`Integration ` section of the architecture document for more details. Boltons is tested against Python 2.6, 2.7, 3.4, 3.5, and PyPy. Third-party packages -------------------- The majority of boltons strive to be "good enough" for a wide range of basic uses, leaving advanced use cases to Python's `myriad specialized 3rd-party libraries`_. In many cases the respective ``boltons`` module will describe 3rd-party alternatives worth investigating when use cases outgrow ``boltons``. If you've found a natural "next-step" library worth mentioning, :ref:`consider filing an issue `! .. _myriad specialized 3rd-party libraries: https://pypi.python.org/pypi .. _gaps: Gaps ---- Found something missing in the standard library that should be in ``boltons``? Found something missing in ``boltons``? First, take a moment to read the very brief :doc:`architecture` statement to make sure the functionality would be a good fit. Then, if you are very motivated, submit `a Pull Request`_. Otherwise, submit a short feature request on `the Issues page`_, and we will figure something out. .. _a Pull Request: https://github.com/mahmoud/boltons/pulls .. _the Issues Page: https://github.com/mahmoud/boltons/issues Section listing --------------- .. toctree:: :maxdepth: 2 architecture cacheutils debugutils dictutils ecoutils fileutils formatutils funcutils gcutils iterutils jsonutils listutils mathutils mboxutils namedutils queueutils setutils socketutils statsutils strutils tableutils tbutils timeutils typeutils boltons-16.4.1/README.md0000644000175000017500000001042712727357214012662 0ustar hlehle# Boltons *boltons should be builtins.* **Boltons** is a set of over 160 BSD-licensed, pure-Python utilities in the same spirit as — and yet conspicuously missing from — [the standard library][stdlib], including: * [Atomic file saving][atomic], bolted on with [fileutils][fileutils] * A highly-optimized [OrderedMultiDict][omd], in [dictutils][dictutils] * *Two* types of [PriorityQueue][pq], in [queueutils][queueutils] * [Chunked][chunked] and [windowed][windowed] iteration, in [iterutils][iterutils] * Recursive data structure [iteration and merging][remap], with [iterutils.remap][iterutils.remap] * Exponential backoff functionality, including jitter, through [iterutils.backoff][iterutils.backoff] * A full-featured [TracebackInfo][tbinfo] type, for representing stack traces, in [tbutils][tbutils] **[Full and extensive docs are available on Read The Docs.][rtd]** See what's new [by checking the CHANGELOG][changelog]. Boltons is tested against Python 2.6, 2.7, 3.4, 3.5, and PyPy. [stdlib]: https://docs.python.org/2.7/library/index.html [rtd]: https://boltons.readthedocs.org/en/latest/ [changelog]: https://github.com/mahmoud/boltons/blob/master/CHANGELOG.md [atomic]: https://boltons.readthedocs.org/en/latest/fileutils.html#boltons.fileutils.atomic_save [omd]: https://boltons.readthedocs.org/en/latest/dictutils.html#boltons.dictutils.OrderedMultiDict [pq]: https://boltons.readthedocs.org/en/latest/queueutils.html#boltons.queueutils.PriorityQueue [chunked]: https://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.chunked [windowed]: https://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.windowed [tbinfo]: https://boltons.readthedocs.org/en/latest/tbutils.html#boltons.tbutils.TracebackInfo [fileutils]: https://boltons.readthedocs.org/en/latest/fileutils.html#module-boltons.fileutils [dictutils]: https://boltons.readthedocs.org/en/latest/dictutils.html#module-boltons.dictutils [queueutils]: https://boltons.readthedocs.org/en/latest/queueutils.html#module-boltons.queueutils [iterutils]: https://boltons.readthedocs.org/en/latest/iterutils.html#module-boltons.iterutils [iterutils.remap]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.remap [iterutils.backoff]: http://boltons.readthedocs.org/en/latest/iterutils.html#boltons.iterutils.backoff [tbutils]: https://boltons.readthedocs.org/en/latest/tbutils.html#module-boltons.tbutils [remap]: http://sedimental.org/remap.html ## Installation Boltons can be added to a project in a few ways. There's the obvious one: ``` pip install boltons ``` Then, [thanks to PyPI][boltons_pypi], dozens of boltons are just an import away: ```python from boltons.cacheutils import LRU my_cache = LRU() ``` However, due to the nature of utilities, application developers might want to consider other options, including vendorization of individual modules into a project. Boltons is pure-Python and has no dependencies. If the whole project is too big, each module is independent, and can be copied directly into a project. See the [Integration][integration] section of the docs for more details. [boltons_pypi]: https://pypi.python.org/pypi/boltons [integration]: https://boltons.readthedocs.org/en/latest/architecture.html#integration ## Third-party packages The majority of boltons strive to be "good enough" for a wide range of basic uses, leaving advanced use cases to Python's [myriad specialized 3rd-party libraries][pypi]. In many cases the respective ``boltons`` module will describe 3rd-party alternatives worth investigating when use cases outgrow `boltons`. If you've found a natural "next-step" library worth mentioning, see the next section! [pypi]: https://pypi.python.org/pypi ## Gaps Found something missing in the standard library that should be in `boltons`? Found something missing in `boltons`? First, take a moment to read the very brief [architecture statement][architecture] to make sure the functionality would be a good fit. Then, if you are very motivated, submit [a Pull Request][prs]. Otherwise, submit a short feature request on [the Issues page][issues], and we will figure something out. [architecture]: https://boltons.readthedocs.org/en/latest/architecture.html [issues]: https://github.com/mahmoud/boltons/issues [prs]: https://github.com/mahmoud/boltons/pulls boltons-16.4.1/requirements-test.txt0000644000175000017500000000005412727357214015637 0ustar hlehlepytest==2.8.7 tox==1.9.2 virtualenv==12.1.1 boltons-16.4.1/LICENSE0000644000175000017500000000273112727357214012407 0ustar hlehleCopyright (c) 2013, Mahmoud Hashemi Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * The names of the contributors may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. boltons-16.4.1/misc/0000755000175000017500000000000012727357214012332 5ustar hlehleboltons-16.4.1/misc/bench_omd.py0000644000175000017500000000635712727357214014635 0ustar hlehle# -*- coding: utf-8 -*- import string import sys sys.path.append('/home/mahmoud/projects/lithoxyl/') import time import lithoxyl from lithoxyl import sinks, logger from dictutils import OMD, FastIterOrderedMultiDict from collections import OrderedDict as OD q_sink = lithoxyl.sinks.QuantileSink() log = lithoxyl.logger.BaseLogger('bench_stats', sinks=[q_sink]) try: profile except NameError: times = 10 size = 10000 redun = 2 else: times = 1 size = 10000 redun = 10 _rng = range(size / redun) * redun _unique_keys = set(_rng) _bad_rng = range(size, size + size) _pairs = zip(_rng, _rng) # order matters because 'pop' mutates _shared_actions = ('setitem', 'iteritems', 'iterkeys', 'getitem', 'keyerror', 'pop') _multi_actions = ('multi_iteritems',) _all_actions = ('init',) + _multi_actions + _shared_actions MULTI_IMPLS = (FastIterOrderedMultiDict, OMD) try: from werkzeug.datastructures import MultiDict, OrderedMultiDict as WOMD MULTI_IMPLS += (WOMD, MultiDict) except ImportError: print '(installing werkzeug is recommended for full comparison)' ALL_IMPLS = MULTI_IMPLS + (OD, dict) def bench(): for impl in ALL_IMPLS: q_sink = lithoxyl.sinks.QuantileSink() impl_name = '.'.join([impl.__module__, impl.__name__]) log = lithoxyl.logger.BaseLogger(impl_name, sinks=[q_sink]) print print '+ %s' % impl_name for _ in range(times): with log.info('total'): for _ in range(times): with log.info('init'): target_dict = impl(_pairs) if impl in MULTI_IMPLS: _actions = _multi_actions + _shared_actions else: _actions = _shared_actions for action in _actions: action_func = globals()['_do_' + action] with log.info(action): action_func(target_dict) for action in _all_actions: try: best_msecs = q_sink.qas[impl_name][action].min * 1000 print ' - %s - %g ms' % (action, best_msecs) except KeyError: pass best_msecs = q_sink.qas[impl_name]['total'].min * 1000 median_msecs = q_sink.qas[impl_name]['total'].median * 1000 print ' > ran %d loops of %d items each, best time: %g ms, median time: %g ms' % (times, size, best_msecs, median_msecs) print return def _do_setitem(target_dict): for k, i in enumerate(string.lowercase): target_dict[k] = i def _do_iteritems(target_dict): [_ for _ in target_dict.iteritems()] def _do_iterkeys(target_dict): [_ for _ in target_dict.iterkeys()] def _do_multi_iteritems(target_dict): [_ for _ in target_dict.iteritems(multi=True)] def _do_multi_iterkeys(target_dict): [_ for _ in target_dict.iterkeys(multi=True)] def _do_getitem(target_dict): for k in _rng: target_dict[k] def _do_keyerror(target_dict): for k in _bad_rng: try: target_dict[k] except KeyError: pass def _do_pop(target_dict): for k in _unique_keys: target_dict.pop(k) assert not target_dict if __name__ == '__main__': bench() boltons-16.4.1/misc/linkify_changelog.py0000644000175000017500000000244712727357214016367 0ustar hlehle import re import sys BASE_RTD_URL = 'http://boltons.readthedocs.org/en/latest/' BASE_ISSUES_URL = 'https://github.com/mahmoud/boltons/issues/' _issues_re = re.compile('#(\d+)') _member_re = re.compile('((\w+utils)\.[a-zA-Z0-9_.]+)') URL_MAP = {} def sub_member_match(match): full_name = match.group(1) mod_name = match.group(2) url = BASE_RTD_URL + mod_name + '.html#boltons.' + full_name ret = '[%s][%s]' % (full_name, full_name) URL_MAP[full_name] = url # print ret return ret def sub_issue_match(match): link_text = match.group(0) issue_num = match.group(1) link_target = 'i%s' % issue_num link_url = BASE_ISSUES_URL + issue_num ret = '[%s][%s]' % (link_text, link_target) URL_MAP[link_target] = link_url # print ret return ret def main(): try: cl_filename = sys.argv[1] except IndexError: cl_filename = 'CHANGELOG.md' cl_text = open(cl_filename).read().decode('utf-8') ret = _member_re.sub(sub_member_match, cl_text) ret = _issues_re.sub(sub_issue_match, ret) link_map_lines = [] for (name, url) in sorted(URL_MAP.items()): link_map_lines.append('[%s]: %s' % (name, url)) print ret print print print '\n'.join(link_map_lines) print if __name__ == '__main__': main() boltons-16.4.1/misc/table_html_app.py0000644000175000017500000001304112727357214015656 0ustar hlehle# -*- coding: utf-8 -*- import os import json import clastic from clastic import Application from clastic.render import JSONRender from clastic.middleware import GetParamMiddleware from clastic import Response from clastic.sinter import getargspec from boltons.tableutils import Table _DATA = json.load(open('meta_stats.json')) _CUR_PATH = os.path.dirname(os.path.abspath(clastic.__file__)) _CA_PATH = _CUR_PATH + '/_clastic_assets' _CSS_PATH = _CA_PATH + '/common.css' _STYLE = open(_CSS_PATH).read() def fetch_json(url): import urllib2 response = urllib2.urlopen(url) content = response.read() data = json.loads(content) return data class AutoTableRenderer(object): _html_doctype = '' _html_wrapper, _html_wrapper_close = '', '' _html_table_tag = '' _html_style_content = _STYLE def __init__(self, max_depth=4, orientation='auto'): self.max_depth = max_depth self.orientation = orientation def _html_format_ep(self, route): # TODO: callable object endpoints? module_name = route.endpoint.__module__ try: func_name = route.endpoint.func_name except: func_name = repr(route.endpoint) args, _, _, _ = getargspec(route.endpoint) argstr = ', '.join(args) title = ('

%s
%s(%s)

' % (module_name, func_name, argstr)) return title def __call__(self, context, _route): content_parts = [self._html_wrapper] if self._html_style_content: content_parts.extend(['']) content_parts.append('') title = self._html_format_ep(_route) content_parts.append(title) table = Table.from_data(context, max_depth=self.max_depth) table._html_table_tag = self._html_table_tag content = table.to_html(max_depth=self.max_depth, orientation=self.orientation) content_parts.append(content) content_parts.append('') content_parts.append(self._html_wrapper_close) return Response('\n'.join(content_parts), mimetype='text/html') class BasicRender(object): _default_mime = 'application/json' _format_mime_map = {'html': 'text/html', 'json': 'application/json'} def __init__(self, dev_mode=True, qp_name='format'): self.qp_name = qp_name self.json_render = JSONRender(dev_mode=dev_mode) self.autotable_render = AutoTableRenderer() def render_response(self, request, context, _route): from collections import Sized if isinstance(context, basestring): # already serialized if self._guess_json(context): return Response(context, mimetype="application/json") elif '