distributed-1.14.3/000077500000000000000000000000001301211143400141105ustar00rootroot00000000000000distributed-1.14.3/.coveragerc000066400000000000000000000005731301211143400162360ustar00rootroot00000000000000[run] include = distributed/* omit = distributed/tests/test* distributed/hdfs.py distributed/cluster.py distributed/*/tests/test* distributed/compatibility.py distributed/cli/utils.py distributed/utils_test.py distributed/deploy/ssh.py distributed/_ipython_utils.py [report] show_missing = True [html] directory = coverage_html_report distributed-1.14.3/.gitignore000066400000000000000000000000271301211143400160770ustar00rootroot00000000000000*.pyc *.py~ docs/build distributed-1.14.3/.travis.yml000066400000000000000000000045051301211143400162250ustar00rootroot00000000000000language: python sudo: false services: - docker matrix: fast_finish: true include: - python: "2.7" - python: "3.4" - python: "3.5" env: HDFS=false - python: "2.7" env: HDFS=true sudo: true dist: trusty - python: "3.5" env: HDFS=true sudo: true dist: trusty before_install: - | if [[ $HDFS == true ]]; then if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then export DOCKER_ENV=/opt/conda; else export DOCKER_ENV=/opt/conda/envs/py3; fi echo $DOCKER_ENV pwd pushd continuous_integration docker build -t distributed-hdfs . popd docker run -d -p 8020:8020 -p 50070:50070 -v $(pwd):/distributed distributed-hdfs export CONTAINER_ID=$(docker ps -l -q) sleep 60 # Wait for namenode and datanode fi; install: # Install conda - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda config --set always_yes yes --set changeps1 no - conda update conda # Install dependencies - conda create -n test-environment python=$TRAVIS_PYTHON_VERSION - source activate test-environment - conda install pytest coverage tornado toolz dill futures dask ipywidgets psutil bokeh requests joblib mock ipykernel jupyter_client h5py netcdf4 - pip install git+https://github.com/dask/dask.git --upgrade - pip install git+https://github.com/joblib/joblib.git --upgrade - pip install git+https://github.com/dask/s3fs.git --upgrade # Install distributed - python setup.py install - | if [[ $HDFS == true ]]; then pwd docker exec -it $CONTAINER_ID $DOCKER_ENV/bin/python setup.py install fi; script: - | if [[ $HDFS == true ]]; then pwd docker exec -it $CONTAINER_ID $DOCKER_ENV/bin/py.test distributed/tests/test_hdfs.py --verbose elif [[ $TRAVIS_PYTHON_VERSION == '3.4' ]]; then coverage run $(which py.test) distributed -m "not avoid_travis" --verbose; else py.test -m "not avoid_travis" distributed --verbose; fi; after_success: - if [[ $TRAVIS_PYTHON_VERSION == '3.4' ]]; then coverage report; pip install coveralls ; coveralls ; fi notifications: email: false distributed-1.14.3/AUTHORS.md000066400000000000000000000001501301211143400155530ustar00rootroot00000000000000[Matthew Rocklin](http://matthewrocklin.com) [Michael Broxton](http://graphics.stanford.edu/~broxton/) distributed-1.14.3/LICENSE.txt000066400000000000000000000027461301211143400157440ustar00rootroot00000000000000Copyright (c) 2015-2016, Continuum Analytics, Inc. and contributors All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. Neither the name of Continuum Analytics nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. distributed-1.14.3/MANIFEST.in000066400000000000000000000004431301211143400156470ustar00rootroot00000000000000recursive-include distributed *.py recursive-include distributed *.js recursive-include distributed *.coffee recursive-include docs *.rst include setup.py include README.rst include LICENSE.txt include MANIFEST.in include requirements.txt include distributed/config.yaml prune docs/_build distributed-1.14.3/README.rst000066400000000000000000000002471301211143400156020ustar00rootroot00000000000000Distributed =========== A library for distributed computation. See documentation_ for more details. .. _documentation: https://distributed.readthedocs.io/en/latest distributed-1.14.3/conftest.py000066400000000000000000000003461301211143400163120ustar00rootroot00000000000000# https://pytest.org/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option import pytest def pytest_addoption(parser): parser.addoption("--runslow", action="store_true", help="run slow tests") distributed-1.14.3/continuous_integration/000077500000000000000000000000001301211143400207215ustar00rootroot00000000000000distributed-1.14.3/continuous_integration/Dockerfile000066400000000000000000000037351301211143400227230ustar00rootroot00000000000000FROM ubuntu:trusty # conda RUN apt-get update && apt-get install -y -q curl bzip2 git RUN curl https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -o /tmp/miniconda.sh RUN /bin/bash /tmp/miniconda.sh -b -p /opt/conda RUN rm /tmp/miniconda.sh ENV PATH /opt/conda/bin:$PATH # hdfs3 - python 2 RUN apt-get install -y -q protobuf-compiler libprotobuf-dev ENV LIBHDFS3_CONF /etc/hadoop/conf/hdfs-site.xml RUN /opt/conda/bin/conda install -y -q libxml2 krb5 boost ipython pytest pip pandas cython mock RUN /opt/conda/bin/conda install -y -q libhdfs3 libgsasl libntlm -c dask RUN /opt/conda/bin/pip install git+https://github.com/dask/hdfs3 --upgrade RUN /opt/conda/bin/pip install git+https://github.com/dask/s3fs --upgrade RUN /opt/conda/bin/pip install git+https://github.com/blaze/dask --upgrade # hdfs3 - python 3 RUN conda create -n py3 -y python=3 RUN conda install -n py3 -y -q libxml2 krb5 boost ipython pytest pip pandas cython mock RUN conda install -n py3 libhdfs3 libgsasl libntlm -c dask RUN /opt/conda/envs/py3/bin/pip install git+https://github.com/dask/hdfs3 --upgrade RUN /opt/conda/envs/py3/bin/pip install git+https://github.com/dask/s3fs --upgrade RUN /opt/conda/envs/py3/bin/pip install git+https://github.com/blaze/dask --upgrade # Cloudera repositories RUN curl -s http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh/archive.key | apt-key add - RUN echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh5 contrib' > /etc/apt/sources.list.d/cloudera.list RUN echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh5 contrib' >> /etc/apt/sources.list.d/cloudera.list ADD docker-files/cloudera.pref /etc/apt/preferences.d/cloudera.pref # Install CDH5 in a single node: Pseudo Distributed ADD docker-files/cdh5-install.sh /tmp/cdh5-install.sh RUN bash /tmp/cdh5-install.sh EXPOSE 8020 EXPOSE 50070 VOLUME /distributed WORKDIR /distributed ADD docker-files/start.sh /tmp/start.sh CMD ["bash", "/tmp/start.sh"] distributed-1.14.3/continuous_integration/README.md000066400000000000000000000015151301211143400222020ustar00rootroot00000000000000# Continuous Integration ## Local test Requirements: - `docker` Build the container: ```bash docker build -t distributed-hdfs . ``` Start the container and wait for the it to be ready: ```bash docker run -it -p 8020:8020 -p 50070:50070 -v $(pwd):/distributed distributed-hdfs ``` Now the port `8020` and `50070` are in the host are pointing to the container and the source code (as a shared volume) is available in the container under `/distributed` Run the following from the root of this project directory to start a bash session in the running container: ```bash # Get the container ID export CONTAINER_ID=$(docker ps -l -q) # Start the bash session docker exec -it $CONTAINER_ID bash ``` Now that we are in the container we can install the library and run the test: ```bash python setup.py install py.test distributed -s -vv ``` distributed-1.14.3/continuous_integration/docker-files/000077500000000000000000000000001301211143400232705ustar00rootroot00000000000000distributed-1.14.3/continuous_integration/docker-files/cdh5-install.sh000066400000000000000000000012231301211143400261110ustar00rootroot00000000000000# Install CDH5 in a single node: Pseudo Distributed # Docs: http://www.cloudera.com/content/www/en-us/documentation/enterprise/latest/topics/cdh_qs_yarn_pseudo.html apt-get update && apt-get install -y -q openjdk-7-jre-headless hadoop-conf-pseudo # Step 1: Format the NameNode sudo -u hdfs hdfs namenode -format -force # Step 2: Start HDFS for x in `cd /etc/init.d ; ls hadoop-hdfs-*` ; do sudo service $x start ; done # Step 3: Create the directories needed for Hadoop processes bash /usr/lib/hadoop/libexec/init-hdfs.sh # Step 6: Create User Directories # sudo -u hdfs hdfs dfs -mkdir -p /user/hadoop # sudo -u hdfs hdfs dfs -chown hadoop /user/hadoop distributed-1.14.3/continuous_integration/docker-files/cloudera.pref000066400000000000000000000001011301211143400257340ustar00rootroot00000000000000Package: * Pin: release o=Cloudera, l=Cloudera Pin-Priority: 501 distributed-1.14.3/continuous_integration/docker-files/libhdfs-build.sh000066400000000000000000000001421301211143400263310ustar00rootroot00000000000000cd /opt/libhdfs3 mkdir build pushd build ../bootstrap --prefix=/usr/local make make install popd distributed-1.14.3/continuous_integration/docker-files/start.sh000066400000000000000000000002201301211143400247530ustar00rootroot00000000000000#!/bin/bash # Start HDFS sudo service hadoop-hdfs-namenode start sudo service hadoop-hdfs-datanode start echo "Ready" # Block sleep infinity distributed-1.14.3/distributed/000077500000000000000000000000001301211143400164325ustar00rootroot00000000000000distributed-1.14.3/distributed/__init__.py000066400000000000000000000010531301211143400205420ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from .config import config from .core import connect, read, write, rpc from .deploy import LocalCluster from .diagnostics import progress from .client import (Client, Executor, CompatibleExecutor, wait, as_completed, default_client) from .nanny import Nanny from .scheduler import Scheduler from .utils import sync from .worker import Worker from .worker_client import local_client try: from .collections import futures_to_collection except: pass __version__ = '1.14.3' distributed-1.14.3/distributed/_ipython_utils.py000066400000000000000000000173411301211143400220630ustar00rootroot00000000000000"""Utilities for integrating with IPython These functions should probably reside in Jupyter and IPython repositories, after which we can import them instead of having our own definitions. """ from __future__ import print_function import atexit import os try: import queue except ImportError: # Python 2 import Queue as queue from subprocess import Popen import sys from threading import Thread from uuid import uuid4 from tornado.gen import TimeoutError from tornado.ioloop import IOLoop from threading import Event from IPython import get_ipython from jupyter_client import BlockingKernelClient, write_connection_file from jupyter_core.paths import jupyter_runtime_dir OUTPUT_TIMEOUT = 10 def run_cell_remote(ip, kc, cell): """Run a cell on a KernelClient Any output from the cell will be redisplayed in the local session. """ msg_id = kc.execute(cell) in_kernel = getattr(ip, 'kernel', False) if in_kernel: socket = ip.display_pub.pub_socket session = ip.display_pub.session parent_header = ip.display_pub.parent_header while True: try: msg = kc.get_iopub_msg(timeout=OUTPUT_TIMEOUT) except queue.Empty: raise TimeoutError("Timeout waiting for IPython output") if msg['parent_header'].get('msg_id') != msg_id: continue msg_type = msg['header']['msg_type'] content = msg['content'] if msg_type == 'status': if content['execution_state'] == 'idle': # idle means output is done break elif msg_type == 'stream': stream = getattr(sys, content['name']) stream.write(content['text']) elif msg_type in ('display_data', 'execute_result', 'error'): if in_kernel: session.send(socket, msg_type, content, parent=parent_header) else: if msg_type == 'error': print('\n'.join(content['traceback']), file=sys.stderr) else: sys.stdout.write(content['data'].get('text/plain', '')) else: pass def register_worker_magic(connection_info, magic_name='worker'): """Register a %worker magic, given connection_info. Both a line and cell magic are registered, which run the given cell in a remote kernel. """ ip = get_ipython() info = dict(connection_info) # copy key = info.pop('key') kc = BlockingKernelClient(**connection_info) kc.session.key = key kc.start_channels() def remote(line, cell=None): """Run the current cell on a remote IPython kernel""" if cell is None: # both line and cell magic cell = line run_cell_remote(ip, kc, cell) remote.client = kc # preserve reference on kc, largely for mocking ip.register_magic_function(remote, magic_kind='line', magic_name=magic_name) ip.register_magic_function(remote, magic_kind='cell', magic_name=magic_name) def remote_magic(line, cell=None): """A magic for running code on a specified remote worker The connection_info dict of the worker will be looked up as the first positional arg to the magic. The rest of the line (or the entire cell for a %%cell magic) will be passed to the remote kernel. Usage: info = e.start_ipython(worker)[worker] %remote info print(worker.data) """ # get connection info from IPython's user namespace ip = get_ipython() split_line = line.split(None, 1) info_name = split_line[0] if info_name not in ip.user_ns: raise NameError(info_name) connection_info = dict(ip.user_ns[info_name]) if not cell: # line magic, use the rest of the line if len(split_line) == 1: raise ValueError("I need some code to run!") cell = split_line[1] # turn info dict to hashable str for use as lookup key in _clients cache key = ','.join(map(str, sorted(connection_info.items()))) session_key = connection_info.pop('key') if key in remote_magic._clients: kc = remote_magic._clients[key] else: kc = BlockingKernelClient(**connection_info) kc.session.key = session_key kc.start_channels() kc.wait_for_ready(timeout=10) remote_magic._clients[key] = kc # actually run the code run_cell_remote(ip, kc, cell) # cache clients for re-use in remote magic remote_magic._clients = {} def register_remote_magic(magic_name='remote'): """Define the parameterized %remote magic See remote_magic above for details. """ ip = get_ipython() if ip is None: return # do nothing if IPython's not running ip.register_magic_function(remote_magic, magic_kind='line', magic_name=magic_name) ip.register_magic_function(remote_magic, magic_kind='cell', magic_name=magic_name) def connect_qtconsole(connection_info, name=None, extra_args=None): """Open a QtConsole connected to a worker who has the given future - identify worker with who_has - start IPython kernel on the worker - start qtconsole connected to the kernel """ runtime_dir = jupyter_runtime_dir() if name is None: name = uuid4().hex path = os.path.join(runtime_dir, name + '.json') write_connection_file(path, **connection_info) cmd = ['jupyter', 'qtconsole', '--existing', path] if extra_args: cmd.extend(extra_args) Popen(cmd) def _cleanup_connection_file(): """Cleanup our connection file when we exit.""" try: os.remove(path) except OSError: pass atexit.register(_cleanup_connection_file) def start_ipython(ip=None, ns=None, log=None): """Start an IPython kernel in a thread Parameters ---------- ip: str The IP address to listen on (likely the parent object's ip). ns: dict Any names that should be injected into the IPython namespace. log: logger instance Hook up IPython's logging to an existing logger instead of the default. """ from IPython import get_ipython if get_ipython() is not None: raise RuntimeError("Cannot start IPython, it's already running.") from zmq.eventloop.ioloop import ZMQIOLoop from ipykernel.kernelapp import IPKernelApp # save the global IOLoop instance # since IPython relies on it, but we are going to put it in a thread. save_inst = IOLoop.instance() IOLoop.clear_instance() zmq_loop = ZMQIOLoop() zmq_loop.install() # start IPython, disabling its signal handlers that won't work due to running in a thread: app = IPKernelApp.instance(log=log) # Don't connect to the history database app.config.HistoryManager.hist_file = ':memory:' # listen on all interfaces, so remote clients can connect: if ip: app.ip = ip # disable some signal handling, logging noop = lambda : None app.init_signal = noop app.log_connection_info = noop # start IPython in a thread # initialization happens in the thread to avoid threading problems # with the sqlite history evt = Event() def _start(): app.initialize([]) app.kernel.pre_handler_hook = noop app.kernel.post_handler_hook = noop app.kernel.start() app.kernel.loop = IOLoop.instance() # save self in the IPython namespace as 'worker' # inject things into the IPython namespace if ns: app.kernel.shell.user_ns.update(ns) evt.set() zmq_loop.start() zmq_loop_thread = Thread(target=_start) zmq_loop_thread.daemon = True zmq_loop_thread.start() assert evt.wait(timeout=5), "IPython didn't start in a reasonable amount of time." # put the global IOLoop instance back: IOLoop.clear_instance() save_inst.install() return app distributed-1.14.3/distributed/batched.py000066400000000000000000000145431301211143400204050ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from datetime import timedelta import logging from timeit import default_timer from tornado import gen from tornado.queues import Queue from tornado.iostream import StreamClosedError from tornado.ioloop import PeriodicCallback, IOLoop from .core import read, write from .utils import log_errors logger = logging.getLogger(__name__) class BatchedSend(object): """ Batch messages in batches on a stream This takes an IOStream and an interval (in ms) and ensures that we send no more than one message every interval milliseconds. We send lists of messages. Example ------- >>> stream = yield connect(ip, port) >>> bstream = BatchedSend(interval=10) # 10 ms >>> bstream.start(stream) >>> bstream.send('Hello,') >>> bstream.send('world!') On the other side, the recipient will get a message like the following:: ['Hello,', 'world!'] """ def __init__(self, interval, loop=None): self.loop = loop or IOLoop.current() self.interval = interval / 1000. self.last_transmission = 0 self.buffer = [] self.stream = None self.last_payload = [] self.last_send = gen.sleep(0) self.message_count = 0 self.batch_count = 0 def start(self, stream): self.stream = stream if self.buffer: self.send_next() def __str__(self): return '' % len(self.buffer) __repr__ = __str__ @gen.coroutine def send_next(self, wait=True): try: now = default_timer() if wait: wait_time = min(self.last_transmission + self.interval - now, self.interval) yield gen.sleep(wait_time) yield self.last_send self.buffer, payload = [], self.buffer self.last_payload = payload self.last_transmission = now self.batch_count += 1 self.last_send = write(self.stream, payload) except Exception as e: logger.exception(e) raise @gen.coroutine def _write(self, payload): yield gen.sleep(0) yield write(self.stream, payload) def send(self, msg): """ Send a message to the other side This completes quickly and synchronously """ try: self.message_count += 1 if self.stream is None: # not yet started self.buffer.append(msg) return if self.stream._closed: raise StreamClosedError() if self.buffer: self.buffer.append(msg) return # If we're new and early, now = default_timer() if (now < self.last_transmission + self.interval or not self.last_send._done): self.buffer.append(msg) self.loop.add_callback(self.send_next) return self.buffer.append(msg) self.loop.add_callback(self.send_next, wait=False) except StreamClosedError: raise except Exception as e: logger.exception(e) @gen.coroutine def close(self, ignore_closed=False): """ Flush existing messages and then close stream """ try: if self.stream._write_buffer: yield self.last_send if self.buffer: self.buffer, payload = [], self.buffer yield write(self.stream, payload) except StreamClosedError: if not ignore_closed: raise self.stream.close() class BatchedStream(object): """ Mostly obsolete, see BatchedSend """ def __init__(self, stream, interval): self.stream = stream self.interval = interval / 1000. self.last_transmission = default_timer() self.send_q = Queue() self.recv_q = Queue() self._background_send_coroutine = self._background_send() self._background_recv_coroutine = self._background_recv() self._broken = None self.pc = PeriodicCallback(lambda: None, 100) self.pc.start() @gen.coroutine def _background_send(self): with log_errors(): while True: msg = yield self.send_q.get() if msg == 'close': break msgs = [msg] now = default_timer() wait_time = self.last_transmission + self.interval - now if wait_time > 0: yield gen.sleep(wait_time) while not self.send_q.empty(): msgs.append(self.send_q.get_nowait()) try: yield write(self.stream, msgs) except StreamClosedError: self.recv_q.put_nowait('close') self._broken = True break if len(msgs) > 1: logger.debug("Batched messages: %d", len(msgs)) for _ in msgs: self.send_q.task_done() @gen.coroutine def _background_recv(self): with log_errors(): while True: try: msgs = yield read(self.stream) except StreamClosedError: self.recv_q.put_nowait('close') self.send_q.put_nowait('close') self._broken = True break assert isinstance(msgs, list) if len(msgs) > 1: logger.debug("Batched messages: %d", len(msgs)) for msg in msgs: self.recv_q.put_nowait(msg) @gen.coroutine def flush(self): yield self.send_q.join() @gen.coroutine def send(self, msg): if self._broken: raise StreamClosedError('Batch Stream is Closed') else: self.send_q.put_nowait(msg) @gen.coroutine def recv(self): result = yield self.recv_q.get() if result == 'close': raise StreamClosedError('Batched Stream is Closed') else: raise gen.Return(result) @gen.coroutine def close(self): yield self.flush() raise gen.Return(self.stream.close()) def closed(self): return self.stream.closed() distributed-1.14.3/distributed/bokeh/000077500000000000000000000000001301211143400175225ustar00rootroot00000000000000distributed-1.14.3/distributed/bokeh/__init__.py000066400000000000000000000023141301211143400216330ustar00rootroot00000000000000from collections import deque from time import time n = 60 m = 100000 messages = { 'workers': {'interval': 1000, 'deque': deque(maxlen=n), 'times': deque(maxlen=n), 'index': deque(maxlen=n), 'plot-data': {'time': deque(maxlen=n), 'cpu': deque(maxlen=n), 'memory_percent': deque(maxlen=n), 'network-send': deque(maxlen=n), 'network-recv': deque(maxlen=n)}}, 'tasks': {'interval': 150, 'deque': deque(maxlen=100), 'times': deque(maxlen=100)}, 'progress': {}, 'processing': {'stacks': {}, 'processing': {}, 'memory': 0, 'waiting': 0, 'ready': 0}, 'task-events': {'interval': 200, 'deque': deque(maxlen=m), 'times': deque(maxlen=m), 'index': deque(maxlen=m), 'rectangles':{name: deque(maxlen=m) for name in 'start duration key name color worker worker_thread y alpha'.split()}, 'workers': dict(), 'last_seen': [time()]} } distributed-1.14.3/distributed/bokeh/application.py000066400000000000000000000063211301211143400224010ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import atexit import json import logging import multiprocessing import os import socket import sys import bokeh import distributed.bokeh from toolz import get_in, concat from ..utils import ignoring from ..compatibility import logging_names from ..config import config dirname = os.path.dirname(distributed.__file__) paths = [os.path.join(dirname, 'bokeh', name) for name in ['background', 'status', 'tasks', 'workers', 'memory-usage.py', 'task-stream.py', 'task-progress.py', 'resource-profiles.py', 'worker-table.py', 'processing-stacks.py']] logger = logging.getLogger(__name__) dask_dir = os.path.join(os.path.expanduser('~'), '.dask') if not os.path.exists(dask_dir): os.mkdir(dask_dir) logging_level = get_in(['logging', 'bokeh'], config, 'critical') logging_level = logging_names[logging_level.upper()] class BokehWebInterface(object): def __init__(self, host='127.0.0.1', http_port=9786, tcp_port=8786, bokeh_port=8787, bokeh_whitelist=[], log_level=logging_level, show=False, prefix=None, use_xheaders=False, quiet=True): self.port = bokeh_port ip = socket.gethostbyname(host) hosts = ['localhost', '127.0.0.1', ip, host] with ignoring(Exception): hosts.append(socket.gethostbyname(ip)) with ignoring(Exception): hosts.append(socket.gethostbyname(socket.gethostname())) hosts = ['%s:%d' % (h, bokeh_port) for h in hosts] hosts.append("*") hosts.extend(map(str, bokeh_whitelist)) args = ([sys.executable, '-m', 'bokeh', 'serve'] + paths + ['--check-unused-sessions=50', '--unused-session-lifetime=1', '--port', str(bokeh_port)] + sum([['--host', h] for h in hosts], [])) if prefix: args.extend(['--prefix', prefix]) if show: args.append('--show') if use_xheaders: args.append('--use-xheaders') if log_level in ('debug', 'info', 'warning', 'error', 'critical'): args.extend(['--log-level', log_level]) bokeh_options = {'host': host, 'http-port': http_port, 'tcp-port': tcp_port, 'bokeh-port': bokeh_port} args.extend(['--args'] + list(map(str, concat(bokeh_options.items())))) import subprocess process = subprocess.Popen(args) self.process = process def cleanup_process(): try: process.terminate() except OSError: pass atexit.register(cleanup_process) if not quiet: logger.info("Web UI: http://%s:%d/status/" % (ip, bokeh_port)) def close(self, join=True, timeout=None): if self.process.poll() is None: self.process.terminate() def __del__(self): self.close() def __enter__(self): return self def __exit__(self, *args): self.close() def bokeh_main(args): from bokeh.command.bootstrap import main main(args) distributed-1.14.3/distributed/bokeh/background/000077500000000000000000000000001301211143400216415ustar00rootroot00000000000000distributed-1.14.3/distributed/bokeh/background/__init__.py000066400000000000000000000000001301211143400237400ustar00rootroot00000000000000distributed-1.14.3/distributed/bokeh/background/main.py000066400000000000000000000000001301211143400231250ustar00rootroot00000000000000distributed-1.14.3/distributed/bokeh/background/server_lifecycle.py000066400000000000000000000107111301211143400255400ustar00rootroot00000000000000#!/usr/bin/env python from __future__ import print_function, division, absolute_import from collections import deque import json import logging import os import sys from time import time from tornado import gen from tornado.httpclient import AsyncHTTPClient, HTTPError from tornado.iostream import StreamClosedError from tornado.ioloop import IOLoop from distributed.compatibility import ConnectionRefusedError from distributed.core import read, connect, write from distributed.protocol.pickle import dumps from distributed.diagnostics.eventstream import eventstream from distributed.diagnostics.progress_stream import (progress_stream, task_stream_append) from distributed.bokeh.worker_monitor import resource_append import distributed.bokeh from distributed.bokeh.utils import parse_args from distributed.utils import log_errors logger = logging.getLogger(__name__) client = AsyncHTTPClient() messages = distributed.bokeh.messages # monkey-patching options = parse_args(sys.argv[1:]) @gen.coroutine def http_get(route): """ Get data from JSON route, store in messages deques """ try: url = 'http://%(host)s:%(http-port)d/' % options + route + '.json' response = yield client.fetch(url) except ConnectionRefusedError as e: logger.info("Can not connect to %s", url, exc_info=True) return except HTTPError: logger.warn("http route %s failed", route) return msg = json.loads(response.body.decode()) messages[route]['deque'].append(msg) messages[route]['times'].append(time()) last_index = [0] @gen.coroutine def workers(): """ Get data from JSON route, store in messages deques """ try: response = yield client.fetch( 'http://%(host)s:%(http-port)d/workers.json' % options) except HTTPError: logger.warn("workers http route failed") return msg = json.loads(response.body.decode()) if msg: messages['workers']['deque'].append(msg) messages['workers']['times'].append(time()) resource_append(messages['workers']['plot-data'], msg) index = messages['workers']['index'] index.append(last_index[0] + 1) last_index[0] += 1 @gen.coroutine def progress(): with log_errors(): stream = yield progress_stream('%(host)s:%(tcp-port)d' % options, 0.050) while True: try: msg = yield read(stream) except StreamClosedError: break else: messages['progress'] = msg @gen.coroutine def processing(): with log_errors(): from distributed.diagnostics.scheduler import processing stream = yield connect(ip=options['host'], port=options['tcp-port']) yield write(stream, {'op': 'feed', 'function': dumps(processing), 'interval': 0.200}) while True: try: msg = yield read(stream) except StreamClosedError: break else: messages['processing'] = msg @gen.coroutine def task_events(interval, deque, times, index, rectangles, workers, last_seen): i = 0 try: stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100) while True: msgs = yield read(stream) if not msgs: continue last_seen[0] = time() for msg in msgs: if 'compute_start' in msg: deque.append(msg) times.append(msg['compute_start']) index.append(i) i += 1 if msg.get('transfer_start') is not None: index.append(i) i += 1 if msg.get('disk_load_start') is not None: index.append(i) i += 1 task_stream_append(rectangles, msg, workers) except StreamClosedError: pass # don't log StreamClosedErrors except Exception as e: logger.exception(e) finally: try: sys.exit(0) except: pass def on_server_loaded(server_context): server_context.add_periodic_callback(workers, 500) server_context.add_periodic_callback(lambda: http_get('tasks'), 100) IOLoop.current().add_callback(processing) IOLoop.current().add_callback(progress) IOLoop.current().add_callback(task_events, **messages['task-events']) distributed-1.14.3/distributed/bokeh/components.py000066400000000000000000000505471301211143400222740ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from bisect import bisect from toolz import valmap from bokeh.layouts import row, column from bokeh.models import ( ColumnDataSource, Plot, Datetime, DataRange1d, Rect, LinearAxis, DatetimeAxis, Grid, BasicTicker, HoverTool, BoxZoomTool, ResetTool, PanTool, WheelZoomTool, Title, Range1d, Quad, Text, value, Line, NumeralTickFormatter, ToolbarBox, Legend, LegendItem, BoxSelectTool, Circle ) from bokeh.models.widgets import DataTable, TableColumn, NumberFormatter from bokeh.palettes import Spectral9 from bokeh.plotting import figure from distributed.diagnostics.progress_stream import progress_quads, nbytes_bar from distributed.utils import log_errors # from .export_tool import ExportTool class DashboardComponent(object): """ Base class for Dask.distributed UI dashboard components. This class must have two attributes, ``root`` and ``source``, and one method ``update``: * source: a Bokeh ColumnDataSource * root: a Bokeh Model * update: a method that consumes the messages dictionary found in distributed.bokeh.messages """ def __init__(self): self.source = None self.root = None def update(self, messages): """ Reads from bokeh.distributed.messages and updates self.source """ pass class TaskStream(DashboardComponent): """ Task Stream The start and stop time of tasks as they occur on each core of the cluster. """ def __init__(self, n_rectangles=1000, clear_interval=20000, **kwargs): """ kwargs are applied to the bokeh.models.plots.Plot constructor """ self.n_rectangles = n_rectangles self.clear_interval = clear_interval self.source = ColumnDataSource(data=dict( start=[], duration=[], key=[], name=[], color=[], worker=[], y=[], worker_thread=[], alpha=[]) ) x_range = DataRange1d() y_range = DataRange1d(range_padding=0) self.root = Plot( title=Title(text="Task Stream"), id='bk-task-stream-plot', x_range=x_range, y_range=y_range, toolbar_location="above", min_border_right=35, **kwargs ) self.root.add_glyph( self.source, Rect(x="start", y="y", width="duration", height=0.8, fill_color="color", line_color="color", line_alpha=0.6, fill_alpha="alpha", line_width=3) ) self.root.add_layout(DatetimeAxis(axis_label="Time"), "below") ticker = BasicTicker(num_minor_ticks=0) self.root.add_layout(LinearAxis(axis_label="Worker Core", ticker=ticker), "left") self.root.add_layout(Grid(dimension=1, grid_line_alpha=0.4, ticker=ticker)) hover = HoverTool( point_policy="follow_mouse", tooltips="""
@name:  @duration ms 
""" ) # export = ExportTool() # export.register_plot(self.root) self.root.add_tools( hover, # export, ResetTool(reset_size=False), PanTool(dimensions="width"), WheelZoomTool(dimensions="width") ) # Required for update callback self.task_stream_index = [0] def update(self, messages): with log_errors(): index = messages['task-events']['index'] old = rectangles = messages['task-events']['rectangles'] if not index or index[-1] == self.task_stream_index[0]: return ind = bisect(index, self.task_stream_index[0]) rectangles = {k: [v[i] for i in range(ind, len(index))] for k, v in rectangles.items()} self.task_stream_index[0] = index[-1] # If there has been a significant delay then clear old rectangles if rectangles['start']: last_end = old['start'][ind - 1] + old['duration'][ind - 1] if min(rectangles['start']) > last_end + self.clear_interval: self.source.data.update(rectangles) return self.source.stream(rectangles, self.n_rectangles) class TaskProgress(DashboardComponent): """ Progress bars per task type """ def __init__(self, **kwargs): data = progress_quads(dict(all={}, memory={}, erred={}, released={})) self.source = ColumnDataSource(data=data) x_range = DataRange1d() y_range = Range1d(-8, 0) self.root = Plot( id='bk-task-progress-plot', x_range=x_range, y_range=y_range, toolbar_location=None, **kwargs ) self.root.add_glyph( self.source, Quad(top='top', bottom='bottom', left='left', right='right', fill_color="#aaaaaa", line_color="#aaaaaa", fill_alpha=0.2) ) self.root.add_glyph( self.source, Quad(top='top', bottom='bottom', left='left', right='released-loc', fill_color="color", line_color="color", fill_alpha=0.6) ) self.root.add_glyph( self.source, Quad(top='top', bottom='bottom', left='released-loc', right='memory-loc', fill_color="color", line_color="color", fill_alpha=1.0) ) self.root.add_glyph( self.source, Quad(top='top', bottom='bottom', left='erred-loc', right='erred-loc', fill_color='#000000', line_color='#000000', fill_alpha=0.3) ) self.root.add_glyph( self.source, Text(text='show-name', y='bottom', x='left', x_offset=5, text_font_size=value('10pt')) ) self.root.add_glyph( self.source, Text(text='done', y='bottom', x='right', x_offset=-5, text_align='right', text_font_size=value('10pt')) ) hover = HoverTool( point_policy="follow_mouse", tooltips="""
Name:  @name
All:  @all
Memory:  @memory
Erred:  @erred
""" ) self.root.add_tools(hover) def update(self, messages): with log_errors(): msg = messages['progress'] if not msg: return d = progress_quads(msg) self.source.data.update(d) if messages['tasks']['deque']: self.root.title.text = ("Progress -- total: %(total)s, " "in-memory: %(in-memory)s, processing: %(processing)s, " "ready: %(ready)s, waiting: %(waiting)s, failed: %(failed)s" % messages['tasks']['deque'][-1]) class MemoryUsage(DashboardComponent): """ The memory usage across the cluster, grouped by task type """ def __init__(self, **kwargs): self.source = ColumnDataSource(data=dict( name=[], left=[], right=[], center=[], color=[], percent=[], MB=[], text=[]) ) self.root = Plot( id='bk-nbytes-plot', x_range=DataRange1d(), y_range=DataRange1d(), toolbar_location=None, outline_line_color=None, **kwargs ) self.root.add_glyph( self.source, Quad(top=1, bottom=0, left='left', right='right', fill_color='color', fill_alpha=1) ) self.root.add_layout(LinearAxis(), 'left') self.root.add_layout(LinearAxis(), 'below') hover = HoverTool( point_policy="follow_mouse", tooltips="""
Name:  @name
Percent:  @percent
MB:  @MB
""" ) self.root.add_tools(hover) def update(self, messages): with log_errors(): msg = messages['progress'] if not msg: return nb = nbytes_bar(msg['nbytes']) self.source.data.update(nb) self.root.title.text = \ "Memory Use: %0.2f MB" % (sum(msg['nbytes'].values()) / 1e6) class ResourceProfiles(DashboardComponent): """ Time plots of the current resource usage on the cluster This is two plots, one for CPU and Memory and another for Network I/O """ def __init__(self, **kwargs): self.source = ColumnDataSource(data={'time': [], 'cpu': [], 'memory_percent':[], 'network-send':[], 'network-recv':[]} ) x_range = DataRange1d(follow='end', follow_interval=30000, range_padding=0) resource_plot = Plot( x_range=x_range, y_range=Range1d(start=0, end=1), toolbar_location=None, min_border_bottom=10, **kwargs ) line_opts = dict(line_width=2, line_alpha=0.8) g1 = resource_plot.add_glyph( self.source, Line(x='time', y='memory_percent', line_color="#33a02c", **line_opts) ) g2 = resource_plot.add_glyph( self.source, Line(x='time', y='cpu', line_color="#1f78b4", **line_opts) ) resource_plot.add_layout( LinearAxis(formatter=NumeralTickFormatter(format="0 %")), 'left' ) legend_opts = dict( location='top_left', orientation='horizontal', padding=5, margin=5, label_height=5) resource_plot.add_layout( Legend(items=[('Memory', [g1]), ('CPU', [g2])], **legend_opts) ) network_plot = Plot( x_range=x_range, y_range=DataRange1d(start=0), toolbar_location=None, **kwargs ) g1 = network_plot.add_glyph( self.source, Line(x='time', y='network-send', line_color="#a6cee3", **line_opts) ) g2 =network_plot.add_glyph( self.source, Line(x='time', y='network-recv', line_color="#b2df8a", **line_opts) ) network_plot.add_layout(DatetimeAxis(axis_label="Time"), "below") network_plot.add_layout(LinearAxis(axis_label="MB/s"), 'left') network_plot.add_layout( Legend(items=[('Network Send', [g1]), ('Network Recv', [g2])], **legend_opts) ) tools = [ PanTool(dimensions='width'), WheelZoomTool(dimensions='width'), BoxZoomTool(), ResetTool() ] if 'sizing_mode' in kwargs: sizing_mode = {'sizing_mode': kwargs['sizing_mode']} else: sizing_mode = {} combo_toolbar = ToolbarBox( tools=tools, logo=None, toolbar_location='right', **sizing_mode ) self.root = row( column(resource_plot, network_plot, **sizing_mode), column(combo_toolbar, **sizing_mode), id='bk-resource-profiles-plot', **sizing_mode ) # Required for update callback self.resource_index = [0] def update(self, messages): with log_errors(): index = messages['workers']['index'] data = messages['workers']['plot-data'] if not index or index[-1] == self.resource_index[0]: return if self.resource_index == [0]: data = valmap(list, data) ind = bisect(index, self.resource_index[0]) indexes = list(range(ind, len(index))) data = {k: [v[i] for i in indexes] for k, v in data.items()} self.resource_index[0] = index[-1] self.source.stream(data, 1000) class WorkerTable(DashboardComponent): """ Status of the current workers This is two plots, a text-based table for each host and a thin horizontal plot laying out hosts by their current memory use. """ def __init__(self, **kwargs): names = ['processes', 'disk-read', 'cores', 'cpu', 'disk-write', 'memory', 'last-seen', 'memory_percent', 'host'] self.source = ColumnDataSource({k: [] for k in names}) columns = {name: TableColumn(field=name, title=name.replace('_percent', ' %')) for name in names} cnames = ['host', 'cores', 'processes', 'memory', 'cpu', 'memory_percent'] formatters = {'cpu': NumberFormatter(format='0.0 %'), 'memory_percent': NumberFormatter(format='0.0 %'), 'memory': NumberFormatter(format='0 b'), 'latency': NumberFormatter(format='0.00000'), 'last-seen': NumberFormatter(format='0.000'), 'disk-read': NumberFormatter(format='0 b'), 'disk-write': NumberFormatter(format='0 b'), 'net-send': NumberFormatter(format='0 b'), 'net-recv': NumberFormatter(format='0 b')} table = DataTable( source=self.source, columns=[columns[n] for n in cnames], ) for name in cnames: if name in formatters: table.columns[cnames.index(name)].formatter = formatters[name] mem_plot = Plot( title=Title(text="Memory Usage (%)"), toolbar_location=None, x_range=Range1d(start=0, end=1), y_range=Range1d(start=-0.1, end=0.1), **kwargs ) mem_plot.add_glyph( self.source, Circle(x='memory_percent', y=0, size=10, fill_alpha=0.5) ) mem_plot.add_layout(LinearAxis(), 'below') hover = HoverTool( point_policy="follow_mouse", tooltips="""
@host: @memory_percent
""" ) mem_plot.add_tools(hover, BoxSelectTool()) if 'sizing_mode' in kwargs: sizing_mode = {'sizing_mode': kwargs['sizing_mode']} else: sizing_mode = {} self.root = column(mem_plot, table, id='bk-worker-table', **sizing_mode) def update(self, messages): with log_errors(): try: d = messages['workers']['deque'][-1] except IndexError: return workers = sorted(d) data = {} data['host'] = workers for name in ['cores', 'cpu', 'memory_percent', 'latency', 'last-seen', 'memory', 'disk-read', 'disk-write', 'net-send', 'net-recv']: try: if name in ('cpu', 'memory_percent'): data[name] = [d[w][name] / 100 for w in workers] else: data[name] = [d[w][name] for w in workers] except KeyError: pass data['processing'] = [sorted(d[w]['processing']) for w in workers] data['processes'] = [len(d[w]['ports']) for w in workers] self.source.data.update(data) class ProcessingStacks(DashboardComponent): """ Processing and Stacks distribution per core This shows how many tasks are actively running on each worker and how many tasks are enqueued for each worker and how many are in the common pool """ def __init__(self, **kwargs): data = self.processing_update({'processing': {}, 'stacks': {}, 'ncores': {}, 'ready': 0}) self.source = ColumnDataSource(data) x_range = Range1d(-1, 1) fig = figure( title='Processing and Pending', tools='resize', x_range=x_range, id='bk-processing-stacks-plot', **kwargs) fig.quad(source=self.source, left=0, right='right', color=Spectral9[0], top='top', bottom='bottom') fig.quad(source=self.source, left='left', right=0, color=Spectral9[1], top='top', bottom='bottom', alpha='alpha') fig.xaxis.minor_tick_line_alpha = 0 fig.yaxis.visible = False fig.ygrid.visible = False hover = HoverTool() fig.add_tools(hover) hover = fig.select(HoverTool) hover.tooltips = """
Host:  @name
Stacks:  @stacks
Processing:  @processing
""" hover.point_policy = 'follow_mouse' self.root = fig def update(self, messages): with log_errors(): msg = messages['processing'] if not msg.get('ncores'): return data = self.processing_update(msg) x_range = self.root.x_range max_right = max(data['right']) min_left = min(data['left'][:-1]) cores = max(data['ncores']) if min_left < x_range.start: # not out there enough, jump ahead x_range.start = min_left - 2 elif x_range.start < 2 * min_left - cores: # way out there, walk back x_range.start = x_range.start * 0.95 + min_left * 0.05 if x_range.end < max_right: x_range.end = max_right + 2 elif x_range.end > 2 * max_right + cores: # way out there, walk back x_range.end = x_range.end * 0.95 + max_right * 0.05 self.source.data.update(data) @staticmethod def processing_update(msg): with log_errors(): names = sorted(msg['stacks']) stacks = msg['stacks'] stacks = [stacks[name] for name in names] names = sorted(names) processing = msg['processing'] processing = [processing[name] for name in names] ncores = msg['ncores'] ncores = [ncores[name] for name in names] n = len(names) d = {'name': list(names), 'processing': processing, 'stacks': list(stacks), 'right': list(processing), 'left': [-s for s in stacks], 'top': list(range(n, 0, -1)), 'bottom': list(range(n - 1, -1, -1)), 'ncores': ncores} d['name'].append('ready') d['processing'].append(0) d['stacks'].append(msg['ready']) d['left'].append(-msg['ready'] / n if n else 0) d['right'].append(0) d['top'].append(n) d['bottom'].append(0) d['ncores'].append(sum(msg['ncores'].values())) d['alpha'] = [0.7] * n + [0.2] return d distributed-1.14.3/distributed/bokeh/export_tool.coffee000066400000000000000000000021051301211143400232470ustar00rootroot00000000000000p = require "core/properties" ActionTool = require "models/tools/actions/action_tool" class ExportToolView extends ActionTool.View initialize: (options) -> super(options) @listenTo(@model, 'change:content', @export) do: () -> # This is just to trigger an event a python callback can respond to @model.event = @model.event + 1 export: () -> if @model.content? blob = new Blob([@model.content], {type: "text/plain"}) url = window.URL.createObjectURL(blob); a = document.createElement("a") a.id = "bk-export-tool-link" a.style = "display: none" a.href = url a.download = 'task-stream.html' document.body.appendChild(a) a.click() document.getElementById('bk-export-tool-link').remove() window.URL.revokeObjectURL(url); class ExportTool extends ActionTool.Model default_view: ExportToolView type: "ExportTool" tool_name: "Export" icon: "bk-tool-icon-save" @define { event: [ p.Int, 0 ] content: [ p.String ] } module.exports = Model: ExportTool View: ExportToolView distributed-1.14.3/distributed/bokeh/export_tool.py000066400000000000000000000015731301211143400224600ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import os from bokeh.core.properties import Int, String from bokeh.driving import cosine from bokeh.embed import file_html from bokeh.io import curdoc from bokeh.models import Tool from bokeh.plotting import figure from bokeh.resources import CDN from bokeh.util.compiler import JavaScript fn = __file__ fn = os.path.join(os.path.dirname(fn), 'export_tool.js') with open(fn) as f: JS_CODE = f.read() class ExportTool(Tool): __implementation__ = JavaScript(JS_CODE) event = Int(default=0) content = String() def register_plot(self, plot): def export_callback(attr, old, new): # really, export the doc as JSON self.content = None html = file_html(plot, CDN, "Task Stream") self.content = html self.on_change('event', export_callback) distributed-1.14.3/distributed/bokeh/memory-usage.py000066400000000000000000000005371301211143400225130ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from bokeh.io import curdoc from distributed.bokeh import messages from distributed.bokeh.components import MemoryUsage component = MemoryUsage(sizing_mode='stretch_both') doc = curdoc() doc.add_periodic_callback(lambda: component.update(messages), 50) doc.add_root(component.root) distributed-1.14.3/distributed/bokeh/processing-stacks.py000066400000000000000000000005521301211143400235400ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from bokeh.io import curdoc from distributed.bokeh import messages from distributed.bokeh.components import ProcessingStacks component = ProcessingStacks(sizing_mode='stretch_both') doc = curdoc() doc.add_periodic_callback(lambda: component.update(messages), 200) doc.add_root(component.root) distributed-1.14.3/distributed/bokeh/resource-profiles.py000066400000000000000000000006441301211143400235500ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from bokeh.io import curdoc from distributed.bokeh import messages from distributed.bokeh.components import ResourceProfiles component = ResourceProfiles(sizing_mode='stretch_both') doc = curdoc() doc.add_periodic_callback(lambda: component.update(messages), messages['task-events']['interval']) doc.add_root(component.root) distributed-1.14.3/distributed/bokeh/status/000077500000000000000000000000001301211143400210455ustar00rootroot00000000000000distributed-1.14.3/distributed/bokeh/status/__init__.py000066400000000000000000000000001301211143400231440ustar00rootroot00000000000000distributed-1.14.3/distributed/bokeh/status/main.py000066400000000000000000000025571301211143400223540ustar00rootroot00000000000000#!/usr/bin/env python from __future__ import print_function, division, absolute_import from bokeh.io import curdoc from bokeh.layouts import column, row from distributed.bokeh.components import ( TaskStream, TaskProgress, MemoryUsage, ResourceProfiles ) import distributed.bokeh SIZING_MODE = 'scale_width' WIDTH = 600 messages = distributed.bokeh.messages # global message store doc = curdoc() task_stream = TaskStream(1000, sizing_mode=SIZING_MODE, width=WIDTH, height=300, clear_interval=10000) doc.add_periodic_callback(lambda: task_stream.update(messages), messages['task-events']['interval']) task_progress = TaskProgress(sizing_mode=SIZING_MODE, width=WIDTH, height=160) doc.add_periodic_callback(lambda: task_progress.update(messages), 50) memory_usage = MemoryUsage(sizing_mode=SIZING_MODE, width=WIDTH, height=60) doc.add_periodic_callback(lambda: memory_usage.update(messages), 50) resource_profiles = ResourceProfiles(sizing_mode=SIZING_MODE, width=WIDTH, height=80) doc.add_periodic_callback(lambda: resource_profiles.update(messages), messages['task-events']['interval']) layout = column( row(resource_profiles.root, sizing_mode=SIZING_MODE), row(memory_usage.root, sizing_mode=SIZING_MODE), row(task_stream.root, sizing_mode=SIZING_MODE), row(task_progress.root, sizing_mode=SIZING_MODE), sizing_mode=SIZING_MODE ) doc.add_root(layout) distributed-1.14.3/distributed/bokeh/status/templates/000077500000000000000000000000001301211143400230435ustar00rootroot00000000000000distributed-1.14.3/distributed/bokeh/status/templates/index.html000066400000000000000000000031161301211143400250410ustar00rootroot00000000000000 Dask - Status dashboard {{ bokeh_css }} {{ bokeh_js }}

Dask -- Documentation -- Scheduler Docs -- Tasks -- Workers

{{ plot_div }}
{{ plot_script }} distributed-1.14.3/distributed/bokeh/task-progress.py000066400000000000000000000005411301211143400227000ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from bokeh.io import curdoc from distributed.bokeh import messages from distributed.bokeh.components import TaskProgress component = TaskProgress(sizing_mode='stretch_both') doc = curdoc() doc.add_periodic_callback(lambda: component.update(messages), 50) doc.add_root(component.root) distributed-1.14.3/distributed/bokeh/task-stream.py000066400000000000000000000006301301211143400223260ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from bokeh.io import curdoc from distributed.bokeh import messages from distributed.bokeh.components import TaskStream component = TaskStream(sizing_mode='stretch_both') doc = curdoc() doc.add_periodic_callback(lambda: component.update(messages), messages['task-events']['interval']) doc.add_root(component.root) distributed-1.14.3/distributed/bokeh/tasks/000077500000000000000000000000001301211143400206475ustar00rootroot00000000000000distributed-1.14.3/distributed/bokeh/tasks/__init__.py000066400000000000000000000000001301211143400227460ustar00rootroot00000000000000distributed-1.14.3/distributed/bokeh/tasks/main.py000066400000000000000000000010041301211143400221400ustar00rootroot00000000000000#!/usr/bin/env python from __future__ import print_function, division, absolute_import from bokeh.io import curdoc from distributed.bokeh.components import TaskStream import distributed.bokeh messages = distributed.bokeh.messages # global message store doc = curdoc() task_stream = TaskStream(100000, sizing_mode='stretch_both', clear_interval=60*1000) task_stream.update(messages) doc.add_periodic_callback(lambda: task_stream.update(messages), 2000) doc.add_root(task_stream.root) distributed-1.14.3/distributed/bokeh/tests/000077500000000000000000000000001301211143400206645ustar00rootroot00000000000000distributed-1.14.3/distributed/bokeh/tests/test_application.py000066400000000000000000000031711301211143400246020ustar00rootroot00000000000000import requests from time import time, sleep import sys import pytest from distributed.bokeh.application import BokehWebInterface from distributed import LocalCluster from distributed.http import HTTPScheduler from distributed.utils_test import cluster, loop from distributed.utils import ignoring def test_BokehWebInterface(loop): with LocalCluster(2, loop=loop, scheduler_port=0, services={('http', 0): HTTPScheduler}, diagnostics_port=None) as c: with pytest.raises(Exception): response = requests.get('http://127.0.0.1:8787/status/') with BokehWebInterface( tcp_port=c.scheduler.port, http_port=c.scheduler.services['http'].port, bokeh_port=8787) as w: start = time() while True: with ignoring(Exception): response = requests.get('http://127.0.0.1:8787/status/') if response.ok: break assert time() < start + 5 sleep(0.01) with pytest.raises(Exception): response = requests.get('http://127.0.0.1:8787/status/') def test_bokeh_shutsdown_without_cluster___del__(loop): c = LocalCluster(2, loop=loop, scheduler_port=0, services={('http', 0): HTTPScheduler}) proc = c.diagnostics.process # don't run the del, as it isn't ever run in python < 3.5 due to cycles c.__del__ = lambda self: None del c start = time() while True: if proc.poll() is not None: break assert time() < start + 5 sleep(0.01) distributed-1.14.3/distributed/bokeh/tests/test_components.py000066400000000000000000000023151301211143400244630ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import pytest from bokeh.models import ColumnDataSource, Model from distributed.bokeh import messages from distributed.bokeh.components import ( TaskStream, TaskProgress, MemoryUsage, ResourceProfiles, WorkerTable, ProcessingStacks ) @pytest.mark.parametrize('Component', [TaskStream, TaskProgress, MemoryUsage, ResourceProfiles, WorkerTable, ProcessingStacks]) def test_basic(Component): c = Component() assert isinstance(c.source, ColumnDataSource) assert isinstance(c.root, Model) c.update(messages) from distributed.utils_test import gen_cluster from tornado import gen from distributed.diagnostics.scheduler import workers @gen_cluster() def test_worker_table(s, a, b): while any('last-seen' not in v for v in s.host_info.values()): yield gen.sleep(0.01) data = workers(s) messages = {'workers': {'deque': [data]}} c = WorkerTable() c.update(messages) assert c.source.data['host'] == ['127.0.0.1'] distributed-1.14.3/distributed/bokeh/tests/test_worker_monitor.py000066400000000000000000000040241301211143400253550ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import pytest pytest.importorskip('bokeh') from tornado import gen from distributed.bokeh.worker_monitor import resource_append from distributed.utils_test import gen_cluster def test_resource_append(): lists = {'time': [], 'cpu': [], 'memory_percent': [], 'network-send': [], 'network-recv': []} msg = {'10.10.20.86': {'cpu': 10, 'memory_percent': 50, 'time': 2000, 'network-send': 2**16, 'network-recv': 2**15}, '10.10.20.87': {'cpu': 30, 'memory_percent': 70, 'time': 1000, 'network-send': 2**17, 'network-recv': 2**16}} resource_append(lists, msg) assert lists == {'time': [1500000], 'cpu': [0.2], 'memory_percent': [0.6], 'network-send': [0.1875], 'network-recv': [0.09375]} def test_processing_update(): from distributed.diagnostics.scheduler import processing from distributed.bokeh.components import ProcessingStacks class C(object): pass s = C() s.stacks = {'alice': ['inc', 'inc', 'add'], 'bob': ['add', 'add']} s.processing = {'alice': {'inc', 'add'}, 'bob': {'inc'}} s.ready = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] s.waiting = {'x': set()} s.who_has = {'z': set()} s.ncores = {'alice': 4, 'bob': 4} msg = processing(s) assert msg == {'processing': {'alice': 2, 'bob': 1}, 'stacks': {'alice': 3, 'bob': 2}, 'ready': 7, 'waiting': 1, 'memory': 1, 'ncores': {'alice': 4, 'bob': 4}} data = ProcessingStacks.processing_update(msg) expected = {'name': ['alice', 'bob', 'ready'], 'processing': [2, 1, 0], 'stacks': [3, 2, 7], 'left': [-3, -2, -7/2], 'right': [2, 1, 0], 'top': [2, 1, 2], 'bottom': [1, 0, 0], 'ncores': [4, 4, 8], 'alpha': [0.7, 0.7, 0.2]} assert data == expected distributed-1.14.3/distributed/bokeh/utils.py000066400000000000000000000004111301211143400212300ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from toolz import partition def parse_args(args): options = dict(partition(2, args)) for k, v in options.items(): if v.isdigit(): options[k] = int(v) return options distributed-1.14.3/distributed/bokeh/worker-table.py000066400000000000000000000006261301211143400224760ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from bokeh.io import curdoc from distributed.bokeh import messages from distributed.bokeh.components import WorkerTable component = WorkerTable(sizing_mode='stretch_both') doc = curdoc() doc.add_periodic_callback(lambda: component.update(messages), messages['workers']['interval']) doc.add_root(component.root) distributed-1.14.3/distributed/bokeh/worker_monitor.py000066400000000000000000000071031301211143400231550ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from collections import defaultdict from itertools import chain import logging from toolz import pluck from ..utils import ignoring, log_errors logger = logging.getLogger(__file__) with ignoring(ImportError): from bokeh.models import ( ColumnDataSource, DataRange1d, Range1d, NumeralTickFormatter, ToolbarBox, ) from bokeh.palettes import Spectral9, Viridis4 from bokeh.models import HoverTool, Paragraph from bokeh.models.widgets import DataTable, TableColumn, NumberFormatter from bokeh.plotting import figure def _format_resource_profile_plot(plot): plot.legend[0].location = 'top_left' plot.legend[0].orientation = 'horizontal' plot.legend[0].padding = 5 plot.legend[0].margin = 5 plot.legend[0].label_height = 5 plot.toolbar.logo = None plot.yaxis[0].ticker.num_minor_ticks = 2 plot.yaxis[0].ticker.desired_num_ticks = 2 return plot def resource_profile_plot(sizing_mode='fixed', **kwargs): names = ['time', 'cpu', 'memory_percent', 'network-send', 'network-recv'] source = ColumnDataSource({k: [] for k in names}) plot_props = dict( tools='xpan,xwheel_zoom,box_zoom,reset', toolbar_location=None, # Because we're making a joint toolbar sizing_mode=sizing_mode, x_axis_type='datetime', x_range=DataRange1d(follow='end', follow_interval=30000, range_padding=0) ) plot_props.update(kwargs) line_props = dict(x='time', line_width=2, line_alpha=0.8, source=source) y_range = Range1d(0, 1) p1 = figure(title=None, y_range=y_range, id='bk-resource-profile-plot', **plot_props) p1.line(y='memory_percent', color="#33a02c", legend='Memory', **line_props) p1.line(y='cpu', color="#1f78b4", legend='CPU', **line_props) p1 = _format_resource_profile_plot(p1) p1.yaxis.bounds = (0, 100) p1.yaxis.formatter = NumeralTickFormatter(format="0 %") p1.xaxis.visible = None p1.min_border_bottom = 10 y_range = DataRange1d(start=0) p2 = figure(title=None, y_range=y_range, id='bk-network-profile-plot', **plot_props) p2.line(y='network-send', color="#a6cee3", legend='Network Send', **line_props) p2.line(y='network-recv', color="#b2df8a", legend='Network Recv', **line_props) p2 = _format_resource_profile_plot(p2) p2.yaxis.axis_label = "MB/s" from bokeh.models import DatetimeAxis for r in list(p1.renderers): if isinstance(r, DatetimeAxis): p1.renderers.remove(r) all_tools = p1.toolbar.tools + p2.toolbar.tools combo_toolbar = ToolbarBox( tools=all_tools, sizing_mode=sizing_mode, logo=None, toolbar_location='right' ) return source, p1, p2, combo_toolbar def resource_append(lists, msg): L = list(msg.values()) if not L: return try: for k in ['cpu', 'memory_percent']: lists[k].append(mean(pluck(k, L)) / 100) except KeyError as e: # initial messages sometimes lack resource data logger.exception(e) return # this is safe to skip lists['time'].append(mean(pluck('time', L)) * 1000) if len(lists['time']) >= 2: t1, t2 = lists['time'][-2], lists['time'][-1] interval = (t2 - t1) / 1000 else: interval = 0.5 send = mean(pluck('network-send', L, 0)) lists['network-send'].append(send / 2**20 / (interval or 0.5)) recv = mean(pluck('network-recv', L, 0)) lists['network-recv'].append(recv / 2**20 / (interval or 0.5)) def mean(seq): seq = list(seq) return sum(seq) / len(seq) distributed-1.14.3/distributed/bokeh/workers/000077500000000000000000000000001301211143400212165ustar00rootroot00000000000000distributed-1.14.3/distributed/bokeh/workers/__init__.py000066400000000000000000000000001301211143400233150ustar00rootroot00000000000000distributed-1.14.3/distributed/bokeh/workers/main.py000066400000000000000000000015771301211143400225260ustar00rootroot00000000000000#!/usr/bin/env python from __future__ import print_function, division, absolute_import from bokeh.io import curdoc from bokeh.layouts import column, row import distributed.bokeh from distributed.bokeh.components import ProcessingStacks, WorkerTable SIZING_MODE = 'scale_width' WIDTH = 600 messages = distributed.bokeh.messages # global message store doc = curdoc() processing_stacks = ProcessingStacks(sizing_mode=SIZING_MODE, width=WIDTH, height=150) doc.add_periodic_callback(lambda: processing_stacks.update(messages), 200) worker_table = WorkerTable(sizing_mode=SIZING_MODE, width=WIDTH, plot_height=80) doc.add_periodic_callback(lambda: worker_table.update(messages), messages['workers']['interval']) layout = column( row(processing_stacks.root, sizing_mode=SIZING_MODE), row(worker_table.root, sizing_mode=SIZING_MODE), sizing_mode=SIZING_MODE ) doc.add_root(layout) distributed-1.14.3/distributed/bokeh/workers/templates/000077500000000000000000000000001301211143400232145ustar00rootroot00000000000000distributed-1.14.3/distributed/bokeh/workers/templates/index.html000066400000000000000000000031161301211143400252120ustar00rootroot00000000000000 Dask - Worker dashboard {{ bokeh_css }} {{ bokeh_js }}

Dask -- Documentation -- Scheduler Docs -- Tasks -- Workers

{{ plot_div }}
{{ plot_script }} distributed-1.14.3/distributed/cli/000077500000000000000000000000001301211143400172015ustar00rootroot00000000000000distributed-1.14.3/distributed/cli/__init__.py000066400000000000000000000000001301211143400213000ustar00rootroot00000000000000distributed-1.14.3/distributed/cli/dask_remote.py000066400000000000000000000010401301211143400220430ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import click from distributed.cli.utils import check_python_3, install_signal_handlers from distributed.submit import _remote @click.command() @click.option('--host', type=str, default=None, help="IP or hostname of this server") @click.option('--port', type=int, default=8788, help="Remote Client Port") def main(host, port): _remote(host, port) def go(): install_signal_handlers() check_python_3() main() if __name__ == '__main__': go() distributed-1.14.3/distributed/cli/dask_scheduler.py000077500000000000000000000066201301211143400225420ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import atexit import json import logging import multiprocessing import os import socket import subprocess import sys from time import sleep import click import distributed from distributed import Scheduler from distributed.utils import get_ip from distributed.http import HTTPScheduler from distributed.cli.utils import check_python_3, install_signal_handlers from tornado.ioloop import IOLoop logger = logging.getLogger('distributed.scheduler') @click.command() @click.option('--port', type=int, default=8786, help="Serving port") @click.option('--http-port', type=int, default=9786, help="HTTP port") @click.option('--bokeh-port', type=int, default=8787, help="Bokeh port") @click.option('--bokeh/--no-bokeh', '_bokeh', default=True, show_default=True, required=False, help="Launch Bokeh Web UI") @click.option('--host', type=str, default=None, help="IP or hostname of this server") @click.option('--show/--no-show', default=False, help="Show web UI") @click.option('--bokeh-whitelist', default=None, multiple=True, help="IP addresses to whitelist for bokeh.") @click.option('--prefix', type=str, default=None, help="Prefix for the bokeh app") @click.option('--use-xheaders', type=bool, default=False, show_default=True, help="User xheaders in bokeh app for ssl termination in header") @click.option('--pid-file', type=str, default='', help="File to write the process PID") def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders, pid_file): if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) if sys.platform.startswith('linux'): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) given_host = host host = host or get_ip() if ':' in host and port == 8786: host, port = host.rsplit(':', 1) port = int(port) ip = socket.gethostbyname(host) loop = IOLoop.current() logger.info('-' * 47) scheduler = Scheduler(ip=ip, loop=loop, services={('http', http_port): HTTPScheduler}) scheduler.start(port) bokeh_proc = None if _bokeh: try: from distributed.bokeh.application import BokehWebInterface bokeh_proc = BokehWebInterface(host=host, http_port=http_port, tcp_port=port, bokeh_port=bokeh_port, bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix, use_xheaders=use_xheaders, quiet=False) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) logger.info('-' * 47) try: loop.start() loop.close() finally: scheduler.stop() if bokeh_proc: bokeh_proc.close() logger.info("End scheduler at %s:%d", ip, port) def go(): install_signal_handlers() check_python_3() main() if __name__ == '__main__': go() distributed-1.14.3/distributed/cli/dask_ssh.py000077500000000000000000000071211301211143400213560ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from distributed.deploy.ssh import SSHCluster import click import os from distributed.cli.utils import check_python_3 @click.command(help = """Launch a distributed cluster over SSH. A 'dask-scheduler' process will run on the first host specified in [HOSTNAMES] or in the hostfile (unless --scheduler is specified explicitly). One or more 'dask-worker' processes will be run each host in [HOSTNAMES] or in the hostfile. Use command line flags to adjust how many dask-worker process are run on each host (--nprocs) and how many cpus are used by each dask-worker process (--nthreads).""") @click.option('--scheduler', default=None, type=str, help="Specify scheduler node. Defaults to first address.") @click.option('--scheduler-port', default=8786, type=int, help="Specify scheduler port number. Defaults to port 8786.") @click.option('--nthreads', default=0, type=int, help="Number of threads per worker process. Defaults to number of cores divided by the number of processes per host.") @click.option('--nprocs', default=1, type=int, help="Number of worker processes per host. Defaults to one.") @click.argument('hostnames', nargs=-1, type=str) @click.option('--hostfile', default=None, type=click.Path(exists=True), help="Textfile with hostnames/IP addresses") @click.option('--ssh-username', default=None, type=str, help="Username to use when establishing SSH connections.") @click.option('--ssh-port', default=22, type=int, help="Port to use for SSH connections.") @click.option('--ssh-private-key', default=None, type=str, help="Private key file to use for SSH connections.") @click.option('--log-directory', default=None, type=click.Path(exists=True), help="Directory to use on all cluster nodes for the output of dask-scheduler and dask-worker commands.") @click.pass_context def main(ctx, scheduler, scheduler_port, hostnames, hostfile, nthreads, nprocs, ssh_username, ssh_port, ssh_private_key, log_directory): try: hostnames = list(hostnames) if hostfile: with open(hostfile) as f: hosts = f.read().split() hostnames.extend(hosts) if not scheduler: scheduler = hostnames[0] except IndexError: print(ctx.get_help()) exit(1) c = SSHCluster(scheduler, scheduler_port, hostnames, nthreads, nprocs, ssh_username, ssh_port, ssh_private_key, log_directory) import distributed print('\n---------------------------------------------------------------') print(' Dask.distributed v{version}\n'.format(version=distributed.__version__)) print('Worker nodes:'.format(n=len(hostnames))) for i, host in enumerate(hostnames): print(' {num}: {host}'.format(num = i, host = host)) print('\nscheduler node: {addr}:{port}'.format(addr=scheduler, port=scheduler_port)) print('---------------------------------------------------------------\n\n') # Monitor the output of remote processes. This blocks until the user issues a KeyboardInterrupt. c.monitor_remote_processes() # Close down the remote processes and exit. print("\n[ dask-ssh ]: Shutting down remote processes (this may take a moment).") c.shutdown() print("[ dask-ssh ]: Remote processes have been terminated. Exiting.") def go(): check_python_3() main() if __name__ == '__main__': start() distributed-1.14.3/distributed/cli/dask_submit.py000066400000000000000000000013551301211143400220640ustar00rootroot00000000000000import sys import click from tornado import gen from tornado.ioloop import IOLoop from distributed.cli.utils import check_python_3, install_signal_handlers from distributed.submit import _submit @click.command() @click.argument('remote_client_address', type=str, required=True) @click.argument('filepath', type=str, required=True) def main(remote_client_address, filepath): @gen.coroutine def f(): stdout, stderr = yield _submit(remote_client_address, filepath) if stdout: sys.stdout.write(str(stdout)) if stderr: sys.stderr.write(str(stderr)) IOLoop.instance().run_sync(f) def go(): install_signal_handlers() check_python_3() main() if __name__ == '__main__': go() distributed-1.14.3/distributed/cli/dask_worker.py000077500000000000000000000140161301211143400220730ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import atexit from datetime import timedelta import logging import os import shutil import socket from sys import argv, exit import sys from time import time, sleep import click from distributed import Nanny, Worker, sync, rpc from distributed.nanny import isalive from distributed.utils import get_ip, All from distributed.worker import _ncores from distributed.http import HTTPWorker from distributed.cli.utils import check_python_3 from tornado.ioloop import IOLoop from tornado import gen logger = logging.getLogger('distributed.dask_worker') global_nannies = [] import signal def handle_signal(sig, frame): loop = IOLoop.instance() for nanny in global_nannies: try: shutil.rmtree(nanny.worker_dir) except (OSError, IOError, TypeError): pass if loop._running: loop.add_callback(loop.stop) else: exit(1) @click.command() @click.argument('scheduler', type=str) @click.option('--worker-port', type=int, default=0, help="Serving worker port, defaults to randomly assigned") @click.option('--http-port', type=int, default=0, help="Serving http port, defaults to randomly assigned") @click.option('--nanny-port', type=int, default=0, help="Serving nanny port, defaults to randomly assigned") @click.option('--host', type=str, default=None, help="Serving host. Defaults to an ip address that can hopefully" " be visible from the scheduler network.") @click.option('--nthreads', type=int, default=0, help="Number of threads per process. Defaults to number of cores") @click.option('--nprocs', type=int, default=1, help="Number of worker processes. Defaults to one.") @click.option('--name', type=str, default='', help="Alias") @click.option('--memory-limit', default='auto', help="Number of bytes before spilling data to disk. " "This can be an integer (nbytes) float (fraction of total memory) or auto") @click.option('--no-nanny', is_flag=True) @click.option('--pid-file', type=str, default='', help="File to write the process PID") @click.option('--temp-filename', default=None, help="Internal use only") def main(scheduler, host, worker_port, http_port, nanny_port, nthreads, nprocs, no_nanny, name, memory_limit, pid_file, temp_filename): if no_nanny: port = worker_port else: port = nanny_port try: scheduler_host, scheduler_port = scheduler.split(':') scheduler_ip = socket.gethostbyname(scheduler_host) scheduler_port = int(scheduler_port) except IndexError: logger.info("Usage: dask-worker scheduler_host:scheduler_port") if nprocs > 1 and worker_port != 0: logger.error("Failed to launch worker. You cannot use the --port argument when nprocs > 1.") exit(1) if nprocs > 1 and name: logger.error("Failed to launch worker. You cannot use the --name argument when nprocs > 1.") exit(1) if not nthreads: nthreads = _ncores // nprocs if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) services = {('http', http_port): HTTPWorker} loop = IOLoop.current() if memory_limit == 'auto': import psutil memory_limit = psutil.virtual_memory().total * 0.60 if memory_limit: memory_limit = float(memory_limit) if memory_limit < 1.0: import psutil memory_limit = psutil.virtual_memory().total * memory_limit memory_limit /= nprocs memory_limit = int(memory_limit) if no_nanny: kwargs = {} if nanny_port: kwargs['service_ports'] = {'nanny': nanny_port} t = Worker else: kwargs = {'worker_port': worker_port} t = Nanny if host is not None: ip = socket.gethostbyname(host) else: # lookup the ip address of a local interface on a network that # reach the scheduler ip = get_ip(scheduler_ip, scheduler_port) nannies = [t(scheduler_ip, scheduler_port, ncores=nthreads, ip=ip, services=services, name=name, loop=loop, memory_limit=memory_limit, **kwargs) for i in range(nprocs)] for nanny in nannies: nanny.start(port) if t is Nanny: global_nannies.append(nanny) if temp_filename: @gen.coroutine def f(): while nannies[0].status != 'running': yield gen.sleep(0.01) import json msg = {'port': nannies[0].port, 'local_directory': nannies[0].local_dir} with open(temp_filename, 'w') as f: json.dump(msg, f) loop.add_callback(f) loop.start() logger.info("End worker") loop.close() loop2 = IOLoop() @gen.coroutine def f(): scheduler = rpc(ip=nannies[0].scheduler.ip, port=nannies[0].scheduler.port) if not no_nanny: yield gen.with_timeout(timedelta(seconds=2), All([scheduler.unregister(address=n.worker_address, close=True) for n in nannies if n.process and n.worker_port]), io_loop=loop2) loop2.run_sync(f) if not no_nanny: for n in nannies: n.process.terminate() if not no_nanny: start = time() while (any(isalive(n.process) for n in nannies) and time() < start + 1): sleep(0.1) for nanny in nannies: nanny.stop() def go(): # NOTE: We can't use the generic install_signal_handlers() function from # distributed.cli.utils because we're handling the signal differently. signal.signal(signal.SIGINT, handle_signal) signal.signal(signal.SIGTERM, handle_signal) check_python_3() main() if __name__ == '__main__': go() distributed-1.14.3/distributed/cli/dcluster.py000066400000000000000000000003731301211143400214030ustar00rootroot00000000000000from warnings import warn def go(): print("The `dcluster` command has been deprecated.") print("Please use `dask-ssh` in the future.") print() from distributed.cli.dask_cluster import go go() if __name__ == '__main__': go() distributed-1.14.3/distributed/cli/dscheduler.py000066400000000000000000000004051301211143400216740ustar00rootroot00000000000000from warnings import warn def go(): print("The `dscheduler` command has been deprecated.") print("Please use `dask-scheduler` in the future.") print() from distributed.cli.dask_scheduler import go go() if __name__ == '__main__': go() distributed-1.14.3/distributed/cli/dworker.py000066400000000000000000000003741301211143400212340ustar00rootroot00000000000000from warnings import warn def go(): print("The `dworker` command has been deprecated.") print("Please use `dask-worker` in the future.") print() from distributed.cli.dask_worker import go go() if __name__ == '__main__': go() distributed-1.14.3/distributed/cli/tests/000077500000000000000000000000001301211143400203435ustar00rootroot00000000000000distributed-1.14.3/distributed/cli/tests/test_dask_remote.py000066400000000000000000000004361301211143400242540ustar00rootroot00000000000000from click.testing import CliRunner from distributed.cli.dask_remote import main def test_dask_remote(): runner = CliRunner() result = runner.invoke(main, ['--help']) assert '--host TEXT IP or hostname of this server' in result.output assert result.exit_code == 0 distributed-1.14.3/distributed/cli/tests/test_dask_scheduler.py000066400000000000000000000127041301211143400247400ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import pytest pytest.importorskip('requests') from contextlib import contextmanager import os import requests import signal import socket import sys from time import sleep, time from distributed import Scheduler, Client from distributed.utils import get_ip, ignoring, tmpfile from distributed.utils_test import loop, popen def test_defaults(loop): with popen(['dask-scheduler', '--no-bokeh']) as proc: with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c: response = requests.get('http://127.0.0.1:9786/info.json') assert response.ok assert response.json()['status'] == 'running' with pytest.raises(Exception): response = requests.get('http://127.0.0.1:9786/info.json') with pytest.raises(Exception): requests.get('http://127.0.0.1:8787/status/') def test_hostport(loop): with popen(['dask-scheduler', '--no-bokeh', '--host', '127.0.0.1:8978']): with Client('127.0.0.1:8978', loop=loop) as c: assert len(c.ncores()) == 0 def test_no_bokeh(loop): pytest.importorskip('bokeh') with popen(['dask-scheduler', '--no-bokeh']) as proc: with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c: for i in range(3): line = proc.stderr.readline() assert b'bokeh' not in line.lower() with pytest.raises(Exception): requests.get('http://127.0.0.1:8787/status/') def test_bokeh(loop): pytest.importorskip('bokeh') with popen(['dask-scheduler']) as proc: with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c: pass while True: line = proc.stderr.readline() if b'Web UI' in line: break names = ['localhost', '127.0.0.1', get_ip()] if 'linux' in sys.platform: names.append(socket.gethostname()) start = time() while True: try: for name in names: response = requests.get('http://%s:8787/status/' % name) assert response.ok break except Exception as f: print(f) sleep(0.1) assert time() < start + 10 with pytest.raises(Exception): requests.get('http://127.0.0.1:8787/status/') def test_bokeh_non_standard_ports(loop): pytest.importorskip('bokeh') with popen(['dask-scheduler', '--port', '3448', '--http-port', '4824', '--bokeh-port', '4832']) as proc: with Client('127.0.0.1:3448', loop=loop) as c: pass while True: line = proc.stderr.readline() if b'Web UI' in line: break start = time() while True: try: response = requests.get('http://localhost:4832/status/') assert response.ok break except: sleep(0.1) assert time() < start + 20 with pytest.raises(Exception): requests.get('http://127.0.0.1:4832/status/') @pytest.mark.skipif(not sys.platform.startswith('linux'), reason="Need 127.0.0.2 to mean localhost") def test_bokeh_whitelist(loop): pytest.importorskip('bokeh') with pytest.raises(Exception): requests.get('http://localhost:8787/status/').ok with popen(['dask-scheduler', '--bokeh-whitelist', '127.0.0.2:8787', '--bokeh-whitelist', '127.0.0.3:8787']) as proc: with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c: pass while True: line = proc.stderr.readline() if b'Web UI' in line: break start = time() while True: try: for name in ['127.0.0.2', '127.0.0.3']: response = requests.get('http://%s:8787/status/' % name) assert response.ok break except Exception as f: print(f) sleep(0.1) assert time() < start + 20 def test_multiple_workers(loop): with popen(['dask-scheduler', '--no-bokeh']) as s: with popen(['dask-worker', 'localhost:8786']) as a: with popen(['dask-worker', 'localhost:8786']) as b: with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c: start = time() while len(c.ncores()) < 2: sleep(0.1) assert time() < start + 10 def test_pid_file(loop): def check_pidfile(proc, pidfile): while not os.path.exists(pidfile): sleep(0.01) text = False while not text: sleep(0.01) with open(pidfile) as f: text = f.read() pid = int(text) if sys.platform.startswith('win'): # On Windows, `dask-XXX` invokes the dask-XXX.exe # shim, but the PID is written out by the child Python process assert pid else: assert proc.pid == pid with tmpfile() as s: with popen(['dask-scheduler', '--pid-file', s]) as sched: check_pidfile(sched, s) with tmpfile() as w: with popen(['dask-worker', '127.0.0.1:8786', '--pid-file', w]) as worker: check_pidfile(worker, w) distributed-1.14.3/distributed/cli/tests/test_dask_submit.py000066400000000000000000000004561301211143400242660ustar00rootroot00000000000000from click.testing import CliRunner from distributed.cli.dask_submit import main def test_submit_runs_as_a_cli(): runner = CliRunner() result = runner.invoke(main, ['--help']) assert result.exit_code == 0 assert 'Usage: main [OPTIONS] REMOTE_CLIENT_ADDRESS FILEPATH' in result.output distributed-1.14.3/distributed/cli/tests/test_dworker.py000066400000000000000000000024731301211143400234370ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import pytest pytest.importorskip('requests') import os import requests from subprocess import Popen, PIPE import signal from time import time, sleep from distributed import Scheduler, Client from distributed.core import rpc from distributed.utils import sync, ignoring from distributed.utils_test import loop, popen def test_nanny_worker_ports(loop): with popen(['dask-scheduler', '--port', '8989']) as sched: with popen(['dask-worker', '127.0.0.1:8989', '--host', '127.0.0.1', '--worker-port', '8788', '--nanny-port', '8789']) as worker: with Client('127.0.0.1:8989', loop=loop) as c: start = time() while True: d = sync(c.loop, c.scheduler.identity) if d['workers']: break else: assert time() - start < 5 sleep(0.1) assert d['workers']['127.0.0.1:8788']['services']['nanny'] == 8789 def test_no_nanny(loop): with popen(['dask-scheduler']) as sched: with popen(['dask-worker', '127.0.0.1:8786', '--no-nanny']) as worker: assert any(b'Registered' in worker.stderr.readline() for i in range(10)) distributed-1.14.3/distributed/cli/utils.py000066400000000000000000000021051301211143400207110ustar00rootroot00000000000000py3_err_msg = """ Your terminal does not properly support unicode text required by command line utilities running Python 3. This is commonly solved by specifying encoding environment variables, though exact solutions may depend on your system: $ export LC_ALL=C.UTF-8 $ export LANG=C.UTF-8 For more information see: http://click.pocoo.org/5/python3/ """.strip() def check_python_3(): """Ensures that the environment is good for unicode on Python 3.""" try: from click import _unicodefun _unicodefun._verify_python3_env() except (TypeError, RuntimeError) as e: import sys import click click.echo(py3_err_msg, err=True) sys.exit(1) def install_signal_handlers(): """Install global signal handlers to halt the Tornado IOLoop in case of a SIGINT or SIGTERM.""" from tornado.ioloop import IOLoop import signal def handle_signal(sig, frame): IOLoop.instance().add_callback(IOLoop.instance().stop) signal.signal(signal.SIGINT, handle_signal) signal.signal(signal.SIGTERM, handle_signal) distributed-1.14.3/distributed/client.py000066400000000000000000002360571301211143400202770ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from collections import defaultdict, Iterator, Iterable from concurrent.futures._base import DoneAndNotDoneFutures, CancelledError from contextlib import contextmanager import copy from datetime import timedelta from functools import partial from glob import glob import logging import os import sys from time import sleep import uuid from threading import Thread import six import socket import dask from dask.base import tokenize, normalize_token, Base from dask.core import flatten, get_dependencies from dask.compatibility import apply from dask.context import _globals from toolz import first, groupby, merge, valmap, keymap from tornado import gen from tornado.gen import Return, TimeoutError from tornado.locks import Event from tornado.ioloop import IOLoop, PeriodicCallback from tornado.iostream import StreamClosedError from tornado.queues import Queue from .batched import BatchedSend from .utils_comm import WrappedKey, unpack_remotedata, pack_data from .compatibility import Queue as pyQueue, Empty, isqueue from .core import (read, write, connect, coerce_to_rpc, clean_exception) from .protocol import to_serialize from .protocol.pickle import dumps, loads from .worker import dumps_function, dumps_task from .utils import (All, sync, funcname, ignoring, queue_to_iterator, tokey, log_errors, str_graph) from .versions import get_versions logger = logging.getLogger(__name__) _global_client = [None] class Future(WrappedKey): """ A remotely running computation A Future is a local proxy to a result running on a remote worker. A user manages future objects in the local Python process to determine what happens in the larger cluster. Examples -------- Futures typically emerge from Client computations >>> my_future = client.submit(add, 1, 2) # doctest: +SKIP We can track the progress and results of a future >>> my_future # doctest: +SKIP We can get the result or the exception and traceback from the future >>> my_future.result() # doctest: +SKIP See Also -------- Client: Creates futures """ def __init__(self, key, client): self.key = key self.client = client self.client._inc_ref(tokey(key)) self._generation = self.client.generation self._cleared = False tkey = tokey(key) if tkey not in client.futures: client.futures[tkey] = {'event': Event(), 'status': 'pending'} @property def executor(self): return self.client @property def status(self): try: return self.client.futures[tokey(self.key)]['status'] except KeyError: return 'cancelled' @property def event(self): return self.client.futures[tokey(self.key)]['event'] def done(self): """ Is the computation complete? """ return self.event.is_set() def result(self): """ Wait until computation completes. Gather result to local process """ result = sync(self.client.loop, self._result, raiseit=False) if self.status == 'error': six.reraise(*result) if self.status == 'cancelled': raise result else: return result @gen.coroutine def _result(self, raiseit=True): try: d = self.client.futures[tokey(self.key)] except KeyError: exception = CancelledError(self.key) if raiseit: raise exception else: raise gen.Return(exception) yield d['event'].wait() if self.status == 'error': if raiseit: six.reraise(*clean_exception(**d)) else: raise Return(clean_exception(**d)) else: result = yield self.client._gather([self]) raise gen.Return(result[0]) @gen.coroutine def _exception(self): yield self.event.wait() if self.status == 'error': exception = self.client.futures[self.key]['exception'] raise Return(exception) else: raise Return(None) def exception(self): """ Return the exception of a failed task See Also -------- Future.traceback """ return sync(self.client.loop, self._exception) def add_done_callback(self, fn): """ Call callback on future when callback has finished The callback ``fn`` should take the future as its only argument. This will be called regardless of if the future completes successfully, errs, or is cancelled """ self.client.loop.add_callback(done_callback, self, fn) def cancel(self): """ Returns True if the future has been cancelled """ return self.client.cancel([self]) def cancelled(self): """ Returns True if the future has been cancelled """ return tokey(self.key) not in self.client.futures @gen.coroutine def _traceback(self): yield self.event.wait() if self.status == 'error': raise Return(self.client.futures[tokey(self.key)]['traceback']) else: raise Return(None) def traceback(self): """ Return the traceback of a failed task This returns a traceback object. You can inspect this object using the ``traceback`` module. Alternatively if you call ``future.result()`` this traceback will accompany the raised exception. Examples -------- >>> import traceback # doctest: +SKIP >>> tb = future.traceback() # doctest: +SKIP >>> traceback.export_tb(tb) # doctest: +SKIP [...] See Also -------- Future.exception """ return sync(self.client.loop, self._traceback) @property def type(self): try: return self.client.futures[tokey(self.key)]['type'] except KeyError: return None def release(self): if not self._cleared and self.client.generation == self._generation: self._cleared = True self.client._dec_ref(tokey(self.key)) def __getstate__(self): return self.key def __setstate__(self, key): c = default_client() Future.__init__(self, key, c) c._send_to_scheduler({'op': 'update-graph', 'tasks': {}, 'keys': [tokey(self.key)], 'client': c.id}) def __del__(self): self.release() def __str__(self): if self.type: try: typ = self.type.__name__ except AttributeError: typ = str(self.type) return '' % (self.status, typ, self.key) else: return '' % (self.status, self.key) __repr__ = __str__ @gen.coroutine def done_callback(future, callback): """ Wait on future, then call callback """ while future.status == 'pending': yield future.event.wait() callback(future) @partial(normalize_token.register, Future) def normalize_future(f): return [f.key, type(f)] class Client(object): """ Drive computations on a distributed cluster The Client connects users to a distributed compute cluster. It provides an asynchronous user interface around functions and futures. This class resembles executors in ``concurrent.futures`` but also allows ``Future`` objects within ``submit/map`` calls. Parameters ---------- address: string, tuple, or ``Scheduler`` This can be the address of a ``Scheduler`` server, either as a string ``'127.0.0.1:8787'`` or tuple ``('127.0.0.1', 8787)`` or it can be a local ``Scheduler`` object. Examples -------- Provide cluster's head node address on initialization: >>> client = Client('127.0.0.1:8787') # doctest: +SKIP Use ``submit`` method to send individual computations to the cluster >>> a = client.submit(add, 1, 2) # doctest: +SKIP >>> b = client.submit(add, 10, 20) # doctest: +SKIP Continue using submit or map on results to build up larger computations >>> c = client.submit(add, a, b) # doctest: +SKIP Gather results with the ``gather`` method. >>> client.gather([c]) # doctest: +SKIP 33 See Also -------- distributed.scheduler.Scheduler: Internal scheduler """ def __init__(self, address=None, start=True, loop=None, timeout=3, set_as_default=True): self.futures = dict() self.refcount = defaultdict(lambda: 0) self._should_close_loop = loop is None and start self.loop = loop or IOLoop() if start else IOLoop.current() self.coroutines = [] self.id = str(uuid.uuid1()) self.generation = 0 self.status = None self._pending_msg_buffer = [] if hasattr(address, 'scheduler_address'): self.cluster = address address = address.scheduler_address self._start_arg = address if set_as_default: self._previous_get = _globals.get('get') dask.set_options(get=self.get) self._previous_shuffle = _globals.get('shuffle') dask.set_options(shuffle='tasks') if start: self.start(timeout=timeout) def __str__(self): if hasattr(self, '_loop_thread'): n = sync(self.loop, self.scheduler.ncores) return '' % ( self.scheduler.ip, self.scheduler.port, len(n), sum(n.values())) else: return '' % ( self.scheduler.ip, self.scheduler.port) __repr__ = __str__ def start(self, **kwargs): """ Start scheduler running in separate thread """ if hasattr(self, '_loop_thread'): return if not self.loop._running: from threading import Thread self._loop_thread = Thread(target=self.loop.start) self._loop_thread.daemon = True self._loop_thread.start() while not self.loop._running: sleep(0.001) pc = PeriodicCallback(lambda: None, 1000, io_loop=self.loop) self.loop.add_callback(pc.start) _global_client[0] = self sync(self.loop, self._start, **kwargs) self.status = 'running' def _send_to_scheduler(self, msg): if self.status is 'running': self.loop.add_callback(self.scheduler_stream.send, msg) elif self.status is 'connecting': self._pending_msg_buffer.append(msg) else: raise Exception("Client not running. Status: %s" % self.status) @gen.coroutine def _start(self, timeout=3, **kwargs): if self._start_arg is None: from distributed.deploy import LocalCluster try: self.cluster = LocalCluster(loop=self.loop, start=False) except (OSError, socket.error): self.cluster = LocalCluster(scheduler_port=0, loop=self.loop, start=False) self._start_arg = self.cluster.scheduler_address while (not self.cluster.workers or len(self.cluster.scheduler.ncores) < len(self.cluster.workers)): yield gen.sleep(0.01) self.scheduler = coerce_to_rpc(self._start_arg, timeout=timeout) self.scheduler_stream = None yield self.ensure_connected() self.coroutines.append(self._handle_report()) @gen.coroutine def reconnect(self, timeout=0.1): with log_errors(): assert self.scheduler_stream.stream.closed() self.status = 'connecting' self.scheduler_stream = None events = [d['event'] for d in self.futures.values()] self.futures.clear() for e in events: e.set() while self.status == 'connecting': try: yield self.ensure_connected() break except IOError: yield gen.sleep(timeout) @gen.coroutine def ensure_connected(self): if self.scheduler_stream and not self.scheduler_stream.closed(): return try: stream = yield connect(self.scheduler.ip, self.scheduler.port) except: raise IOError("Could not connect to %s:%d" % (self.scheduler.ip, self.scheduler.port)) ident = yield self.scheduler.identity() yield write(stream, {'op': 'register-client', 'client': self.id}) msg = yield read(stream) assert len(msg) == 1 assert msg[0]['op'] == 'stream-start' bstream = BatchedSend(interval=10, loop=self.loop) bstream.start(stream) self.scheduler_stream = bstream _global_client[0] = self self.status = 'running' for msg in self._pending_msg_buffer: self._send_to_scheduler(msg) del self._pending_msg_buffer[:] logger.debug("Started scheduling coroutines. Synchronized") def __enter__(self): if not self.loop._running: self.start() return self def __exit__(self, type, value, traceback): self.shutdown() def __del__(self): self.shutdown() def _inc_ref(self, key): self.refcount[key] += 1 def _dec_ref(self, key): self.refcount[key] -= 1 if self.refcount[key] == 0: del self.refcount[key] self._release_key(key) def _release_key(self, key): """ Release key from distributed memory """ logger.debug("Release key %s", key) if key in self.futures: self.futures[key]['event'].clear() del self.futures[key] if self.status != 'closed': self._send_to_scheduler({'op': 'client-releases-keys', 'keys': [key], 'client': self.id}) @gen.coroutine def _handle_report(self): """ Listen to scheduler """ with log_errors(): while True: try: msgs = yield read(self.scheduler_stream.stream) except StreamClosedError: logger.debug("Stream closed to scheduler", exc_info=True) if self.status == 'running': logger.info("Reconnecting...") self.status = 'connecting' yield self.reconnect() continue else: break if not isinstance(msgs, list): msgs = [msgs] breakout = False for msg in msgs: logger.debug("Client receives message %s", msg) if msg.get('status') == 'scheduler-error': six.reraise(*clean_exception(**msg)) elif msg['op'] == 'close': breakout = True break elif msg['op'] == 'key-in-memory': if msg['key'] in self.futures: self.futures[msg['key']]['status'] = 'finished' self.futures[msg['key']]['event'].set() if (msg.get('type') and not self.futures[msg['key']].get('type')): self.futures[msg['key']]['type'] = loads(msg['type']) elif msg['op'] == 'lost-data': if msg['key'] in self.futures: self.futures[msg['key']]['status'] = 'lost' self.futures[msg['key']]['event'].clear() elif msg['op'] == 'cancelled-key': if msg['key'] in self.futures: self.futures[msg['key']]['event'].set() del self.futures[msg['key']] elif msg['op'] == 'task-erred': if msg['key'] in self.futures: self.futures[msg['key']]['status'] = 'error' try: self.futures[msg['key']]['exception'] = loads(msg['exception']) except TypeError: self.futures[msg['key']]['exception'] = \ Exception('Undeserializable exception', msg['exception']) self.futures[msg['key']]['traceback'] = (loads(msg['traceback']) if msg['traceback'] else None) self.futures[msg['key']]['event'].set() elif msg['op'] == 'restart': logger.info("Receive restart signal from scheduler") events = [d['event'] for d in self.futures.values()] self.futures.clear() for e in events: e.set() with ignoring(AttributeError): self._restart_event.set() elif 'error' in msg['op']: logger.warn("Scheduler exception:") logger.exception(msg['exception']) if breakout: break @gen.coroutine def _shutdown(self, fast=False): """ Send shutdown signal and wait until scheduler completes """ with log_errors(): if self.status == 'closed': raise Return() self._send_to_scheduler({'op': 'close-stream'}) self.status = 'closed' if _global_client[0] is self: _global_client[0] = None if not fast: with ignoring(TimeoutError): yield [gen.with_timeout(timedelta(seconds=2), f) for f in self.coroutines] with ignoring(AttributeError): yield self.scheduler_stream.close(ignore_closed=True) with ignoring(AttributeError): self.scheduler.close_rpc() def shutdown(self, timeout=10): """ Send shutdown signal and wait until scheduler terminates This cancels all currently running tasks, clears the state of the scheduler, and shuts down all workers and scheduler. You do not need to call this when you finish your session. You only need to call this if you want to take down the distributed cluster. See Also -------- Client.restart """ # XXX handling of self.status here is not thread-safe if self.status == 'closed': return sync(self.loop, self._shutdown, fast=True) assert self.status == 'closed' if self._should_close_loop: sync(self.loop, self.loop.stop) self.loop.close(all_fds=True) self._loop_thread.join(timeout=timeout) with ignoring(AttributeError): dask.set_options(get=self._previous_get) with ignoring(AttributeError): dask.set_options(shuffle=self._previous_shuffle) if self.get == _globals.get('get'): del _globals['get'] with ignoring(AttributeError): self.cluster.close() def submit(self, func, *args, **kwargs): """ Submit a function application to the scheduler Parameters ---------- func: callable *args: **kwargs: pure: bool (defaults to True) Whether or not the function is pure. Set ``pure=False`` for impure functions like ``np.random.random``. workers: set, iterable of sets A set of worker hostnames on which computations may be performed. Leave empty to default to all workers (common case) allow_other_workers: bool (defaults to False) Used with `workers`. Inidicates whether or not the computations may be performed on workers that are not in the `workers` set(s). Examples -------- >>> c = client.submit(add, a, b) # doctest: +SKIP Returns ------- Future See Also -------- Client.map: Submit on many arguments at once """ if not callable(func): raise TypeError("First input to submit must be a callable function") key = kwargs.pop('key', None) pure = kwargs.pop('pure', True) workers = kwargs.pop('workers', None) allow_other_workers = kwargs.pop('allow_other_workers', False) if allow_other_workers not in (True, False, None): raise TypeError("allow_other_workers= must be True or False") if key is None: if pure: key = funcname(func) + '-' + tokenize(func, kwargs, *args) else: key = funcname(func) + '-' + str(uuid.uuid4()) skey = tokey(key) if skey in self.futures: return Future(key, self) if allow_other_workers and workers is None: raise ValueError("Only use allow_other_workers= if using workers=") if isinstance(workers, str): workers = [workers] if workers is not None: restrictions = {skey: workers} loose_restrictions = [skey] if allow_other_workers else [] else: restrictions = {} loose_restrictions = [] if kwargs: dsk = {skey: (apply, func, list(args), kwargs)} else: dsk = {skey: (func,) + tuple(args)} futures = self._graph_to_futures(dsk, [skey], restrictions, loose_restrictions, priority={skey: 0}) logger.debug("Submit %s(...), %s", funcname(func), key) return futures[skey] def _threaded_map(self, q_out, func, qs_in, **kwargs): """ Internal function for mapping Queue """ if isqueue(qs_in[0]): get = pyQueue.get elif isinstance(qs_in[0], Iterator): get = next else: raise NotImplementedError() while True: try: args = [get(q) for q in qs_in] except StopIteration as e: q_out.put(e) break f = self.submit(func, *args, **kwargs) q_out.put(f) def map(self, func, *iterables, **kwargs): """ Map a function on a sequence of arguments Arguments can be normal objects or Futures Parameters ---------- func: callable iterables: Iterables, Iterators, or Queues key: str, list Prefix for task names if string. Explicit names if list. pure: bool (defaults to True) Whether or not the function is pure. Set ``pure=False`` for impure functions like ``np.random.random``. workers: set, iterable of sets A set of worker hostnames on which computations may be performed. Leave empty to default to all workers (common case) Examples -------- >>> L = client.map(func, sequence) # doctest: +SKIP Returns ------- List, iterator, or Queue of futures, depending on the type of the inputs. See also -------- Client.submit: Submit a single function """ if not callable(func): raise TypeError("First input to map must be a callable function") if (all(map(isqueue, iterables)) or all(isinstance(i, Iterator) for i in iterables)): maxsize = kwargs.pop('maxsize', 0) q_out = pyQueue(maxsize=maxsize) t = Thread(target=self._threaded_map, args=(q_out, func, iterables), kwargs=kwargs) t.daemon = True t.start() if isqueue(iterables[0]): return q_out else: return queue_to_iterator(q_out) key = kwargs.pop('key', None) key = key or funcname(func) pure = kwargs.pop('pure', True) workers = kwargs.pop('workers', None) allow_other_workers = kwargs.pop('allow_other_workers', False) if allow_other_workers and workers is None: raise ValueError("Only use allow_other_workers= if using workers=") iterables = list(zip(*zip(*iterables))) if isinstance(key, list): keys = key else: if pure: keys = [key + '-' + tokenize(func, kwargs, *args) for args in zip(*iterables)] else: uid = str(uuid.uuid4()) keys = [key + '-' + uid + '-' + str(i) for i in range(min(map(len, iterables)))] if iterables else [] if not kwargs: dsk = {key: (func,) + args for key, args in zip(keys, zip(*iterables))} else: dsk = {key: (apply, func, (tuple, list(args)), kwargs) for key, args in zip(keys, zip(*iterables))} if isinstance(workers, str): workers = [workers] if isinstance(workers, (list, set)): if workers and isinstance(first(workers), (list, set)): if len(workers) != len(keys): raise ValueError("You only provided %d worker restrictions" " for a sequence of length %d" % (len(workers), len(keys))) restrictions = dict(zip(keys, workers)) else: restrictions = {key: workers for key in keys} elif workers is None: restrictions = {} else: raise TypeError("Workers must be a list or set of workers or None") if allow_other_workers not in (True, False, None): raise TypeError("allow_other_workers= must be True or False") if allow_other_workers is True: loose_restrictions = set(keys) else: loose_restrictions = set() priority = dict(zip(keys, range(len(keys)))) futures = self._graph_to_futures(dsk, keys, restrictions, loose_restrictions, priority=priority) logger.debug("map(%s, ...)", funcname(func)) return [futures[tokey(key)] for key in keys] @gen.coroutine def _gather(self, futures, errors='raise'): futures2, keys = unpack_remotedata(futures, byte_keys=True) keys = [tokey(key) for key in keys] bad_data = dict() @gen.coroutine def wait(k): """ Want to stop the All(...) early if we find an error """ yield self.futures[k]['event'].wait() if self.futures[k]['status'] != 'finished': raise Exception() while True: logger.debug("Waiting on futures to clear before gather") with ignoring(Exception): yield All([wait(key) for key in keys if key in self.futures]) exceptions = set() bad_keys = set() for key in keys: if (key not in self.futures or self.futures[key]['status'] == 'error'): exceptions.add(key) if errors == 'raise': try: d = self.futures[key] six.reraise(type(d['exception']), d['exception'], d['traceback']) except KeyError: six.reraise(CancelledError, CancelledError(key), None) if errors == 'skip': bad_keys.add(key) bad_data[key] = None else: raise ValueError("Bad value, `errors=%s`" % errors) keys = [k for k in keys if k not in bad_keys] response = yield self.scheduler.gather(keys=keys) if response['status'] == 'error': logger.debug("Couldn't gather keys %s", response['keys']) self._send_to_scheduler({'op': 'missing-data', 'keys': response['keys']}) for key in response['keys']: self.futures[key]['event'].clear() else: break if bad_data and errors == 'skip' and isinstance(futures2, list): futures2 = [f for f in futures2 if f not in bad_data] data = response['data'] result = pack_data(futures2, merge(data, bad_data)) raise gen.Return(result) def _threaded_gather(self, qin, qout, **kwargs): """ Internal function for gathering Queue """ while True: L = [qin.get()] while qin.empty(): try: L.append(qin.get_nowait()) except Empty: break results = self.gather(L, **kwargs) for item in results: qout.put(item) def gather(self, futures, errors='raise', maxsize=0): """ Gather futures from distributed memory Accepts a future, nested container of futures, iterator, or queue. The return type will match the input type. Returns ------- Future results Examples -------- >>> from operator import add # doctest: +SKIP >>> c = Client('127.0.0.1:8787') # doctest: +SKIP >>> x = c.submit(add, 1, 2) # doctest: +SKIP >>> c.gather(x) # doctest: +SKIP 3 >>> c.gather([x, [x], x]) # support lists and dicts # doctest: +SKIP [3, [3], 3] >>> seq = c.gather(iter([x, x])) # support iterators # doctest: +SKIP >>> next(seq) # doctest: +SKIP 3 See Also -------- Client.scatter: Send data out to cluster """ if isqueue(futures): qout = pyQueue(maxsize=maxsize) t = Thread(target=self._threaded_gather, args=(futures, qout), kwargs={'errors': errors}) t.daemon = True t.start() return qout elif isinstance(futures, Iterator): return (self.gather(f, errors=errors) for f in futures) else: return sync(self.loop, self._gather, futures, errors=errors) @gen.coroutine def _scatter(self, data, workers=None, broadcast=False): if isinstance(workers, str): workers = [workers] if isinstance(data, dict) and not all(isinstance(k, (bytes, str)) for k in data): d = yield self._scatter(keymap(tokey, data), workers, broadcast) raise gen.Return({k: d[tokey(k)] for k in data}) if isinstance(data, dict): data2 = valmap(to_serialize, data) types = valmap(type, data) elif isinstance(data, (list, tuple, set, frozenset)): data2 = list(map(to_serialize, data)) types = list(map(type, data)) elif isinstance(data, (Iterable, Iterator)): data2 = list(map(to_serialize, data)) types = list(map(type, data)) else: raise TypeError("Don't know how to scatter %s" % type(data)) keys = yield self.scheduler.scatter(data=data2, workers=workers, client=self.id, broadcast=broadcast) if isinstance(data, dict): out = {k: Future(k, self) for k in keys} elif isinstance(data, (tuple, list, set, frozenset)): out = type(data)([Future(k, self) for k in keys]) elif isinstance(data, (Iterable, Iterator)): out = [Future(k, self) for k in keys] else: raise TypeError( "Input to scatter must be a list, iterator, or queue") for key in keys: self.futures[key]['status'] = 'finished' self.futures[key]['event'].set() if isinstance(types, list): for key, typ in zip(keys, types): self.futures[key]['type'] = typ elif isinstance(types, dict): for key in keys: self.futures[key]['type'] = types[key] raise gen.Return(out) def _threaded_scatter(self, q_or_i, qout, **kwargs): """ Internal function for scattering Iterable/Queue data """ while True: if isqueue(q_or_i): L = [q_or_i.get()] while not q_or_i.empty(): try: L.append(q_or_i.get_nowait()) except Empty: break else: try: L = [next(q_or_i)] except StopIteration as e: qout.put(e) break futures = self.scatter(L, **kwargs) for future in futures: qout.put(future) def scatter(self, data, workers=None, broadcast=False, maxsize=0): """ Scatter data into distributed memory This moves data from the local client process into the workers of the distributed scheduler. Note that it is often better to submit jobs to your workers to have them load the data rather than loading data locally and then scattering it out to them. Parameters ---------- data: list, iterator, dict, or Queue Data to scatter out to workers. Output type matches input type. workers: list of tuples (optional) Optionally constrain locations of data. Specify workers as hostname/port pairs, e.g. ``('127.0.0.1', 8787)``. broadcast: bool (defaults to False) Whether to send each data element to all workers. By default we round-robin based on number of cores. maxsize: int (optional) Maximum size of queue if using queues, 0 implies infinite Returns ------- List, dict, iterator, or queue of futures matching the type of input. Examples -------- >>> c = Client('127.0.0.1:8787') # doctest: +SKIP >>> c.scatter([1, 2, 3]) # doctest: +SKIP [, , ] >>> c.scatter({'x': 1, 'y': 2, 'z': 3}) # doctest: +SKIP {'x': , 'y': , 'z': } Constrain location of data to subset of workers >>> c.scatter([1, 2, 3], workers=[('hostname', 8788)]) # doctest: +SKIP Handle streaming sequences of data with iterators or queues >>> seq = c.scatter(iter([1, 2, 3])) # doctest: +SKIP >>> next(seq) # doctest: +SKIP , Broadcast data to all workers >>> [future] = c.scatter([element], broadcast=True) # doctest: +SKIP See Also -------- Client.gather: Gather data back to local process """ if isqueue(data) or isinstance(data, Iterator): logger.debug("Starting thread for streaming data") qout = pyQueue(maxsize=maxsize) t = Thread(target=self._threaded_scatter, args=(data, qout), kwargs={'workers': workers, 'broadcast': broadcast}) t.daemon = True t.start() if isqueue(data): return qout else: return queue_to_iterator(qout) else: return sync(self.loop, self._scatter, data, workers=workers, broadcast=broadcast) @gen.coroutine def _cancel(self, futures): keys = {tokey(f.key) for f in futures_of(futures)} yield self.scheduler.cancel(keys=list(keys), client=self.id) for k in keys: with ignoring(KeyError): del self.futures[k] def cancel(self, futures): """ Cancel running futures This stops future tasks from being scheduled if they have not yet run and deletes them if they have already run. After calling, this result and all dependent results will no longer be accessible Parameters ---------- futures: list of Futures """ return sync(self.loop, self._cancel, futures) @gen.coroutine def _publish_dataset(self, **kwargs): coroutines = [] for name, data in kwargs.items(): keys = [tokey(f.key) for f in futures_of(data)] coroutines.append(self.scheduler.publish_dataset(keys=keys, name=tokey(name), data=dumps(data), client=self.id)) outs = yield coroutines def publish_dataset(self, **kwargs): """ Publish named datasets to scheduler This stores a named reference to a dask collection or list of futures on the scheduler. These references are available to other Clients which can download the collection or futures with ``get_dataset``. Datasets are not immediately computed. You may wish to call ``Client.persist`` prior to publishing a dataset. Parameters ---------- kwargs: dict named collections to publish on the scheduler Examples -------- Publishing client: >>> df = dd.read_csv('s3://...') # doctest: +SKIP >>> df = c.persist(df) # doctest: +SKIP >>> c.publish_dataset(my_dataset=df) # doctest: +SKIP Receiving client: >>> c.list_datasets() # doctest: +SKIP ['my_dataset'] >>> df2 = c.get_dataset('my_dataset') # doctest: +SKIP Returns ------- None See Also -------- Client.list_datasets Client.get_dataset Client.unpublish_dataset Client.persist """ return sync(self.loop, self._publish_dataset, **kwargs) def unpublish_dataset(self, name): """ Remove named datasets from scheduler Examples -------- >>> c.list_datasets() # doctest: +SKIP ['my_dataset'] >>> c.unpublish_datasets('my_dataset') # doctest: +SKIP >>> c.list_datasets() # doctest: +SKIP [] See Also -------- Client.publish_dataset """ return sync(self.loop, self.scheduler.unpublish_dataset, name=name) def list_datasets(self): """ List named datasets available on the scheduler See Also -------- Client.publish_dataset Client.get_dataset """ return sync(self.loop, self.scheduler.list_datasets) @gen.coroutine def _get_dataset(self, name): out = yield self.scheduler.get_dataset(name=name, client=self.id) with temp_default_client(self): data = loads(out['data']) raise Return(data) def get_dataset(self, name): """ Get named dataset from the scheduler See Also -------- Client.publish_dataset Client.list_datasets """ return sync(self.loop, self._get_dataset, tokey(name)) @gen.coroutine def _run(self, function, *args, **kwargs): nanny = kwargs.pop('nanny', False) workers = kwargs.pop('workers', None) responses = yield self.scheduler.broadcast(msg=dict(op='run', function=dumps(function), args=dumps(args), kwargs=dumps(kwargs)), workers=workers, nanny=nanny) results = {} for key, resp in responses.items(): if resp['status'] == 'OK': results[key] = resp['result'] elif resp['status'] == 'error': raise loads(resp['exception']) raise Return(results) def run(self, function, *args, **kwargs): """ Run a function on all workers outside of task scheduling system This calls a function on all currently known workers immediately, blocks until those results come back, and returns the results asynchronously as a dictionary keyed by worker address. This method if generally used for side effects, such and collecting diagnostic information or installing libraries. Parameters ---------- function: callable *args: arguments for remote function **kwargs: keyword arguments for remote function workers: list Workers on which to run the function. Defaults to all known workers. Examples -------- >>> c.run(os.getpid) # doctest: +SKIP {'192.168.0.100:9000': 1234, '192.168.0.101:9000': 4321, '192.168.0.102:9000': 5555} Restrict computation to particular workers with the ``workers=`` keyword argument. >>> c.run(os.getpid, workers=['192.168.0.100:9000', ... '192.168.0.101:9000']) # doctest: +SKIP {'192.168.0.100:9000': 1234, '192.168.0.101:9000': 4321} """ return sync(self.loop, self._run, function, *args, **kwargs) def _graph_to_futures(self, dsk, keys, restrictions=None, loose_restrictions=None, allow_other_workers=True, priority=None): keyset = set(keys) flatkeys = list(map(tokey, keys)) futures = {key: Future(key, self) for key in keyset} values = {k for k, v in dsk.items() if isinstance(v, Future) and k not in keyset} if values: dsk = dask.optimize.inline(dsk, keys=values) d = {k: unpack_remotedata(v) for k, v in dsk.items()} extra_keys = set.union(*[v[1] for v in d.values()]) if d else set() dsk2 = str_graph({k: v[0] for k, v in d.items()}, extra_keys) dsk3 = {k: v for k, v in dsk2.items() if k is not v} if restrictions: restrictions = keymap(tokey, restrictions) restrictions = valmap(list, restrictions) if loose_restrictions is not None: loose_restrictions = list(map(tokey, loose_restrictions)) dependencies = {tokey(k): set(map(tokey, v[1])) for k, v in d.items()} for s in dependencies.values(): for v in s: if v not in self.futures: raise CancelledError(v) for k, v in dsk3.items(): dependencies[k] |= get_dependencies(dsk3, task=v) self._send_to_scheduler({'op': 'update-graph', 'tasks': valmap(dumps_task, dsk3), 'dependencies': valmap(list, dependencies), 'keys': list(flatkeys), 'restrictions': restrictions or {}, 'loose_restrictions': loose_restrictions, 'client': self.id, 'priority': priority}) return futures @gen.coroutine def _get(self, dsk, keys, restrictions=None, raise_on_error=True): futures = self._graph_to_futures(dsk, set(flatten([keys])), restrictions) packed = pack_data(keys, futures) try: result = yield self._gather(packed) except Exception as e: if raise_on_error: raise else: result = 'error', e raise gen.Return(result) finally: for f in futures.values(): f.release() if not raise_on_error: result = 'OK', result raise gen.Return(result) def get(self, dsk, keys, restrictions=None, loose_restrictions=None, **kwargs): """ Compute dask graph Parameters ---------- dsk: dict keys: object, or nested lists of objects restrictions: dict (optional) A mapping of {key: {set of worker hostnames}} that restricts where jobs can take place Examples -------- >>> from operator import add # doctest: +SKIP >>> c = Client('127.0.0.1:8787') # doctest: +SKIP >>> c.get({'x': (add, 1, 2)}, 'x') # doctest: +SKIP 3 See Also -------- Client.compute: Compute asynchronous collections """ futures = self._graph_to_futures(dsk, set(flatten([keys])), restrictions, loose_restrictions) try: results = self.gather(futures) except (KeyboardInterrupt, Exception) as e: for f in futures.values(): f.release() raise results2 = pack_data(keys, results) return results2 def _optimize_insert_futures(self, dsk, keys): """ Replace known keys in dask graph with Futures When given a Dask graph that might have overlapping keys with our known results we replace the values of that graph with futures. This can be used as an optimization to avoid recomputation. This returns the same graph if unchanged but a new graph if any changes were necessary. """ changed = False for key in list(dsk): if tokey(key) in self.futures: if not changed: changed = True dsk = dsk.copy() dsk[key] = Future(key, self) if changed: dsk, _ = dask.optimize.cull(dsk, keys) return dsk def normalize_collection(self, collection): """ Replace collection's tasks by already existing futures if they exist This normalizes the tasks within a collections task graph against the known futures within the scheduler. It returns a copy of the collection with a task graph that includes the overlapping futures. Examples -------- >>> len(x.dask) # x is a dask collection with 100 tasks 100 >>> set(client.futures).intersection(x.dask) # some overlap exists 10 >>> x = client.normalize_collection(x) >>> len(x.dask) # smaller computational graph 20 See Also -------- Client.persist: trigger computation of collection's tasks """ dsk = self._optimize_insert_futures(collection.dask, collection._keys()) if dsk is collection.dask: return collection else: return redict_collection(collection, dsk) def compute(self, collections, sync=False, optimize_graph=True, workers=None, allow_other_workers=False, **kwargs): """ Compute dask collections on cluster Parameters ---------- collections: iterable of dask objects or single dask object Collections like dask.array or dataframe or dask.value objects sync: bool (optional) Returns Futures if False (default) or concrete values if True optimize_graph: bool Whether or not to optimize the underlying graphs workers: str, list, dict Which workers can run which parts of the computation If a string a list then the output collections will run on the listed workers, but other sub-computations can run anywhere If a dict then keys should be (tuples of) collections and values should be addresses or lists. allow_other_workers: bool, list If True then all restrictions in workers= are considered loose If a list then only the keys for the listed collections are loose **kwargs: Options to pass to the graph optimize calls Returns ------- List of Futures if input is a sequence, or a single future otherwise Examples -------- >>> from dask import do, value >>> from operator import add >>> x = dask.do(add)(1, 2) >>> y = dask.do(add)(x, x) >>> xx, yy = client.compute([x, y]) # doctest: +SKIP >>> xx # doctest: +SKIP >>> xx.result() # doctest: +SKIP 3 >>> yy.result() # doctest: +SKIP 6 Also support single arguments >>> xx = client.compute(x) # doctest: +SKIP See Also -------- Client.get: Normal synchronous dask.get function """ if isinstance(collections, (list, tuple, set, frozenset)): singleton = False else: collections = [collections] singleton = True variables = [a for a in collections if isinstance(a, Base)] dsk = collections_to_dsk(variables, optimize_graph, **kwargs) names = ['finalize-%s' % tokenize(v) for v in variables] dsk2 = {name: (v._finalize, v._keys()) for name, v in zip(names, variables)} restrictions, loose_restrictions = get_restrictions(collections, workers, allow_other_workers) futures_dict = self._graph_to_futures(merge(dsk2, dsk), names, restrictions, loose_restrictions) i = 0 futures = [] for arg in collections: if isinstance(arg, Base): futures.append(futures_dict[names[i]]) i += 1 else: futures.append(arg) if sync: result = self.gather(futures) else: result = futures if singleton: return first(result) else: return result def persist(self, collections, optimize_graph=True, workers=None, allow_other_workers=None, **kwargs): """ Persist dask collections on cluster Starts computation of the collection on the cluster in the background. Provides a new dask collection that is semantically identical to the previous one, but now based off of futures currently in execution. Parameters ---------- collections: sequence or single dask object Collections like dask.array or dataframe or dask.value objects optimize_graph: bool Whether or not to optimize the underlying graphs workers: str, list, dict Which workers can run which parts of the computation If a string a list then the output collections will run on the listed workers, but other sub-computations can run anywhere If a dict then keys should be (tuples of) collections and values should be addresses or lists. allow_other_workers: bool, list If True then all restrictions in workers= are considered loose If a list then only the keys for the listed collections are loose kwargs: Options to pass to the graph optimize calls Returns ------- List of collections, or single collection, depending on type of input. Examples -------- >>> xx = client.persist(x) # doctest: +SKIP >>> xx, yy = client.persist([x, y]) # doctest: +SKIP See Also -------- Client.compute """ if isinstance(collections, (tuple, list, set, frozenset)): singleton = False else: singleton = True collections = [collections] assert all(isinstance(c, Base) for c in collections) dsk = collections_to_dsk(collections, optimize_graph, **kwargs) names = {k for c in collections for k in flatten(c._keys())} restrictions, loose_restrictions = get_restrictions(collections, workers, allow_other_workers) futures = self._graph_to_futures(dsk, names, restrictions, loose_restrictions) result = [redict_collection(c, {k: futures[k] for k in flatten(c._keys())}) for c in collections] if singleton: return first(result) else: return result @gen.coroutine def _upload_environment(self, zipfile): name = os.path.split(zipfile)[1] yield self._upload_large_file(zipfile, name) def unzip(dask_worker=None): from distributed.utils import log_errors import zipfile import shutil with log_errors(): a = os.path.join(dask_worker.worker_dir, name) b = os.path.join(dask_worker.local_dir, name) c = os.path.dirname(b) shutil.move(a, b) with zipfile.ZipFile(b) as f: f.extractall(path=c) for fn in glob(os.path.join(c, name[:-4], 'bin', '*')): st = os.stat(fn) os.chmod(fn, st.st_mode | 64) # chmod u+x fn assert os.path.exists(os.path.join(c, name[:-4])) return c responses = yield self._run(unzip, nanny=True) raise gen.Return(name[:-4]) def upload_environment(self, name, zipfile): return sync(self.loop, self._upload_environment, name, zipfile) @gen.coroutine def _restart(self, environment=None): if environment: environment = yield self._upload_environment(environment) self._send_to_scheduler({'op': 'restart', 'environment': environment}) self._restart_event = Event() yield self._restart_event.wait() self.generation += 1 self.refcount.clear() raise gen.Return(self) def restart(self, environment=None): """ Restart the distributed network This kills all active work, deletes all data on the network, and restarts the worker processes. """ return sync(self.loop, self._restart, environment=environment) @gen.coroutine def _upload_file(self, filename, raise_on_error=True): with open(filename, 'rb') as f: data = f.read() _, fn = os.path.split(filename) d = yield self.scheduler.broadcast(msg={'op': 'upload_file', 'filename': fn, 'data': to_serialize(data)}) if any(v['status'] == 'error' for v in d.values()): exceptions = [loads(v['exception']) for v in d.values() if v['status'] == 'error'] if raise_on_error: raise exceptions[0] else: raise gen.Return(exceptions[0]) assert all(len(data) == v['nbytes'] for v in d.values()) @gen.coroutine def _upload_large_file(self, local_filename, remote_filename=None): if remote_filename is None: remote_filename = os.path.split(local_filename)[1] with open(local_filename, 'rb') as f: data = f.read() [future] = yield self._scatter([data]) key = future.key yield self._replicate(future) def dump_to_file(dask_worker=None): if not os.path.isabs(remote_filename): fn = os.path.join(dask_worker.local_dir, remote_filename) else: fn = remote_filename with open(fn, 'wb') as f: f.write(dask_worker.data[key]) return len(dask_worker.data[key]) response = yield self._run(dump_to_file) assert all(len(data) == v for v in response.values()) def upload_file(self, filename): """ Upload local package to workers This sends a local file up to all worker nodes. This file is placed into a temporary directory on Python's system path so any .py or .egg files will be importable. Parameters ---------- filename: string Filename of .py or .egg file to send to workers Examples -------- >>> client.upload_file('mylibrary.egg') # doctest: +SKIP >>> from mylibrary import myfunc # doctest: +SKIP >>> L = c.map(myfunc, seq) # doctest: +SKIP """ result = sync(self.loop, self._upload_file, filename, raise_on_error=False) if isinstance(result, Exception): raise result @gen.coroutine def _rebalance(self, futures=None, workers=None): yield _wait(futures) keys = list({tokey(f.key) for f in self.futures_of(futures)}) result = yield self.scheduler.rebalance(keys=keys, workers=workers) assert result['status'] == 'OK' def rebalance(self, futures=None, workers=None): """ Rebalance data within network Move data between workers to roughly balance memory burden. This either affects a subset of the keys/workers or the entire network, depending on keyword arguments. This operation is generally not well tested against normal operation of the scheduler. It it not recommended to use it while waiting on computations. Parameters ---------- futures: list, optional A list of futures to balance, defaults all data workers: list, optional A list of workers on which to balance, defaults to all workers """ sync(self.loop, self._rebalance, futures, workers), @gen.coroutine def _replicate(self, futures, n=None, workers=None, branching_factor=2): futures = self.futures_of(futures) yield _wait(futures) keys = {tokey(f.key) for f in futures} yield self.scheduler.replicate(keys=list(keys), n=n, workers=workers, branching_factor=branching_factor) def replicate(self, futures, n=None, workers=None, branching_factor=2): """ Set replication of futures within network Copy data onto many workers. This helps to broadcast frequently accessed data and it helps to improve resilience. This performs a tree copy of the data throughout the network individually on each piece of data. This operation blocks until complete. It does not guarantee replication of data to future workers. Parameters ---------- futures: list of futures Futures we wish to replicate n: int, optional Number of processes on the cluster on which to replicate the data. Defaults to all. workers: list of worker addresses Workers on which we want to restrict the replication. Defaults to all. branching_factor: int, optional The number of workers that can copy data in each generation Examples -------- >>> x = c.submit(func, *args) # doctest: +SKIP >>> c.replicate([x]) # send to all workers # doctest: +SKIP >>> c.replicate([x], n=3) # send to three workers # doctest: +SKIP >>> c.replicate([x], workers=['alice', 'bob']) # send to specific # doctest: +SKIP >>> c.replicate([x], n=1, workers=['alice', 'bob']) # send to one of specific workers # doctest: +SKIP >>> c.replicate([x], n=1) # reduce replications # doctest: +SKIP See also -------- Client.rebalance """ sync(self.loop, self._replicate, futures, n=n, workers=workers, branching_factor=branching_factor) def ncores(self, workers=None): """ The number of threads/cores available on each worker node Parameters ---------- workers: list (optional) A list of workers that we care about specifically. Leave empty to receive information about all workers. Examples -------- >>> c.ncores() # doctest: +SKIP {'192.168.1.141:46784': 8, '192.167.1.142:47548': 8, '192.167.1.143:47329': 8, '192.167.1.144:37297': 8} See Also -------- Client.who_has Client.has_what """ if (isinstance(workers, tuple) and all(isinstance(i, (str, tuple)) for i in workers)): workers = list(workers) if workers is not None and not isinstance(workers, (list, set)): workers = [workers] return sync(self.loop, self.scheduler.ncores, workers=workers) def who_has(self, futures=None): """ The workers storing each future's data Parameters ---------- futures: list (optional) A list of futures, defaults to all data Examples -------- >>> x, y, z = c.map(inc, [1, 2, 3]) # doctest: +SKIP >>> wait([x, y, z]) # doctest: +SKIP >>> c.who_has() # doctest: +SKIP {'inc-1c8dd6be1c21646c71f76c16d09304ea': ['192.168.1.141:46784'], 'inc-1e297fc27658d7b67b3a758f16bcf47a': ['192.168.1.141:46784'], 'inc-fd65c238a7ea60f6a01bf4c8a5fcf44b': ['192.168.1.141:46784']} >>> c.who_has([x, y]) # doctest: +SKIP {'inc-1c8dd6be1c21646c71f76c16d09304ea': ['192.168.1.141:46784'], 'inc-1e297fc27658d7b67b3a758f16bcf47a': ['192.168.1.141:46784']} See Also -------- Client.has_what Client.ncores """ if futures is not None: futures = self.futures_of(futures) keys = list({f.key for f in futures}) else: keys = None return sync(self.loop, self.scheduler.who_has, keys=keys) def has_what(self, workers=None): """ Which keys are held by which workers Parameters ---------- workers: list (optional) A list of worker addresses, defaults to all Examples -------- >>> x, y, z = c.map(inc, [1, 2, 3]) # doctest: +SKIP >>> wait([x, y, z]) # doctest: +SKIP >>> c.has_what() # doctest: +SKIP {'192.168.1.141:46784': ['inc-1c8dd6be1c21646c71f76c16d09304ea', 'inc-fd65c238a7ea60f6a01bf4c8a5fcf44b', 'inc-1e297fc27658d7b67b3a758f16bcf47a']} See Also -------- Client.who_has Client.ncores """ if (isinstance(workers, tuple) and all(isinstance(i, (str, tuple)) for i in workers)): workers = list(workers) if workers is not None and not isinstance(workers, (list, set)): workers = [workers] return sync(self.loop, self.scheduler.has_what, workers=workers) def stacks(self, workers=None): """ The task queues on each worker Parameters ---------- workers: list (optional) A list of worker addresses, defaults to all Examples -------- >>> x, y, z = c.map(inc, [1, 2, 3]) # doctest: +SKIP >>> c.stacks() # doctest: +SKIP {'192.168.1.141:46784': ['inc-1c8dd6be1c21646c71f76c16d09304ea', 'inc-fd65c238a7ea60f6a01bf4c8a5fcf44b', 'inc-1e297fc27658d7b67b3a758f16bcf47a']} See Also -------- Client.processing Client.who_has Client.has_what Client.ncores """ if (isinstance(workers, tuple) and all(isinstance(i, (str, tuple)) for i in workers)): workers = list(workers) if workers is not None and not isinstance(workers, (list, set)): workers = [workers] return sync(self.loop, self.scheduler.stacks, workers=workers) def processing(self, workers=None): """ The tasks currently running on each worker Parameters ---------- workers: list (optional) A list of worker addresses, defaults to all Examples -------- >>> x, y, z = c.map(inc, [1, 2, 3]) # doctest: +SKIP >>> c.processing() # doctest: +SKIP {'192.168.1.141:46784': ['inc-1c8dd6be1c21646c71f76c16d09304ea', 'inc-fd65c238a7ea60f6a01bf4c8a5fcf44b', 'inc-1e297fc27658d7b67b3a758f16bcf47a']} See Also -------- Client.stacks Client.who_has Client.has_what Client.ncores """ if (isinstance(workers, tuple) and all(isinstance(i, (str, tuple)) for i in workers)): workers = list(workers) if workers is not None and not isinstance(workers, (list, set)): workers = [workers] return valmap(set, sync(self.loop, self.scheduler.processing, workers=workers)) def nbytes(self, keys=None, summary=True): """ The bytes taken up by each key on the cluster This is as measured by ``sys.getsizeof`` which may not accurately reflect the true cost. Parameters ---------- keys: list (optional) A list of keys, defaults to all keys summary: boolean, (optional) Summarize keys into key types Examples -------- >>> x, y, z = c.map(inc, [1, 2, 3]) # doctest: +SKIP >>> c.nbytes(summary=False) # doctest: +SKIP {'inc-1c8dd6be1c21646c71f76c16d09304ea': 28, 'inc-1e297fc27658d7b67b3a758f16bcf47a': 28, 'inc-fd65c238a7ea60f6a01bf4c8a5fcf44b': 28} >>> c.nbytes(summary=True) # doctest: +SKIP {'inc': 84} See Also -------- Client.who_has """ return sync(self.loop, self.scheduler.nbytes, keys=keys, summary=summary) def scheduler_info(self): """ Basic information about the workers in the cluster Examples -------- >>> c.scheduler_info() # doctest: +SKIP {'id': '2de2b6da-69ee-11e6-ab6a-e82aea155996', 'services': {}, 'type': 'Scheduler', 'workers': {'127.0.0.1:40575': {'active': 0, 'last-seen': 1472038237.4845693, 'name': '127.0.0.1:40575', 'services': {}, 'stored': 0, 'time-delay': 0.0061032772064208984}}} """ return sync(self.loop, self.scheduler.identity) def get_versions(self): """ Return version info for the scheduler, all workers and myself Examples -------- >>> c.get_versions() # doctest: +SKIP """ client = get_versions() try: scheduler = sync(self.loop, self.scheduler.versions) except KeyError: scheduler = None def f(worker=None): # use our local version try: from distributed.versions import get_versions return get_versions() except ImportError: return None workers = sync(self.loop, self._run, f) return {'scheduler': scheduler, 'workers': workers, 'client': client} def futures_of(self, futures): return futures_of(futures, client=self) def start_ipython(self, *args, **kwargs): raise Exception("Method moved to start_ipython_workers") @gen.coroutine def _start_ipython_workers(self, workers): if workers is None: workers = yield self.scheduler.ncores() responses = yield self.scheduler.broadcast( msg=dict(op='start_ipython'), workers=workers, ) raise gen.Return((workers, responses)) def start_ipython_workers(self, workers=None, magic_names=False, qtconsole=False, qtconsole_args=None): """ Start IPython kernels on workers Parameters ---------- workers: list (optional) A list of worker addresses, defaults to all magic_names: str or list(str) (optional) If defined, register IPython magics with these names for executing code on the workers. If string has asterix then expand asterix into 0, 1, ..., n for n workers qtconsole: bool (optional) If True, launch a Jupyter QtConsole connected to the worker(s). qtconsole_args: list(str) (optional) Additional arguments to pass to the qtconsole on startup. Examples -------- >>> info = c.start_ipython_workers() # doctest: +SKIP >>> %remote info['192.168.1.101:5752'] worker.data # doctest: +SKIP {'x': 1, 'y': 100} >>> c.start_ipython_workers('192.168.1.101:5752', magic_names='w') # doctest: +SKIP >>> %w worker.data # doctest: +SKIP {'x': 1, 'y': 100} >>> c.start_ipython_workers('192.168.1.101:5752', qtconsole=True) # doctest: +SKIP Add asterix * in magic names to add one magic per worker >>> c.start_ipython_workers(magic_names='w_*') # doctest: +SKIP >>> %w_0 worker.data # doctest: +SKIP {'x': 1, 'y': 100} >>> %w_1 worker.data # doctest: +SKIP {'z': 5} Returns ------- iter_connection_info: list List of connection_info dicts containing info necessary to connect Jupyter clients to the workers. See Also -------- Client.start_ipython_scheduler: start ipython on the scheduler """ if isinstance(workers, six.string_types): workers = [workers] (workers, info_dict) = sync(self.loop, self._start_ipython_workers, workers) if magic_names and isinstance(magic_names, six.string_types): if '*' in magic_names: magic_names = [magic_names.replace('*', str(i)) for i in range(len(workers))] else: magic_names = [magic_names] if 'IPython' in sys.modules: from ._ipython_utils import register_remote_magic register_remote_magic() if magic_names: from ._ipython_utils import register_worker_magic for worker, magic_name in zip(workers, magic_names): connection_info = info_dict[worker] register_worker_magic(connection_info, magic_name) if qtconsole: from ._ipython_utils import connect_qtconsole for worker, connection_info in info_dict.items(): connect_qtconsole(connection_info, name='dask-' + worker.replace(':','-'), extra_args=qtconsole_args, ) return info_dict def start_ipython_scheduler(self, magic_name='scheduler_if_ipython', qtconsole=False, qtconsole_args=None): """ Start IPython kernel on the scheduler Parameters ---------- magic_name: str or None (optional) If defined, register IPython magic with this name for executing code on the scheduler. If not defined, register %scheduler magic if IPython is running. qtconsole: bool (optional) If True, launch a Jupyter QtConsole connected to the worker(s). qtconsole_args: list(str) (optional) Additional arguments to pass to the qtconsole on startup. Examples -------- >>> c.start_ipython_scheduler() # doctest: +SKIP >>> %scheduler scheduler.processing # doctest: +SKIP {'127.0.0.1:3595': {'inc-1', 'inc-2'}, '127.0.0.1:53589': {'inc-2', 'add-5'}} >>> c.start_ipython_scheduler(qtconsole=True) # doctest: +SKIP Returns ------- connection_info: dict connection_info dict containing info necessary to connect Jupyter clients to the scheduler. See Also -------- Client.start_ipython_workers: Start IPython on the workers """ info = sync(self.loop, self.scheduler.start_ipython) if magic_name == 'scheduler_if_ipython': # default to %scheduler if in IPython, no magic otherwise in_ipython = False if 'IPython' in sys.modules: from IPython import get_ipython in_ipython = bool(get_ipython()) if in_ipython: magic_name = 'scheduler' else: magic_name = None if magic_name: from ._ipython_utils import register_worker_magic register_worker_magic(info, magic_name) if qtconsole: from ._ipython_utils import connect_qtconsole connect_qtconsole(info, name='dask-scheduler', extra_args=qtconsole_args,) return info Executor = Client class CompatibleExecutor(Client): """ A concurrent.futures-compatible Client A subclass of Client that conforms to concurrent.futures API, allowing swapping in for other Clients. """ def map(self, func, *iterables, **kwargs): """ Map a function on a sequence of arguments Returns ------- iter_results: iterable Iterable yielding results of the map. See Also -------- Client.map: for more info """ list_of_futures = super(CompatibleExecutor, self).map( func, *iterables, **kwargs) for f in list_of_futures: yield f.result() @gen.coroutine def _wait(fs, timeout=None, return_when='ALL_COMPLETED'): fs = futures_of(fs) if timeout is not None: raise NotImplementedError("Timeouts not yet supported") if return_when == 'ALL_COMPLETED': try: yield All({f.event.wait() for f in fs}) except KeyError: raise CancelledError([f.key for f in fs if f.key not in f.client.futures]) done, not_done = set(fs), set() else: raise NotImplementedError("Only return_when='ALL_COMPLETED' supported") raise gen.Return(DoneAndNotDoneFutures(done, not_done)) ALL_COMPLETED = 'ALL_COMPLETED' def wait(fs, timeout=None, return_when='ALL_COMPLETED'): """ Wait until all futures are complete Parameters ---------- fs: list of futures Returns ------- Named tuple of completed, not completed """ client = default_client() result = sync(client.loop, _wait, fs, timeout, return_when) return result @gen.coroutine def _as_completed(fs, queue): fs = futures_of(fs) groups = groupby(lambda f: f.key, fs) firsts = [v[0] for v in groups.values()] wait_iterator = gen.WaitIterator(*[f.event.wait() for f in firsts]) while not wait_iterator.done(): yield wait_iterator.next() # TODO: handle case of restarted futures future = firsts[wait_iterator.current_index] for f in groups[future.key]: queue.put_nowait(f) @gen.coroutine def _first_completed(futures): """ Return a single completed future See Also: _as_completed """ q = Queue() yield _as_completed(futures, q) result = yield q.get() raise gen.Return(result) def as_completed(fs): """ Return futures in the order in which they complete This returns an iterator that yields the input future objects in the order in which they complete. Calling ``next`` on the iterator will block until the next future completes, irrespective of order. This function does not return futures in the order in which they are input. """ fs = list(fs) if len(set(f.client for f in fs)) == 1: loop = first(fs).client.loop else: # TODO: Groupby client, spawn many _as_completed coroutines raise NotImplementedError( "as_completed on many event loops not yet supported") queue = pyQueue() coroutine = lambda: _as_completed(fs, queue) loop.add_callback(coroutine) for i in range(len(fs)): yield queue.get() def default_client(c=None): """ Return an client if exactly one has started """ if c: return c if _global_client[0]: return _global_client[0] else: raise ValueError("No clients found\n" "Start an client and point it to the scheduler address\n" " from distributed import Client\n" " client = Client('ip-addr-of-scheduler:8786')\n") def ensure_default_get(client): if _globals['get'] != client.get: print("Setting global dask scheduler to use distributed") dask.set_options(get=client.get) def redict_collection(c, dsk): from dask.delayed import Delayed if isinstance(c, Delayed): return Delayed(c.key, [dsk]) else: cc = copy.copy(c) cc.dask = dsk return cc def futures_of(o, client=None): """ Future objects in a collection """ stack = [o] futures = set() while stack: x = stack.pop() if type(x) in (tuple, set, list): stack.extend(x) if type(x) is dict: stack.extend(x.values()) if type(x) is Future: futures.add(x) if hasattr(x, 'dask'): stack.extend(x.dask.values()) if client is not None: bad = {f for f in futures if f.cancelled()} if bad: raise CancelledError(bad) return list(futures) @contextmanager def temp_default_client(c): """ Set the default client for the duration of the context Parameters ---------- c : Client This is what default_client() will return within the with-block. """ old_exec = default_client() _global_client[0] = c try: yield finally: _global_client[0] = old_exec def get_restrictions(collections, workers, allow_other_workers): """ Get restrictions from inputs to compute/persist """ if isinstance(workers, (str, tuple, list)): workers = {tuple(collections): workers} if isinstance(workers, dict): restrictions = {} for colls, ws in workers.items(): if isinstance(ws, str): ws = [ws] if hasattr(colls, '._keys'): keys = flatten(colls._keys()) else: keys = list({k for c in flatten(colls) for k in flatten(c._keys())}) restrictions.update({k: ws for k in keys}) else: restrictions = {} if allow_other_workers is True: loose_restrictions = list(restrictions) elif allow_other_workers: loose_restrictions = list({k for c in flatten(allow_other_workers) for k in c._keys()}) else: loose_restrictions = [] return restrictions, loose_restrictions def collections_to_dsk(collections, optimize_graph=True, **kwargs): """ Convert many collections into a single dask graph, after optimization """ optimizations = _globals.get('optimizations', []) if optimize_graph: groups = groupby(lambda x: x._optimize, collections) groups = {opt: [merge([v.dask for v in val]), [v._keys() for v in val]] for opt, val in groups.items()} for opt in optimizations: groups = {k: [opt(dsk, keys), keys] for k, (dsk, keys) in groups.items()} dsk = merge([opt(dsk, keys, **kwargs) for opt, (dsk, keys) in groups.items()]) else: dsk = merge(c.dask for c in collections) return dsk distributed-1.14.3/distributed/collections.py000066400000000000000000000171671301211143400213360ustar00rootroot00000000000000""" This file is experimental and may disappear without warning """ from __future__ import print_function, division, absolute_import from itertools import product import dask.array as da import dask.bag as db import dask.dataframe as dd from dask.base import tokenize import numpy as np from tornado import gen from .client import (default_client, Future, _first_completed, ensure_default_get) from .utils import sync def index_min(df): return df.index.min() def index_max(df): return df.index.max() def get_empty(df): return df.iloc[0:0] @gen.coroutine def _futures_to_dask_dataframe(futures, divisions=None, client=None): client = default_client(client) f = yield _first_completed(futures) empty = client.submit(get_empty, f) if divisions is True: divisions = client.map(index_min, futures) divisions.append(client.submit(index_max, futures[-1])) divisions2 = yield client._gather(divisions) if sorted(divisions2) != divisions2: divisions2 = [None] * (len(futures) + 1) elif divisions in (None, False): divisions2 = [None] * (len(futures) + 1) else: raise NotImplementedError() empty = yield empty._result() name = 'distributed-pandas-to-dask-' + tokenize(*futures) dsk = {(name, i): f for i, f in enumerate(futures)} ensure_default_get(client) raise gen.Return(dd.DataFrame(dsk, name, empty, divisions2)) def futures_to_dask_dataframe(futures, divisions=None, client=None): """ Convert a list of futures into a dask.dataframe Parameters ---------- client: Client Client through which we access the remote dataframes futures: iterable of Futures Futures that create dataframes to form the divisions divisions: bool Set to True if the data is cleanly partitioned along the index """ client = default_client(client) return sync(client.loop, _futures_to_dask_dataframe, futures, divisions=divisions, client=client) def get_shape(x): return x.shape def get_dim(x, i): return x.shape[i] def get_dtype(x): return x.dtype @gen.coroutine def _futures_to_dask_array(futures, client=None): client = default_client(client) futures = np.array(futures, dtype=object) slices = [((0,) * i + (slice(None, None),) + (0,) * (futures.ndim - i - 1)) for i in range(futures.ndim)] chunks = [[client.submit(get_dim, x, i) for x in futures[slc]] for i, slc in enumerate(slices)] dtype = client.submit(get_dtype, futures.flat[0]) chunks, dtype = yield client._gather([chunks, dtype]) chunks = tuple(map(tuple, chunks)) name = 'array-from-futures-' + tokenize(*futures.flat) keys = list(product([name], *map(range, futures.shape))) values = list(futures.flat) dsk = dict(zip(keys, values)) ensure_default_get(client) raise gen.Return(da.Array(dsk, name, chunks, dtype)) def futures_to_dask_array(futures, client=None): """ Convert a nested list of futures into a dask.array The futures must satisfy the following: * Evaluate to numpy ndarrays * Be in a nested list of the same depth as the intended dimension * Serve as individual chunks of the full dask array * Have shapes consistent with how dask.array sets up chunks (all dimensions consistent across any particular direction) This function will block until all futures are accessible. It queries the futures directly for shape and dtype information. Parameters ---------- futures: iterable of Futures Futures that create arrays client: Client (optional) Client through which we access the remote dataframes """ client = default_client(client) return sync(client.loop, _futures_to_dask_array, futures, client=client) @gen.coroutine def _futures_to_dask_bag(futures, client=None): client = default_client(client) name = 'bag-from-futures-' + tokenize(*futures) dsk = {(name, i): future for i, future in enumerate(futures)} ensure_default_get(client) raise gen.Return(db.Bag(dsk, name, len(futures))) def futures_to_dask_bag(futures, client=None): """ Convert a list of futures into a dask.bag The futures should be point to Python iterables Parameters ---------- futures: iterable of Futures client: Client (optional) """ client = default_client(client) return sync(client.loop, _futures_to_dask_bag, futures, client=client) @gen.coroutine def _futures_to_collection(futures, client=None, **kwargs): client = default_client(client) element = futures while not isinstance(element, Future): element = element[0] typ = yield client.submit(type, element)._result() if 'pandas' in typ.__module__: func = _futures_to_dask_dataframe elif 'numpy' in typ.__module__: func = _futures_to_dask_array elif issubclass(typ, (tuple, list, set, frozenset)): func = _futures_to_dask_bag else: raise NotImplementedError("First future of type %s. Expected " "numpy or pandas object" % typ.__name__) result = yield func(futures, client=client, **kwargs) raise gen.Return(result) def futures_to_collection(futures, client=None, **kwargs): """ Convert futures into a dask collection Convert a list of futures pointing to a pandas dataframe into one logical dask.dataframe or a nested list of numpy arrays into a logical dask.array. The type of the collection will be chosen based on the type of the futures. See also -------- futures_to_dask_array futures_to_dask_dataframe """ client = default_client(client) return sync(client.loop, _futures_to_collection, futures, client=client, **kwargs) @gen.coroutine def _future_to_dask_array(future, client=None): e = default_client(client) name = 'array-' + str(future.key) shape, dtype = yield e._gather([e.submit(get_shape, future), e.submit(get_dtype, future)]) dsk = {(name,) + (0,) * len(shape): future} chunks = tuple((d,) for d in shape) ensure_default_get(e) raise gen.Return(da.Array(dsk, name, chunks, dtype)) def future_to_dask_array(future, client=None): """ Convert a single future to a single-chunked dask.array See Also: dask.array.stack dask.array.concatenate futures_to_dask_array futures_to_dask_arrays """ client = default_client(client) return sync(client.loop, _future_to_dask_array, future, client=client) @gen.coroutine def _futures_to_dask_arrays(futures, client=None): e = default_client(client) names = ['array-' + str(future.key) for future in futures] shapes, dtypes = yield e._gather([e.map(get_shape, futures), e.map(get_dtype, futures)]) dsks = [{(name,) + (0,) * len(shape): future} for name, shape, future in zip(names, shapes, futures)] chunkss = [tuple((d,) for d in shape) for shape in shapes] ensure_default_get(e) raise gen.Return([da.Array(dsk, name, chunks, dtype) for dsk, name, chunks, dtype in zip(dsks, names, chunkss, dtypes)]) def futures_to_dask_arrays(futures, client=None): """ Convert a list of futures into a list of dask arrays Computation will be done to evaluate the shape and dtype of the futures. See Also: dask.array.stack dask.array.concatenate future_to_dask_array futures_to_dask_array """ client = default_client(client) return sync(client.loop, _futures_to_dask_arrays, futures, client=client) distributed-1.14.3/distributed/compatibility.py000066400000000000000000000033361301211143400216620ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import logging import sys if sys.version_info[0] == 2: from Queue import Queue, Empty from io import BytesIO from thread import get_ident as get_thread_identity reload = reload unicode = unicode PY2 = True PY3 = False ConnectionRefusedError = OSError FileExistsError = OSError import gzip def gzip_decompress(b): f = gzip.GzipFile(fileobj=BytesIO(b)) result = f.read() f.close() return result def gzip_compress(b): bio = BytesIO() f = gzip.GzipFile(fileobj=bio, mode='w') f.write(b) f.close() bio.seek(0) result = bio.read() return result def isqueue(o): return (hasattr(o, 'queue') and hasattr(o, '__module__') and o.__module__ == 'Queue') logging_names = logging._levelNames if sys.version_info[0] == 3: from queue import Queue, Empty from importlib import reload from threading import get_ident as get_thread_identity PY2 = False PY3 = True unicode = str from gzip import decompress as gzip_decompress from gzip import compress as gzip_compress ConnectionRefusedError = ConnectionRefusedError FileExistsError = FileExistsError def isqueue(o): return isinstance(o, Queue) logging_names = logging._levelToName.copy() logging_names.update(logging._nameToLevel) WINDOWS = sys.platform.startswith('win') try: from json.decoder import JSONDecodeError except (ImportError, AttributeError): JSONDecodeError = ValueError try: from functools import singledispatch except ImportError: from singledispatch import singledispatch distributed-1.14.3/distributed/config.py000066400000000000000000000020021301211143400202430ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import os from .compatibility import FileExistsError try: import yaml except ImportError: config = {} else: dirname = os.path.dirname(__file__) default_path = os.path.join(dirname, 'config.yaml') dask_config_path = os.path.join(os.path.expanduser('~'), '.dask', 'config.yaml') def ensure_config_file(destination=dask_config_path): if not os.path.exists(destination): import shutil if not os.path.exists(os.path.dirname(destination)): try: os.mkdir(os.path.dirname(destination)) except FileExistsError: pass shutil.copy(default_path, destination) def load_config_file(path=dask_config_path): if not os.path.exists(path): path = default_path with open(path) as f: text = f.read() return yaml.load(text) ensure_config_file() config = load_config_file() distributed-1.14.3/distributed/config.yaml000066400000000000000000000005461301211143400205700ustar00rootroot00000000000000logging: distributed: info distributed.client: warning bokeh: critical compression: auto # Scheduler options bandwidth: 100000000 # 100 MB/s estimated worker-worker bandwidth allowed-failures: 3 # number of retries before a task is considered bad pdb-on-err: False # enter debug mode on scheduling error transition-log-length: 100000 distributed-1.14.3/distributed/core.py000066400000000000000000000502211301211143400177340ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from collections import defaultdict from datetime import timedelta import logging import six import socket import struct from time import time import traceback import uuid from toolz import assoc, first try: import cPickle as pickle except ImportError: import pickle import cloudpickle from tornado import gen from tornado.locks import Event from tornado.tcpserver import TCPServer from tornado.tcpclient import TCPClient from tornado.ioloop import IOLoop from tornado.iostream import IOStream, StreamClosedError from .compatibility import PY3, unicode, WINDOWS from .utils import get_traceback, truncate_exception, ignoring from . import protocol class RPCClosed(IOError): pass logger = logging.getLogger(__name__) def get_total_physical_memory(): try: import psutil return psutil.virtual_memory().total / 2 except ImportError: return 2e9 MAX_BUFFER_SIZE = get_total_physical_memory() def handle_signal(sig, frame): IOLoop.instance().add_callback(IOLoop.instance().stop) class Server(TCPServer): """ Distributed TCP Server Superclass for both Worker and Scheduler objects. Inherits from ``tornado.tcpserver.TCPServer``, adding a protocol for RPC. **Handlers** Servers define operations with a ``handlers`` dict mapping operation names to functions. The first argument of a handler function must be a stream for the connection to the client. Other arguments will receive inputs from the keys of the incoming message which will always be a dictionary. >>> def pingpong(stream): ... return b'pong' >>> def add(stream, x, y): ... return x + y >>> handlers = {'ping': pingpong, 'add': add} >>> server = Server(handlers) # doctest: +SKIP >>> server.listen(8000) # doctest: +SKIP **Message Format** The server expects messages to be dictionaries with a special key, `'op'` that corresponds to the name of the operation, and other key-value pairs as required by the function. So in the example above the following would be good messages. * ``{'op': 'ping'}`` * ``{'op': 'add': 'x': 10, 'y': 20}`` """ default_port = 0 def __init__(self, handlers, max_buffer_size=MAX_BUFFER_SIZE, connection_limit=512, deserialize=True, **kwargs): self.handlers = assoc(handlers, 'identity', self.identity) self.id = str(uuid.uuid1()) self._port = None self.rpc = ConnectionPool(limit=connection_limit, deserialize=deserialize) self.deserialize = deserialize super(Server, self).__init__(max_buffer_size=max_buffer_size, **kwargs) @property def port(self): if not self._port: try: self._port = first(self._sockets.values()).getsockname()[1] except StopIteration: raise OSError("Server has no port. Please call .listen first") return self._port def identity(self, stream): return {'type': type(self).__name__, 'id': self.id} def listen(self, port=None): if port is None: port = self.default_port while True: try: super(Server, self).listen(port) break except (socket.error, OSError): if port: raise else: logger.info('Randomly assigned port taken for %s. Retrying', type(self).__name__) @gen.coroutine def handle_stream(self, stream, address): """ Dispatch new connections to coroutine-handlers Handlers is a dictionary mapping operation names to functions or coroutines. {'get_data': get_data, 'ping': pingpong} Coroutines should expect a single IOStream object. """ stream.set_nodelay(True) ip, port = address logger.info("Connection from %s:%d to %s", ip, port, type(self).__name__) try: while True: try: msg = yield read(stream, deserialize=self.deserialize) logger.debug("Message from %s:%d: %s", ip, port, msg) except StreamClosedError: logger.info("Lost connection: %s", str(address)) break except Exception as e: logger.exception(e) yield write(stream, error_message(e, status='uncaught-error')) continue if not isinstance(msg, dict): raise TypeError("Bad message type. Expected dict, got\n " + str(msg)) op = msg.pop('op') close = msg.pop('close', False) reply = msg.pop('reply', True) if op == 'close': if reply: yield write(stream, 'OK') break try: handler = self.handlers[op] except KeyError: result = "No handler found: %s" % op logger.warn(result, exc_info=True) else: logger.debug("Calling into handler %s", handler.__name__) try: result = yield gen.maybe_future(handler(stream, **msg)) except StreamClosedError as e: logger.info("%s", e) result = error_message(e, status='uncaught-error') except Exception as e: logger.exception(e) result = error_message(e, status='uncaught-error') if reply: try: yield write(stream, result) except StreamClosedError: logger.info("Lost connection: %s" % str(address)) break if close: break finally: try: stream.close() except Exception as e: logger.warn("Failed while closing writer", exc_info=True) logger.info("Close connection from %s:%d to %s", address[0], address[1], type(self).__name__) @gen.coroutine def read(stream, deserialize=True): """ Read a message from a stream """ if isinstance(stream, BatchedStream): msg = yield stream.recv() raise gen.Return(msg) else: n_frames = yield stream.read_bytes(8) n_frames = struct.unpack('Q', n_frames)[0] lengths = yield stream.read_bytes(8 * n_frames) lengths = struct.unpack('Q' * n_frames, lengths) frames = [] for length in lengths: if length: frame = yield stream.read_bytes(length) else: frame = b'' frames.append(frame) msg = protocol.loads(frames, deserialize=deserialize) raise gen.Return(msg) @gen.coroutine def write(stream, msg): """ Write a message to a stream """ if isinstance(stream, BatchedStream): stream.send(msg) else: try: frames = protocol.dumps(msg) except Exception as e: logger.info("Unserializable Message: %s", msg) logger.exception(e) raise futures = [] lengths = ([struct.pack('Q', len(frames))] + [struct.pack('Q', len(frame)) for frame in frames]) futures.append(stream.write(b''.join(lengths))) for frame in frames[:-1]: futures.append(stream.write(frame)) futures.append(stream.write(frames[-1])) while stream._write_buffer: try: yield gen.with_timeout(timedelta(seconds=0.01), futures[-1]) break except gen.TimeoutError: pass def pingpong(stream): return b'pong' @gen.coroutine def connect(ip, port, timeout=3): client = TCPClient() start = time() while True: future = client.connect(ip, port, max_buffer_size=MAX_BUFFER_SIZE) try: stream = yield gen.with_timeout(timedelta(seconds=timeout), future) stream.set_nodelay(True) raise gen.Return(stream) except StreamClosedError: if time() - start < timeout: yield gen.sleep(0.01) logger.debug("sleeping on connect") else: raise except gen.TimeoutError: raise IOError("Timed out while connecting to %s:%d" % (ip, port)) @gen.coroutine def send_recv(stream=None, arg=None, ip=None, port=None, addr=None, reply=True, deserialize=True, **kwargs): """ Send and recv with a stream Keyword arguments turn into the message response = yield send_recv(stream, op='ping', reply=True) """ if arg: if isinstance(arg, (unicode, bytes)): addr = arg if isinstance(arg, tuple): ip, port = arg if addr: assert not ip and not port if PY3 and isinstance(addr, bytes): addr = addr.decode() ip, port = addr.rsplit(':', 1) port = int(port) if PY3 and isinstance(ip, bytes): ip = ip.decode() if stream is None: stream = yield connect(ip, port) msg = kwargs msg['reply'] = reply yield write(stream, msg) if reply: response = yield read(stream, deserialize=deserialize) if isinstance(response, dict) and response.get('status') == 'uncaught-error': six.reraise(*clean_exception(**response)) else: response = None if kwargs.get('close'): stream.close() raise gen.Return(response) def ip_port_from_args(arg=None, addr=None, ip=None, port=None): if arg: if isinstance(arg, (unicode, bytes)): addr = arg if isinstance(arg, tuple): ip, port = arg if addr: if PY3 and isinstance(addr, bytes): addr = addr.decode() assert not ip and not port ip, port = addr.rsplit(':', 1) port = int(port) if PY3 and isinstance(ip, bytes): ip = ip.decode() return ip, port class rpc(object): """ Conveniently interact with a remote server Normally we construct messages as dictionaries and send them with read/write >>> stream = yield connect(ip, port) # doctest: +SKIP >>> msg = {'op': 'add', 'x': 10, 'y': 20} # doctest: +SKIP >>> yield write(stream, msg) # doctest: +SKIP >>> response = yield read(stream) # doctest: +SKIP To reduce verbosity we use an ``rpc`` object. >>> remote = rpc(ip=ip, port=port) # doctest: +SKIP >>> response = yield remote.add(x=10, y=20) # doctest: +SKIP One rpc object can be reused for several interactions. Additionally, this object creates and destroys many streams as necessary and so is safe to use in multiple overlapping communications. When done, close streams explicitly. >>> remote.close_streams() # doctest: +SKIP """ active = 0 def __init__(self, arg=None, stream=None, ip=None, port=None, addr=None, deserialize=True, timeout=3): ip, port = ip_port_from_args(arg=arg, addr=addr, ip=ip, port=port) self.streams = dict() self.ip = ip self.port = port self.timeout = timeout self.status = 'running' self.deserialize = deserialize rpc.active += 1 assert self.ip assert self.port @property def address(self): return '%s:%d' % (self.ip, self.port) @gen.coroutine def live_stream(self): """ Get an open stream Some streams to the ip/port target may be in current use by other coroutines. We track this with the `streams` dict :: {stream: True/False if open and ready for use} This function produces an open stream, either by taking one that we've already made or making a new one if they are all taken. This also removes streams that have been closed. When the caller is done with the stream they should set self.streams[stream] = True As is done in __getattr__ below. """ if self.status == 'closed': raise RPCClosed("RPC Closed") to_clear = set() open = False for stream, open in self.streams.items(): if stream.closed(): to_clear.add(stream) if open: break if not open or stream.closed(): stream = yield connect(self.ip, self.port, timeout=self.timeout) for s in to_clear: del self.streams[s] self.streams[stream] = False # mark as taken raise gen.Return(stream) def close_streams(self): for stream in self.streams: if stream and not stream.closed(): try: stream.close() except (OSError, IOError, StreamClosedError): pass self.streams.clear() def __getattr__(self, key): @gen.coroutine def send_recv_from_rpc(**kwargs): stream = yield self.live_stream() result = yield send_recv(stream=stream, op=key, deserialize=self.deserialize, **kwargs) self.streams[stream] = True # mark as open raise gen.Return(result) return send_recv_from_rpc def close_rpc(self): if self.status != 'closed': rpc.active -= 1 self.status = 'closed' self.close_streams() def __enter__(self): return self def __exit__(self, *args): self.close_rpc() class RPCCall(object): """ The result of ConnectionPool()('host:port') See Also: ConnectionPool """ def __init__(self, ip, port, pool): self.ip = ip self.port = port self.pool = pool def __getattr__(self, key): @gen.coroutine def send_recv_from_rpc(**kwargs): stream = yield self.pool.connect(self.ip, self.port) try: result = yield send_recv(stream=stream, op=key, deserialize=self.pool.deserialize, **kwargs) finally: if not stream.closed(): self.pool.available[self.ip, self.port].add(stream) self.pool.occupied[self.ip, self.port].remove(stream) self.pool.active -= 1 raise gen.Return(result) return send_recv_from_rpc def close_rpc(self): pass class ConnectionPool(object): """ A maximum sized pool of Tornado IOStreams This provides a connect method that mirrors the normal distributed.connect method, but provides connection sharing and tracks connection limits. This object provides an ``rpc`` like interface:: >>> rpc = ConnectionPool(limit=512) >>> scheduler = rpc('127.0.0.1:8786') >>> workers = [rpc(ip=ip, port=port) for ip, port in ...] >>> info = yield scheduler.identity() It creates enough streams to satisfy concurrent connections to any particular address:: >>> a, b = yield [scheduler.who_has(), scheduler.has_what()] It reuses existing streams so that we don't have to continuously reconnect. It also maintains a stream limit to avoid "too many open file handle" issues. Whenever this maximum is reached we clear out all idling streams. If that doesn't do the trick then we wait until one of the occupied streams closes. Parameters ---------- limit: int The number of open streams to maintain at once deserialize: bool Whether or not to deserialize data by default or pass it through """ def __init__(self, limit=512, deserialize=True): self.open = 0 self.active = 0 self.limit = limit self.available = defaultdict(set) self.occupied = defaultdict(set) self.deserialize = deserialize self.event = Event() def __str__(self): return "" % (self.open, self.active) __repr__ = __str__ def __call__(self, arg=None, ip=None, port=None, addr=None): """ Cached rpc objects """ ip, port = ip_port_from_args(arg=arg, addr=addr, ip=ip, port=port) return RPCCall(ip, port, self) @gen.coroutine def connect(self, ip, port, timeout=3): if self.available.get((ip, port)): stream = self.available[ip, port].pop() self.active += 1 self.occupied[ip, port].add(stream) raise gen.Return(stream) while self.open >= self.limit: self.event.clear() self.collect() yield self.event.wait() self.open += 1 stream = yield connect(ip=ip, port=port, timeout=timeout) stream.set_close_callback(lambda: self.on_close(ip, port, stream)) self.active += 1 self.occupied[ip, port].add(stream) if self.open >= self.limit: self.event.clear() raise gen.Return(stream) def on_close(self, ip, port, stream): self.open -= 1 if stream in self.available[ip, port]: self.available[ip, port].remove(stream) if stream in self.occupied[ip, port]: self.occupied[ip, port].remove(stream) self.active -= 1 if self.open <= self.limit: self.event.set() def collect(self): logger.info("Collecting unused streams. open: %d, active: %d", self.open, self.active) for streams in list(self.available.values()): for stream in streams: stream.close() def close(self): for streams in list(self.available.values()): for stream in streams: stream.close() for streams in list(self.occupied.values()): for stream in streams: stream.close() def coerce_to_address(o, out=str): if PY3 and isinstance(o, bytes): o = o.decode() if isinstance(o, (unicode, str)): ip, port = o.rsplit(':', 1) port = int(port) o = (ip, port) if isinstance(o, list): o = tuple(o) if isinstance(o, tuple) and isinstance(o[0], bytes): o = (o[0].decode(), o[1]) if out == str: o = '%s:%s' % o return o def coerce_to_rpc(o, **kwargs): if isinstance(o, (bytes, str, tuple, list)): ip, port = coerce_to_address(o, out=tuple) return rpc(ip=ip, port=int(port), **kwargs) elif isinstance(o, IOStream): return rpc(stream=o, **kwargs) elif isinstance(o, rpc): return o else: raise TypeError() def error_message(e, status='error'): """ Produce message to send back given an exception has occurred This does the following: 1. Gets the traceback 2. Truncates the exception and the traceback 3. Serializes the exception and traceback or 4. If they can't be serialized send string versions 5. Format a message and return See Also -------- clean_exception: deserialize and unpack message into exception/traceback six.reraise: raise exception/traceback """ tb = get_traceback() e2 = truncate_exception(e, 1000) try: e3 = protocol.pickle.dumps(e2) protocol.pickle.loads(e3) except Exception: e3 = Exception(str(e2)) e3 = protocol.pickle.dumps(e3) try: tb2 = protocol.pickle.dumps(tb) except Exception: tb2 = ''.join(traceback.format_tb(tb)) tb2 = protocol.pickle.dumps(tb2) if len(tb2) > 10000: tb2 = None return {'status': status, 'exception': e3, 'traceback': tb2} def clean_exception(exception, traceback, **kwargs): """ Reraise exception and traceback. Deserialize if necessary See Also -------- error_message: create and serialize errors into message """ if isinstance(exception, bytes): exception = protocol.pickle.loads(exception) if isinstance(traceback, bytes): traceback = protocol.pickle.loads(traceback) if isinstance(traceback, str): traceback = None return type(exception), exception, traceback from .batched import BatchedStream distributed-1.14.3/distributed/deploy/000077500000000000000000000000001301211143400177265ustar00rootroot00000000000000distributed-1.14.3/distributed/deploy/__init__.py000066400000000000000000000002311301211143400220330ustar00rootroot00000000000000from ..utils import ignoring from .local import LocalCluster from .adaptive import Adaptive with ignoring(ImportError): from .ssh import SSHCluster distributed-1.14.3/distributed/deploy/adaptive.py000066400000000000000000000052431301211143400221010ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import logging from ..utils import log_errors from tornado import gen from tornado.ioloop import PeriodicCallback logger = logging.getLogger(__file__) class Adaptive(object): ''' Adaptively allocate workers based on scheduler load. A superclass. Contains logic to dynamically resize a Dask cluster based on current use. Parameters ---------- scheduler: distributed.Scheduler cluster: object Must have scale_up and scale_down methods/coroutines Examples -------- >>> class MyCluster(object): ... def scale_up(self, n): ... """ Bring worker count up to n """ ... def scale_down(self, workers): ... """ Remove worker addresses from cluster """ ''' def __init__(self, scheduler, cluster, interval=1000): self.scheduler = scheduler self.cluster = cluster self._adapt_callback = PeriodicCallback(self._adapt, interval, self.scheduler.loop) self._adapt_callback.start() self._adapting = False def should_scale_up(self): if (self.scheduler.ready or any(self.scheduler.stealable) and not self.scheduler.idle): return True limit_bytes = {w: self.scheduler.worker_info[w]['memory_limit'] for w in self.scheduler.worker_info} worker_bytes = self.scheduler.worker_bytes limit = sum(limit_bytes.values()) total = sum(worker_bytes.values()) if total > 0.6 * limit: return True return False @gen.coroutine def _retire_workers(self): with log_errors(): workers = yield self.scheduler.retire_workers(remove=False) logger.info("Retiring workers %s", workers) f = self.cluster.scale_down(workers) if gen.is_future(f): yield f for w in workers: self.scheduler.remove_worker(address=w, safe=True) @gen.coroutine def _adapt(self): if self._adapting: # Semaphore to avoid overlapping adapt calls raise gen.Return() self._adapting = True try: if self.should_scale_up(): instances = max(1, len(self.scheduler.ncores) * 2) logger.info("Scaling up to %d workers", instances) f = self.cluster.scale_up(instances) if gen.is_future(f): yield f yield self._retire_workers() finally: self._adapting = False def adapt(self): self.scheduler.loop.add_callback(self._adapt) distributed-1.14.3/distributed/deploy/local.py000066400000000000000000000215501301211143400213750ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from functools import partial import logging import math from threading import Thread from time import sleep from tornado.ioloop import IOLoop from tornado.iostream import StreamClosedError from tornado import gen from ..http.scheduler import HTTPScheduler from ..utils import sync, ignoring, All from ..client import Client from ..nanny import Nanny from ..scheduler import Scheduler from ..worker import Worker, _ncores logger = logging.getLogger(__file__) class LocalCluster(object): """ Create local Scheduler and Workers This creates a "cluster" of a scheduler and workers running on the local machine. Parameters ---------- n_workers: int Number of workers to start threads_per_worker: int Number of threads per each worker nanny: boolean If true start the workers in separate processes managed by a nanny. If False keep the workers in the main calling process scheduler_port: int Port of the scheduler. 8786 by default, use 0 to choose a random port kwargs: dict Extra worker arguments, will be passed to the Worker constructor. Examples -------- >>> c = LocalCluster() # Create a local cluster with as many workers as cores # doctest: +SKIP >>> c # doctest: +SKIP LocalCluster("127.0.0.1:8786", workers=8, ncores=8) >>> c = Client(c) # connect to local cluster # doctest: +SKIP Add a new worker to the cluster >>> w = c.start_worker(ncores=2) # doctest: +SKIP Shut down the extra worker >>> c.remove_worker(w) # doctest: +SKIP Start a diagnostic web server and open a new browser tab >>> c.start_diagnostics_server(show=True) # doctest: +SKIP """ def __init__(self, n_workers=None, threads_per_worker=None, nanny=True, loop=None, start=True, scheduler_port=8786, silence_logs=logging.CRITICAL, diagnostics_port=8787, services={'http': HTTPScheduler}, **kwargs): self.status = None self.nanny = nanny if silence_logs: for l in ['distributed.scheduler', 'distributed.worker', 'distributed.core', 'distributed.nanny']: logging.getLogger(l).setLevel(silence_logs) if n_workers is None and threads_per_worker is None: if nanny: n_workers = _ncores threads_per_worker = 1 else: n_workers = 1 threads_per_worker = _ncores if n_workers is None and threads_per_worker is not None: n_workers = max(1, _ncores // threads_per_worker) if n_workers and threads_per_worker is None: # Overcommit threads per worker, rather than undercommit threads_per_worker = max(1, int(math.ceil(_ncores / n_workers))) self.loop = loop or IOLoop() if not self.loop._running: self._thread = Thread(target=self.loop.start) self._thread.daemon = True self._thread.start() while not self.loop._running: sleep(0.001) self.scheduler = Scheduler(loop=self.loop, ip='127.0.0.1', services=services) self.scheduler.start(scheduler_port) self.workers = [] if start: _start_worker = self.start_worker else: _start_worker = partial(self.loop.add_callback, self._start_worker) for i in range(n_workers): _start_worker(ncores=threads_per_worker, nanny=nanny, **kwargs) self.status = 'running' self.diagnostics = None if diagnostics_port is not None: self.start_diagnostics_server(diagnostics_port, silence=silence_logs) def __str__(self): return 'LocalCluster("%s", workers=%d, ncores=%d)' % ( self.scheduler_address, len(self.workers), sum(w.ncores for w in self.workers)) __repr__ = __str__ @gen.coroutine def _start_worker(self, port=0, nanny=None, **kwargs): if nanny is None: nanny = self.nanny if nanny: W = Nanny kwargs['quiet'] = True else: W = Worker w = W(self.scheduler.ip, self.scheduler.port, loop=self.loop, **kwargs) yield w._start(port) self.workers.append(w) while w.worker_address not in self.scheduler.worker_info: yield gen.sleep(0.01) raise gen.Return(w) def start_worker(self, port=0, ncores=0, **kwargs): """ Add a new worker to the running cluster Parameters ---------- port: int (optional) Port on which to serve the worker, defaults to 0 or random ncores: int (optional) Number of threads to use. Defaults to number of logical cores nanny: boolean If true start worker in separate process managed by a nanny Examples -------- >>> c = LocalCluster() # doctest: +SKIP >>> c.start_worker(ncores=2) # doctest: +SKIP Returns ------- The created Worker or Nanny object. Can be discarded. """ return sync(self.loop, self._start_worker, port, ncores=ncores, **kwargs) @gen.coroutine def _stop_worker(self, w): yield w._close() self.workers.remove(w) def stop_worker(self, w): """ Stop a running worker Examples -------- >>> c = LocalCluster() # doctest: +SKIP >>> w = c.start_worker(ncores=2) # doctest: +SKIP >>> c.stop_worker(w) # doctest: +SKIP """ sync(self.loop, self._stop_worker, w) def start_diagnostics_server(self, port=8787, show=False, silence=logging.CRITICAL): """ Start Diagnostics Web Server This starts a web application to show diagnostics of what is happening on the cluster. This application runs in a separate process and is generally available at the following location: http://localhost:8787/status/ """ try: from distributed.bokeh.application import BokehWebInterface except ImportError: logger.info("To start diagnostics web server please install Bokeh") return assert self.diagnostics is None if 'http' not in self.scheduler.services: self.scheduler.services['http'] = HTTPScheduler(self.scheduler, io_loop=self.scheduler.loop) self.scheduler.services['http'].listen(0) self.diagnostics = BokehWebInterface( tcp_port=self.scheduler.port, http_port=self.scheduler.services['http'].port, bokeh_port=port, show=show, log_level=logging.getLevelName(silence).lower()) @gen.coroutine def _close(self): with ignoring(gen.TimeoutError, StreamClosedError, OSError): yield All([w._close() for w in self.workers]) with ignoring(gen.TimeoutError, StreamClosedError, OSError): yield self.scheduler.close(fast=True) del self.workers[:] if self.diagnostics: self.diagnostics.close() def close(self): """ Close the cluster """ if self.status == 'running': self.status = 'closed' if self.loop._running: sync(self.loop, self._close) if hasattr(self, '_thread'): sync(self.loop, self.loop.stop) self._thread.join(timeout=1) self.loop.close() del self._thread @gen.coroutine def scale_up(self, n, **kwargs): """ Bring the total count of workers up to ``n`` This function/coroutine should bring the total number of workers up to the number ``n``. This can be implemented either as a function or as a Tornado coroutine. """ yield [self._start_worker(**kwargs) for i in range(n - len(self.workers))] @gen.coroutine def scale_down(self, workers): """ Remove ``workers`` from the cluster Given a list of worker addresses this function should remove those workers from the cluster. This may require tracking which jobs are associated to which worker address. This can be implemented either as a function or as a Tornado coroutine. """ workers = set(workers) yield [self._stop_worker(w) for w in self.workers if w.worker_address in workers] while workers & set(self.workers): yield gen.sleep(0.01) def __del__(self): self.close() def __enter__(self): return self def __exit__(self, *args): self.close() @property def scheduler_address(self): return self.scheduler.address distributed-1.14.3/distributed/deploy/ssh.py000066400000000000000000000262061301211143400211030ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from time import sleep import socket import os import sys import traceback try: from queue import Queue except ImportError: # Python 2.7 fix from Queue import Queue from threading import Thread from toolz import merge # These are handy for creating colorful terminal output to enhance readability # of the output generated by dask-ssh. class bcolors: HEADER = '\033[95m' OKBLUE = '\033[94m' OKGREEN = '\033[92m' WARNING = '\033[93m' FAIL = '\033[91m' ENDC = '\033[0m' BOLD = '\033[1m' UNDERLINE = '\033[4m' def async_ssh(cmd_dict): import paramiko from paramiko.buffered_pipe import PipeTimeout from paramiko.ssh_exception import (SSHException, PasswordRequiredException) ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) retries = 0 while True: # Be robust to transient SSH failures. try: # Set paramiko logging to WARN or higher to squelch INFO messages. import logging logger = logging.getLogger('paramiko') logger.setLevel(logging.WARN) ssh.connect(hostname = cmd_dict['address'], username = cmd_dict['ssh_username'], port = cmd_dict['ssh_port'], key_filename = cmd_dict['ssh_private_key'], compress = True, timeout = 20, banner_timeout = 20) # Helps prevent timeouts when many concurrent ssh connections are opened. # Connection successful, break out of while loop break except (SSHException, PasswordRequiredException) as e: print('[ dask-ssh ] : ' + bcolors.FAIL + 'SSH connection error when connecting to {addr}:{port} to run \'{cmd}\''.format(addr = cmd_dict['address'], port = cmd_dict['ssh_port'], cmd = cmd_dict['cmd']) + bcolors.ENDC) print( bcolors.FAIL + ' SSH reported this exception: ' + str(e) + bcolors.ENDC ) # Print an exception traceback traceback.print_exc() # Transient SSH errors can occur when many SSH connections are # simultaneously opened to the same server. This makes a few # attempts to retry. retries += 1 if retries >= 3: print( '[ dask-ssh ] : ' + bcolors.FAIL + 'SSH connection failed after 3 retries. Exiting.' + bcolors.ENDC) # Connection failed after multiple attempts. Terminate this thread. os._exit(1) # Wait a moment before retrying print( ' ' + bcolors.FAIL + 'Retrying... (attempt {n}/{total})'.format(n = retries, total = 3) + bcolors.ENDC) sleep(1) # Execute the command, and grab file handles for stdout and stderr. Note # that we run the command using the user's default shell, but force it to # run in an interactive login shell, which hopefully ensures that all of the # user's normal environment variables (via the dot files) have been loaded # before the command is run. This should help to ensure that important # aspects of the environment like PATH and PYTHONPATH are configured. print('[ {label} ] : {cmd}'.format(label = cmd_dict['label'], cmd = cmd_dict['cmd'])) stdin, stdout, stderr = ssh.exec_command('$SHELL -i -c \'' + cmd_dict['cmd'] + '\'', get_pty = True) # Set up channel timeouts (which we rely on below to make readline() # non-blocking. stdout.channel.settimeout(0.1) stderr.channel.settimeout(0.1) # Wait for a message on the input_queue. Any message received signals this # thread to shut itself down. while(cmd_dict['input_queue'].empty()): # Read stdout stream, time out if necessary. try: line = stdout.readline() while len(line) > 0: # Loops until a timeout exception occurs cmd_dict['output_queue'].put('[ {label} ] : {output}'.format(label = cmd_dict['label'], output = line.rstrip())) line = stdout.readline() except PipeTimeout: continue except socket.timeout: continue # Read stderr stream, time out if necessary try: line = stderr.readline() while len(line) > 0: cmd_dict['output_queue'].put('[ {label} ] : '.format(label = cmd_dict['label']) + bcolors.FAIL + '{output}'.format(output = line.rstrip()) + bcolors.ENDC) line = stderr.readline() except PipeTimeout: continue except socket.timeout: continue # Check to see if the process has exited. If it has, we let this thread # terminate. if stdout.channel.exit_status_ready(): exit_status = stdout.channel.recv_exit_status() cmd_dict['output_queue'].put('[ {label} ] : '.format(label = cmd_dict['label']) + bcolors.FAIL + "remote process exited with exit status " + str(exit_status) + bcolors.ENDC) break # Kill some time so that this thread does not hog the CPU. sleep(1.0) # end while() # Shutdown the channel, and close the SSH connection stdout.channel.close() stderr.channel.close() ssh.close() def start_scheduler(logdir, addr, port, ssh_username, ssh_port, ssh_private_key): cmd = '{python} -m distributed.cli.dask_scheduler --port {port}'.format(python=sys.executable, port=port, logdir=logdir) # Optionally re-direct stdout and stderr to a logfile if logdir is not None: cmd = 'mkdir -p {logdir} && '.format(logdir=logdir) + cmd cmd += '&> {logdir}/dask_scheduler_{addr}:{port}.log'.format(addr=addr, port=port, logdir=logdir) # Format output labels we can prepend to each line of output, and create # a 'status' key to keep track of jobs that terminate prematurely. label = (bcolors.BOLD + 'scheduler {addr}:{port}'.format(addr=addr, port=port) + bcolors.ENDC) # Create a command dictionary, which contains everything we need to run and # interact with this command. input_queue = Queue() output_queue = Queue() cmd_dict = {'cmd': cmd, 'label': label, 'address': addr, 'port': port, 'input_queue': input_queue, 'output_queue': output_queue, 'ssh_username': ssh_username, 'ssh_port': ssh_port, 'ssh_private_key': ssh_private_key} # Start the thread thread = Thread(target=async_ssh, args=[cmd_dict]) thread.daemon = True thread.start() return merge(cmd_dict, {'thread': thread}) def start_worker(logdir, scheduler_addr, scheduler_port, worker_addr, nthreads, nprocs, ssh_username, ssh_port, ssh_private_key): cmd = '{python} -m distributed.cli.dask_worker {scheduler_addr}:{scheduler_port} --host {worker_addr} --nthreads {nthreads} --nprocs {nprocs}'.format( python = sys.executable, scheduler_addr = scheduler_addr, scheduler_port = scheduler_port, worker_addr = worker_addr, nthreads = nthreads, nprocs = nprocs) # Optionally redirect stdout and stderr to a logfile if logdir is not None: cmd = 'mkdir -p {logdir} && '.format(logdir=logdir) + cmd cmd += '&> {logdir}/dask_scheduler_{addr}.log'.format( addr = worker_addr, logdir = logdir) label = 'worker {addr}'.format(addr = worker_addr) # Create a command dictionary, which contains everything we need to run and # interact with this command. input_queue = Queue() output_queue = Queue() cmd_dict = {'cmd': cmd, 'label': label, 'address': worker_addr, 'input_queue': input_queue, 'output_queue': output_queue, 'ssh_username': ssh_username, 'ssh_port': ssh_port, 'ssh_private_key': ssh_private_key} # Start the thread thread = Thread(target=async_ssh, args=[cmd_dict]) thread.daemon = True thread.start() return merge(cmd_dict, {'thread': thread}) class SSHCluster(object): def __init__(self, scheduler_addr, scheduler_port, worker_addrs, nthreads = 0, nprocs = 1, ssh_username = None, ssh_port = 22, ssh_private_key = None, logdir = None): self.scheduler_addr = scheduler_addr self.scheduler_port = scheduler_port self.nthreads = nthreads self.nprocs = nprocs self.ssh_username = ssh_username self.ssh_port = ssh_port self.ssh_private_key = ssh_private_key # Generate a universal timestamp to use for log files import datetime if logdir is not None: logdir = os.path.join(logdir, "dask-ssh_" + datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")) print(bcolors.WARNING + 'Output will be redirected to logfiles stored locally on individual worker nodes under "{logdir}".'.format(logdir=logdir) + bcolors.ENDC) self.logdir = logdir # Keep track of all running threads self.threads = [] # Start the scheduler node self.scheduler = start_scheduler(logdir, scheduler_addr, scheduler_port, ssh_username, ssh_port, ssh_private_key) # Start worker nodes self.workers = [] for i, addr in enumerate(worker_addrs): self.add_worker(addr) @property def scheduler_address(self): return '%s:%d' % (self.scheduler_addr, self.scheduler_port) def monitor_remote_processes(self): # Form a list containing all processes, since we treat them equally from here on out. all_processes = [self.scheduler] + self.workers try: while True: for process in all_processes: while not process['output_queue'].empty(): print(process['output_queue'].get()) # Kill some time and free up CPU before starting the next sweep # through the processes. sleep(0.1) # end while true except KeyboardInterrupt: pass # Return execution to the calling process def add_worker(self, address): self.workers.append(start_worker(self.logdir, self.scheduler_addr, self.scheduler_port, address, self.nthreads, self.nprocs, self.ssh_username, self.ssh_port, self.ssh_private_key)) def shutdown(self): all_processes = [self.scheduler] + self.workers for process in all_processes: process['input_queue'].put('shutdown') process['thread'].join() def __enter__(self): return self def __exit__(self, *args): self.shutdown() distributed-1.14.3/distributed/deploy/tests/000077500000000000000000000000001301211143400210705ustar00rootroot00000000000000distributed-1.14.3/distributed/deploy/tests/test_adaptive.py000066400000000000000000000032071301211143400243000ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from time import sleep, time from tornado import gen from tornado.ioloop import IOLoop from distributed import Client from distributed.deploy import Adaptive, LocalCluster from distributed.utils_test import loop, slowinc, gen_test def test_adaptive_local_cluster(loop): with LocalCluster(0, scheduler_port=0, silence_logs=False, diagnostics_port=None, loop=loop) as cluster: alc = Adaptive(cluster.scheduler, cluster, interval=100) with Client(cluster, loop=loop) as c: assert not c.ncores() future = c.submit(lambda x: x + 1, 1) assert future.result() == 2 assert c.ncores() sleep(0.1) assert c.ncores() # still there after some time del future start = time() while cluster.scheduler.ncores: sleep(0.01) assert time() < start + 5 assert not c.ncores() @gen_test(timeout=120) def test_adaptive_local_cluster_multi_workers(): loop = IOLoop.current() cluster = LocalCluster(0, scheduler_port=0, silence_logs=False, nanny=False, diagnostics_port=None, loop=loop, start=False) cluster.scheduler.allowed_failures = 1000 alc = Adaptive(cluster.scheduler, cluster, interval=100) c = Client(cluster, start=False, loop=loop) yield c._start() for i in range(20): futures = c.map(slowinc, range(100), delay=0.01) yield c._gather(futures) del futures yield gen.sleep(0.1) yield c._shutdown() yield cluster._close() distributed-1.14.3/distributed/deploy/tests/test_local.py000066400000000000000000000163671301211143400236100ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from functools import partial import sys from time import sleep, time import unittest from tornado.ioloop import IOLoop from tornado import gen import pytest from distributed.deploy.local import LocalCluster from distributed import Client, Worker, Nanny from distributed.utils_test import inc, loop, raises, gen_test from distributed.utils import ignoring from distributed.deploy.utils_test import ClusterTest def test_simple(loop): with LocalCluster(4, scheduler_port=0, nanny=False, silence_logs=False, diagnostics_port=None, loop=loop) as c: with Client((c.scheduler.ip, c.scheduler.port), loop=loop) as e: x = e.submit(inc, 1) x.result() assert x.key in c.scheduler.tasks assert any(w.data == {x.key: 2} for w in c.workers) @pytest.mark.skipif('sys.version_info[0] == 2', reason='multi-loop') def test_procs(loop): with LocalCluster(2, scheduler_port=0, nanny=False, threads_per_worker=3, diagnostics_port=None, silence_logs=False) as c: assert len(c.workers) == 2 assert all(isinstance(w, Worker) for w in c.workers) with Client((c.scheduler.ip, c.scheduler.port), loop=loop) as e: assert all(w.ncores == 3 for w in c.workers) repr(c) with LocalCluster(2, scheduler_port=0, nanny=True, threads_per_worker=3, diagnostics_port=None, silence_logs=False) as c: assert len(c.workers) == 2 assert all(isinstance(w, Nanny) for w in c.workers) with Client((c.scheduler.ip, c.scheduler.port), loop=loop) as e: assert all(v == 3 for v in e.ncores().values()) c.start_worker(nanny=False) assert isinstance(c.workers[-1], Worker) repr(c) @pytest.mark.skipif('sys.version_info[0] == 2', reason='') class LocalTest(ClusterTest, unittest.TestCase): Cluster = partial(LocalCluster, silence_logs=False, diagnostics_port=None) def test_Client_with_local(loop): with LocalCluster(1, scheduler_port=0, silence_logs=False, diagnostics_port=None, loop=loop) as c: with Client(c, loop=loop) as e: assert len(e.ncores()) == len(c.workers) assert c.scheduler_address in repr(c) def test_Client_solo(loop): with Client(loop=loop) as c: pass assert c.cluster.status == 'closed' def test_Client_twice(loop): with Client(loop=loop) as c: with Client(loop=loop) as f: assert c.cluster.scheduler.port != f.cluster.scheduler.port def test_defaults(): from distributed.worker import _ncores with LocalCluster(scheduler_port=0, silence_logs=False, diagnostics_port=None) as c: assert sum(w.ncores for w in c.workers) == _ncores assert all(isinstance(w, Nanny) for w in c.workers) assert all(w.ncores == 1 for w in c.workers) with LocalCluster(nanny=False, scheduler_port=0, silence_logs=False, diagnostics_port=None) as c: assert sum(w.ncores for w in c.workers) == _ncores assert all(isinstance(w, Worker) for w in c.workers) assert len(c.workers) == 1 with LocalCluster(n_workers=2, scheduler_port=0, silence_logs=False, diagnostics_port=None) as c: if _ncores % 2 == 0: expected_total_threads = max(2, _ncores) else: # n_workers not a divisor of _ncores => threads are overcommitted expected_total_threads = max(2, _ncores + 1) assert sum(w.ncores for w in c.workers) == expected_total_threads with LocalCluster(threads_per_worker=_ncores * 2, scheduler_port=0, silence_logs=False, diagnostics_port=None) as c: assert len(c.workers) == 1 with LocalCluster(n_workers=_ncores * 2, scheduler_port=0, silence_logs=False, diagnostics_port=None) as c: assert all(w.ncores == 1 for w in c.workers) with LocalCluster(threads_per_worker=2, n_workers=3, scheduler_port=0, silence_logs=False, diagnostics_port=None) as c: assert len(c.workers) == 3 assert all(w.ncores == 2 for w in c.workers) def test_worker_params(): with LocalCluster(n_workers=2, scheduler_port=0, silence_logs=False, diagnostics_port=None, memory_limit=500) as c: assert [w.memory_limit for w in c.workers] == [500] * 2 def test_cleanup(): c = LocalCluster(2, scheduler_port=0, silence_logs=False, diagnostics_port=None) port = c.scheduler.port c.close() c2 = LocalCluster(2, scheduler_port=port, silence_logs=False, diagnostics_port=None) c.close() def test_repeated(): with LocalCluster(scheduler_port=8448, silence_logs=False, diagnostics_port=None) as c: pass with LocalCluster(scheduler_port=8448, silence_logs=False, diagnostics_port=None) as c: pass def test_http(loop): from distributed.http import HTTPScheduler import requests with LocalCluster(scheduler_port=0, silence_logs=False, services={('http', 3485): HTTPScheduler}, diagnostics_port=None, loop=loop) as c: response = requests.get('http://127.0.0.1:3485/info.json') assert response.ok def test_bokeh(loop): from distributed.http import HTTPScheduler import requests with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop, diagnostics_port=4724, services={('http', 0): HTTPScheduler}, ) as c: start = time() while True: with ignoring(Exception): response = requests.get('http://127.0.0.1:%d/status/' % c.diagnostics.port) if response.ok: break assert time() < start + 20 sleep(0.01) start = time() while not raises(lambda: requests.get('http://127.0.0.1:%d/status/' % 4724)): assert time() < start + 10 sleep(0.01) def test_start_diagnostics(loop): from distributed.http import HTTPScheduler import requests with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop, diagnostics_port=None) as c: c.start_diagnostics_server(show=False, port=3748) start = time() while True: with ignoring(Exception): response = requests.get('http://127.0.0.1:%d/status/' % c.diagnostics.port) if response.ok: break assert time() < start + 20 sleep(0.01) def test_blocks_until_full(loop): with Client(loop=loop) as c: assert len(c.ncores()) > 0 @gen_test() def test_scale_up_and_down(): loop = IOLoop.current() cluster = LocalCluster(0, scheduler_port=0, nanny=False, silence_logs=False, diagnostics_port=None, loop=loop, start=False) c = Client(cluster, start=False, loop=loop) yield c._start() assert not cluster.workers yield cluster.scale_up(2) assert len(cluster.workers) == 2 assert len(cluster.scheduler.ncores) == 2 addr = cluster.workers[0].address yield cluster.scale_down([addr]) assert len(cluster.workers) == 1 assert addr not in cluster.scheduler.ncores yield c._shutdown() yield cluster._close() distributed-1.14.3/distributed/deploy/tests/test_ssh.py000066400000000000000000000015431301211143400233010ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from time import sleep, time import pytest pytest.importorskip('paramiko') from distributed import Client from distributed.deploy.ssh import SSHCluster from distributed.utils_test import slow from distributed.utils_test import loop @pytest.mark.avoid_travis def test_cluster(loop): with SSHCluster(scheduler_addr = '127.0.0.1', scheduler_port = 7437, worker_addrs = ['127.0.0.1', '127.0.0.1']) as c: with Client(c, loop=loop) as e: start = time() while len(e.ncores()) != 2: sleep(0.01) assert time() < start + 5 c.add_worker('127.0.0.1') start = time() while len(e.ncores()) != 3: sleep(0.01) assert time() < start + 5 distributed-1.14.3/distributed/deploy/utils_test.py000066400000000000000000000020421301211143400224750ustar00rootroot00000000000000from ..client import Client class ClusterTest(object): Cluster = None def setUp(self): self.cluster = self.Cluster(2, scheduler_port=0) self.client = Client(self.cluster.scheduler_address) def tearDown(self): self.client.shutdown() self.cluster.close() def test_cores(self): assert len(self.client.ncores()) == 2 def test_submit(self): future = self.client.submit(lambda x: x + 1, 1) assert future.result() == 2 def test_start_worker(self): a = self.client.ncores() w = self.cluster.start_worker(ncores=3) b = self.client.ncores() assert len(b) == 1 + len(a) assert any(v == 3 for v in b.values()) self.cluster.stop_worker(w) c = self.client.ncores() assert c == a def test_context_manager(self): with self.Cluster() as c: with Client(c) as e: assert e.ncores() def test_no_workers(self): with self.Cluster(0, scheduler_port=0) as c: pass distributed-1.14.3/distributed/diagnostics/000077500000000000000000000000001301211143400207415ustar00rootroot00000000000000distributed-1.14.3/distributed/diagnostics/__init__.py000066400000000000000000000004711301211143400230540ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from ..utils import ignoring with ignoring(ImportError): from .progressbar import progress with ignoring(ImportError): from .resource_monitor import Occupancy with ignoring(ImportError): from .scheduler_widgets import scheduler_status distributed-1.14.3/distributed/diagnostics/eventstream.py000066400000000000000000000042601301211143400236520ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import logging from tornado import gen from .plugin import SchedulerPlugin from ..core import connect, write, coerce_to_address from ..worker import dumps_function logger = logging.getLogger(__name__) class EventStream(SchedulerPlugin): """ Maintain a copy of worker events """ def __init__(self, scheduler=None): self.buffer = [] if scheduler: scheduler.add_plugin(self) def transition(self, key, start, finish, *args, **kwargs): if start == 'processing': kwargs['key'] = key if finish == 'memory' or finish == 'erred': self.buffer.append(kwargs) def swap_buffer(scheduler, es): es.buffer, buffer = [], es.buffer return buffer def teardown(scheduler, es): scheduler.remove_plugin(es) @gen.coroutine def eventstream(address, interval): """ Open a TCP connection to scheduler, receive batched task messages The messages coming back are lists of dicts. Each dict is of the following form:: {'key': 'mykey', 'worker': 'host:port', 'status': status, 'compute_start': time(), 'compute_stop': time(), 'transfer_start': time(), 'transfer_stop': time(), 'disk_load_start': time(), 'disk_load_stop': time(), 'other': 'junk'} Where ``status`` is either 'OK', or 'error' Parameters ---------- address: address of scheduler interval: time between batches, in seconds Examples -------- >>> stream = yield eventstream('127.0.0.1:8786', 0.100) # doctest: +SKIP >>> print(yield read(stream)) # doctest: +SKIP [{'key': 'x', 'status': 'OK', 'worker': '192.168.0.1:54684', ...}, {'key': 'y', 'status': 'error', 'worker': '192.168.0.1:54684', ...}] """ ip, port = coerce_to_address(address, out=tuple) stream = yield connect(ip, port) yield write(stream, {'op': 'feed', 'setup': dumps_function(EventStream), 'function': dumps_function(swap_buffer), 'interval': interval, 'teardown': dumps_function(teardown)}) raise gen.Return(stream) distributed-1.14.3/distributed/diagnostics/plugin.py000066400000000000000000000033631301211143400226160ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import logging logger = logging.getLogger(__name__) class SchedulerPlugin(object): """ Interface to extend the Scheduler The scheduler operates by triggering and responding to events like ``task_finished``, ``update_graph``, ``task_erred``, etc.. A plugin enables custom code to run at each of those same events. The scheduler will run the analogous methods on this class when each event is triggered. This runs user code within the scheduler thread that can perform arbitrary operations in synchrony with the scheduler itself. Plugins are often used for diagnostics and measurement, but have full access to the scheduler and could in principle affect core scheduling. To implement a plugin implement some of the methods of this class and add the plugin to the scheduler with ``Scheduler.add_plugin(myplugin)``. Examples -------- >>> class Counter(SchedulerPlugin): ... def __init__(self): ... self.counter = 0 ... ... def transition(self, key, start, finish, *args, **kwargs): ... if start == 'processing' and finish == 'memory': ... self.counter += 1 ... ... def restart(self, scheduler): ... self.counter = 0 >>> c = Counter() >>> scheduler.add_plugin(c) # doctest: +SKIP """ def update_graph(self, scheduler, dsk=None, keys=None, restrictions=None, **kwargs): """ Run when a new graph / tasks enter the scheduler """ pass def restart(self, scheduler, **kwargs): """ Run when the scheduler restarts itself """ pass def transition(self, key, start, finish, *args, **kwargs): pass distributed-1.14.3/distributed/diagnostics/progress.py000066400000000000000000000230461301211143400231640ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from collections import defaultdict import logging import sys import threading import time from timeit import default_timer import dask from toolz import valmap, groupby, concat from tornado.ioloop import PeriodicCallback, IOLoop from tornado import gen from .plugin import SchedulerPlugin from ..utils import sync, key_split, tokey logger = logging.getLogger(__name__) def dependent_keys(keys, who_has, processing, stacks, dependencies, exceptions, complete=False): """ All keys that need to compute for these keys to finish """ out = set() errors = set() stack = list(keys) while stack: key = stack.pop() if key in out: continue if not complete and (who_has.get(key) or key in processing or key in stacks): continue if key in exceptions: errors.add(key) if not complete: continue out.add(key) stack.extend(dependencies.get(key, [])) return out, errors class Progress(SchedulerPlugin): """ Tracks progress of a set of keys or futures On creation we provide a set of keys or futures that interest us as well as a scheduler. We traverse through the scheduler's dependencies to find all relevant keys on which our keys depend. We then plug into the scheduler to learn when our keys become available in memory at which point we record their completion. State ----- keys: set Set of keys that are not yet computed all_keys: set Set of all keys that we track This class performs no visualization. However it is used by other classes, notably TextProgressBar and ProgressWidget, which do perform visualization. """ def __init__(self, keys, scheduler, minimum=0, dt=0.1, complete=False): self.keys = {k.key if hasattr(k, 'key') else k for k in keys} self.keys = {tokey(k) for k in self.keys} self.scheduler = scheduler self.complete = complete self._minimum = minimum self._dt = dt self.last_duration = 0 self._start_time = default_timer() self._running = False self.status = None @gen.coroutine def setup(self): keys = self.keys while not keys.issubset(self.scheduler.task_state): yield gen.sleep(0.05) self.keys = None self.scheduler.add_plugin(self) # subtle race condition here self.all_keys, errors = dependent_keys(keys, self.scheduler.who_has, self.scheduler.processing, self.scheduler.stacks, self.scheduler.dependencies, self.scheduler.exceptions, complete=self.complete) if not self.complete: self.keys = self.all_keys.copy() else: self.keys, _ = dependent_keys(keys, self.scheduler.who_has, self.scheduler.processing, self.scheduler.stacks, self.scheduler.dependencies, self.scheduler.exceptions, complete=False) self.all_keys.update(keys) self.keys |= errors & self.all_keys if not self.keys: self.stop(exception=None, key=None) logger.debug("Set up Progress keys") for k in errors: self.transition(k, None, 'erred', exception=True) def transition(self, key, start, finish, *args, **kwargs): if key in self.keys and start == 'processing' and finish == 'memory': logger.debug("Progress sees key %s", key) self.keys.remove(key) if not self.keys: self.stop() if key in self.all_keys and finish == 'erred': logger.debug("Progress sees task erred") self.stop(exception=kwargs['exception'], key=key) if key in self.keys and finish == 'forgotten': logger.debug("A task was cancelled (%s), stopping progress", key) self.stop(exception=True) def restart(self, scheduler): self.stop() def stop(self, exception=None, key=None): if self in self.scheduler.plugins: self.scheduler.plugins.remove(self) if exception: self.status = 'error' else: self.status = 'finished' logger.debug("Remove Progress plugin") class MultiProgress(Progress): """ Progress variant that keeps track of different groups of keys See Progress for most details. This only adds a function ``func=`` that splits keys. This defaults to ``key_split`` which aligns with naming conventions chosen in the dask project (tuples, hyphens, etc..) State ----- keys: dict Maps group name to set of not-yet-complete keys for that group all_keys: dict Maps group name to set of all keys for that group Examples -------- >>> split = lambda s: s.split('-')[0] >>> p = MultiProgress(['y-2'], func=split) # doctest: +SKIP >>> p.keys # doctest: +SKIP {'x': {'x-1', 'x-2', 'x-3'}, 'y': {'y-1', 'y-2'}} """ def __init__(self, keys, scheduler=None, func=key_split, minimum=0, dt=0.1, complete=False): self.func = func Progress.__init__(self, keys, scheduler, minimum=minimum, dt=dt, complete=complete) @gen.coroutine def setup(self): keys = self.keys while not keys.issubset(self.scheduler.tasks): yield gen.sleep(0.05) self.keys = None self.scheduler.add_plugin(self) # subtle race condition here self.all_keys, errors = dependent_keys(keys, self.scheduler.who_has, self.scheduler.processing, self.scheduler.stacks, self.scheduler.dependencies, self.scheduler.exceptions, complete=self.complete) if not self.complete: self.keys = self.all_keys.copy() else: self.keys, _ = dependent_keys(keys, self.scheduler.who_has, self.scheduler.processing, self.scheduler.stacks, self.scheduler.dependencies, self.scheduler.exceptions, complete=False) self.all_keys.update(keys) self.keys |= errors & self.all_keys if not self.keys: self.stop(exception=None, key=None) # Group keys by func name self.keys = valmap(set, groupby(self.func, self.keys)) self.all_keys = valmap(set, groupby(self.func, self.all_keys)) for k in self.all_keys: if k not in self.keys: self.keys[k] = set() for k in errors: self.transition(k, None, 'erred', exception=True) logger.debug("Set up Progress keys") def transition(self, key, start, finish, *args, **kwargs): if start == 'processing' and finish == 'memory': s = self.keys.get(self.func(key), None) if s and key in s: s.remove(key) if not self.keys or not any(self.keys.values()): self.stop() if finish == 'erred': logger.debug("Progress sees task erred") k = self.func(key) if (k in self.all_keys and key in self.all_keys[k]): self.stop(exception=kwargs.get('exception'), key=key) if finish == 'forgotten': k = self.func(key) if k in self.all_keys and key in self.all_keys[k]: logger.debug("A task was cancelled (%s), stopping progress", key) self.stop(exception=True) def format_time(t): """Format seconds into a human readable form. >>> format_time(10.4) '10.4s' >>> format_time(1000.4) '16min 40.4s' >>> format_time(100000.4) '27hr 46min 40.4s' """ m, s = divmod(t, 60) h, m = divmod(m, 60) if h: return '{0:2.0f}hr {1:2.0f}min {2:4.1f}s'.format(h, m, s) elif m: return '{0:2.0f}min {1:4.1f}s'.format(m, s) else: return '{0:4.1f}s'.format(s) class AllProgress(SchedulerPlugin): """ Keep track of all keys, grouped by key_split """ def __init__(self, scheduler): self.all = defaultdict(set) self.nbytes = defaultdict(lambda: 0) self.state = defaultdict(lambda: defaultdict(set)) self.scheduler = scheduler for key, state in self.scheduler.task_state.items(): k = key_split(key) self.all[k].add(key) self.state[state][k].add(key) if key in self.scheduler.nbytes: self.nbytes[k] += self.scheduler.nbytes[key] scheduler.add_plugin(self) def transition(self, key, start, finish, *args, **kwargs): k = key_split(key) self.all[k].add(key) try: self.state[start][k].remove(key) except KeyError: # TODO: remove me once we have a new or clean state pass if finish != 'forgotten': self.state[finish][k].add(key) else: self.all[k].remove(key) if not self.all[k]: del self.all[k] try: del self.nbytes[k] except KeyError: pass for v in self.state.values(): try: del v[k] except KeyError: pass if start == 'memory': self.nbytes[k] -= self.scheduler.nbytes[key] if finish == 'memory': self.nbytes[k] += self.scheduler.nbytes[key] def restart(self, scheduler): self.all.clear() self.state.clear() distributed-1.14.3/distributed/diagnostics/progress_stream.py000066400000000000000000000155171301211143400245430ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import itertools import logging import random from bokeh.palettes import viridis from toolz import valmap, merge, memoize from tornado import gen from .progress import AllProgress from ..core import connect, write, coerce_to_address from ..scheduler import Scheduler from ..utils import key_split from ..worker import dumps_function logger = logging.getLogger(__name__) task_stream_palette = list(viridis(25)) random.shuffle(task_stream_palette) def counts(scheduler, allprogress): return merge({'all': valmap(len, allprogress.all), 'nbytes': allprogress.nbytes}, {state: valmap(len, allprogress.state[state]) for state in ['memory', 'erred', 'released']}) counter = itertools.count() @memoize def incrementing_index(o): return next(counter) @gen.coroutine def progress_stream(address, interval): """ Open a TCP connection to scheduler, receive progress messages The messages coming back are dicts containing counts of key groups:: {'inc': {'all': 5, 'memory': 2, 'erred': 0, 'released': 1}, 'dec': {'all': 1, 'memory': 0, 'erred': 0, 'released': 0}} Parameters ---------- address: address of scheduler interval: time between batches, in seconds Examples -------- >>> stream = yield eventstream('127.0.0.1:8786', 0.100) # doctest: +SKIP >>> print(yield read(stream)) # doctest: +SKIP """ ip, port = coerce_to_address(address, out=tuple) stream = yield connect(ip, port) yield write(stream, {'op': 'feed', 'setup': dumps_function(AllProgress), 'function': dumps_function(counts), 'interval': interval, 'teardown': dumps_function(Scheduler.remove_plugin)}) raise gen.Return(stream) def nbytes_bar(nbytes): """ Convert nbytes message into rectangle placements >>> nbytes_bar({'inc': 1000, 'dec': 3000}) # doctest: +NORMALIZE_WHITESPACE {'names': ['dec', 'inc'], 'left': [0, 0.75], 'center': [0.375, 0.875], 'right': [0.75, 1.0]} """ total = sum(nbytes.values()) names = sorted(nbytes) d = {'name': [], 'text': [], 'left': [], 'right': [], 'center': [], 'color': [], 'percent': [], 'MB': []} if not total: return d right = 0 for name in names: left = right right = nbytes[name] / total + left center = (right + left) / 2 d['MB'].append(nbytes[name] / 1000000) d['percent'].append(round(nbytes[name] / total * 100, 2)) d['left'].append(left) d['right'].append(right) d['center'].append(center) d['color'].append(task_stream_palette[incrementing_index(name) % len(task_stream_palette)]) d['name'].append(name) if right - left > 0.1: d['text'].append(name) else: d['text'].append('') return d def progress_quads(msg, nrows=8, ncols=3): """ >>> msg = {'all': {'inc': 5, 'dec': 1, 'add': 4}, ... 'memory': {'inc': 2, 'dec': 0, 'add': 1}, ... 'erred': {'inc': 0, 'dec': 1, 'add': 0}, ... 'released': {'inc': 1, 'dec': 0, 'add': 1}} >>> progress_quads(msg, nrows=2) # doctest: +SKIP {'name': ['inc', 'add', 'dec'], 'left': [0, 0, 1], 'right': [0.9, 0.9, 1.9], 'top': [0, -1, 0], 'bottom': [-.8, -1.8, -.8], 'released': [1, 1, 0], 'memory': [2, 1, 0], 'erred': [0, 0, 1], 'done': ['3 / 5', '2 / 4', '1 / 1'], 'released-loc': [.2/.9, .25 / 0.9, 1], 'memory-loc': [3 / 5 / .9, .5 / 0.9, 1], 'erred-loc': [3 / 5 / .9, .5 / 0.9, 1.9]} """ width = 0.9 names = sorted(msg['all'], key=msg['all'].get, reverse=True) names = names[:nrows * ncols] n = len(names) d = {k: [v.get(name, 0) for name in names] for k, v in msg.items()} d['name'] = names d['show-name'] = [name if len(name) <= 15 else name[:12] + '...' for name in names] d['left'] = [i // nrows for i in range(n)] d['right'] = [i // nrows + width for i in range(n)] d['top'] = [-(i % nrows) for i in range(n)] d['bottom'] = [-(i % nrows) - 0.8 for i in range(n)] d['color'] = [task_stream_palette[incrementing_index(name) % len(task_stream_palette)] for name in names] d['released-loc'] = [] d['memory-loc'] = [] d['erred-loc'] = [] d['done'] = [] for r, m, e, a, l in zip(d['released'], d['memory'], d['erred'], d['all'], d['left']): rl = width * r / a + l ml = width * (r + m) / a + l el = width * (r + m + e) / a + l done = '%d / %d' % (r + m + e, a) d['released-loc'].append(rl) d['memory-loc'].append(ml) d['erred-loc'].append(el) d['done'].append(done) return d def task_stream_append(lists, msg, workers, palette=task_stream_palette): start, stop = msg['compute_start'], msg['compute_stop'] lists['start'].append((start + stop) / 2 * 1000) lists['duration'].append(1000 * (stop - start)) key = msg['key'] name = key_split(key) if msg['status'] == 'OK': color = palette[incrementing_index(name) % len(palette)] else: color = 'black' lists['key'].append(key) lists['name'].append(name) lists['color'].append(color) lists['alpha'].append(1) lists['worker'].append(msg['worker']) worker_thread = '%s-%d' % (msg['worker'], msg['thread']) lists['worker_thread'].append(worker_thread) if worker_thread not in workers: workers[worker_thread] = len(workers) lists['y'].append(workers[worker_thread]) if msg.get('transfer_start') is not None: start, stop = msg['transfer_start'], msg['transfer_stop'] lists['start'].append((start + stop) / 2 * 1000) lists['duration'].append(1000 * (stop - start)) lists['key'].append(key) lists['name'].append('transfer-to-' + name) lists['worker'].append(msg['worker']) lists['color'].append('#FF0020') lists['alpha'].append('0.4') lists['worker_thread'].append(worker_thread) lists['y'].append(workers[worker_thread]) if msg.get('disk_load_start') is not None: start, stop = msg['disk_load_start'], msg['disk_load_stop'] lists['start'].append((start + stop) / 2 * 1000) lists['duration'].append(1000 * (stop - start)) lists['key'].append(key) lists['name'].append('disk-load-' + name) lists['worker'].append(msg['worker']) lists['color'].append('#FF2000') lists['alpha'].append('0.4') lists['worker_thread'].append(worker_thread) lists['y'].append(workers[worker_thread]) distributed-1.14.3/distributed/diagnostics/progressbar.py000066400000000000000000000270601301211143400236510ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import logging from timeit import default_timer import sys from dask.core import flatten from toolz import valmap from tornado import gen from tornado.ioloop import IOLoop from .progress import format_time, Progress, MultiProgress from ..core import connect, read, write from ..client import default_client, futures_of from ..protocol.pickle import dumps from ..utils import sync, ignoring, key_split, is_kernel logger = logging.getLogger(__name__) def get_scheduler(scheduler): if scheduler is None: scheduler = default_client().scheduler return scheduler.ip, scheduler.port if isinstance(scheduler, str): scheduler = scheduler.split(':') if isinstance(scheduler, tuple): return scheduler raise TypeError("Expected None, string, or tuple") class ProgressBar(object): def __init__(self, keys, scheduler=None, interval=0.1, complete=True): self.scheduler = get_scheduler(scheduler) self.keys = {k.key if hasattr(k, 'key') else k for k in keys} self.interval = interval self.complete = complete self._start_time = default_timer() @property def elapsed(self): return default_timer() - self._start_time @gen.coroutine def listen(self): complete = self.complete keys = self.keys @gen.coroutine def setup(scheduler): p = Progress(keys, scheduler, complete=complete) yield p.setup() raise gen.Return(p) def function(scheduler, p): return {'all': len(p.all_keys), 'remaining': len(p.keys), 'status': p.status} self.stream = yield connect(*self.scheduler) logger.debug("Progressbar Connected to scheduler") yield write(self.stream, {'op': 'feed', 'setup': dumps(setup), 'function': dumps(function), 'interval': self.interval}) while True: response = yield read(self.stream) self._last_response = response self.status = response['status'] self._draw_bar(**response) if response['status'] in ('error', 'finished'): self.stream.close() self._draw_stop(**response) break logger.debug("Progressbar disconnected from scheduler") def _draw_stop(self, **kwargs): pass def __del__(self): with ignoring(AttributeError): self.stream.close() class TextProgressBar(ProgressBar): def __init__(self, keys, scheduler=None, interval=0.1, width=40, loop=None, complete=True, start=True): super(TextProgressBar, self).__init__(keys, scheduler, interval, complete) self.width = width self.loop = loop or IOLoop() if start: sync(self.loop, self.listen) def _draw_bar(self, remaining, all, **kwargs): frac = (1 - remaining / all) if all else 1.0 bar = '#' * int(self.width * frac) percent = int(100 * frac) elapsed = format_time(self.elapsed) msg = '\r[{0:<{1}}] | {2}% Completed | {3}'.format(bar, self.width, percent, elapsed) with ignoring(ValueError): sys.stdout.write(msg) sys.stdout.flush() class ProgressWidget(ProgressBar): """ ProgressBar that uses an IPython ProgressBar widget for the notebook See Also -------- progress: User function TextProgressBar: Text version suitable for the console """ def __init__(self, keys, scheduler=None, interval=0.1, complete=False, loop=None): super(ProgressWidget, self).__init__(keys, scheduler, interval, complete) from ipywidgets import FloatProgress, HBox, VBox, HTML self.elapsed_time = HTML('') self.bar = FloatProgress(min=0, max=1, description='', height = '10px') self.bar_text = HTML('', width = "140px") self.bar_widget = HBox([ self.bar_text, self.bar ]) self.widget = VBox([self.elapsed_time, self.bar_widget]) def _ipython_display_(self, **kwargs): IOLoop.current().add_callback(self.listen) return self.widget._ipython_display_(**kwargs) def _draw_stop(self, remaining, status, **kwargs): if status == 'error': self.bar.bar_style = 'danger' self.elapsed_time.value = '
Warning: the computation terminated due to an error after ' + format_time(self.elapsed) + '
' elif not remaining: self.bar.bar_style = 'success' def _draw_bar(self, remaining, all, **kwargs): ndone = all - remaining self.elapsed_time.value = '
Elapsed time: ' + format_time(self.elapsed) + '
' self.bar.value = ndone / all if all else 1.0 self.bar_text.value = '
%d / %d
' % (ndone, all) class MultiProgressBar(object): def __init__(self, keys, scheduler=None, func=key_split, interval=0.1, complete=False): self.scheduler = get_scheduler(scheduler) self.keys = {k.key if hasattr(k, 'key') else k for k in keys} self.func = func self.interval = interval self.complete = complete self._start_time = default_timer() @property def elapsed(self): return default_timer() - self._start_time @gen.coroutine def listen(self): complete = self.complete keys = self.keys func = self.func @gen.coroutine def setup(scheduler): p = MultiProgress(keys, scheduler, complete=complete, func=func) yield p.setup() raise gen.Return(p) def function(scheduler, p): return {'all': valmap(len, p.all_keys), 'remaining': valmap(len, p.keys), 'status': p.status} self.stream = yield connect(*self.scheduler) logger.debug("Progressbar Connected to scheduler") yield write(self.stream, {'op': 'feed', 'setup': dumps(setup), 'function': dumps(function), 'interval': self.interval}) while True: response = yield read(self.stream) self._last_response = response self.status = response['status'] self._draw_bar(**response) if response['status'] in ('error', 'finished'): self.stream.close() self._draw_stop(**response) break logger.debug("Progressbar disconnected from scheduler") def _draw_stop(self, **kwargs): pass def __del__(self): with ignoring(AttributeError): self.stream.close() class MultiProgressWidget(MultiProgressBar): """ Multiple progress bar Widget suitable for the notebook Displays multiple progress bars for a computation, split on computation type. See Also -------- progress: User-level function <--- use this MultiProgress: Non-visualization component that contains most logic ProgressWidget: Single progress bar widget """ def __init__(self, keys, scheduler=None, minimum=0, interval=0.1, func=key_split, complete=False): super(MultiProgressWidget, self).__init__(keys, scheduler, func, interval, complete) from ipywidgets import VBox self.widget = VBox([]) def make_widget(self, all): from ipywidgets import FloatProgress, HBox, VBox, HTML import cgi self.elapsed_time = HTML('') self.bars = {key: FloatProgress(min=0, max=1, description='', height='10px') for key in all} self.bar_texts = {key: HTML('', width = "140px") for key in all} self.bar_labels = {key: HTML('
' + cgi.escape(key.decode() if isinstance(key, bytes) else key) + '
') for key in all} def key(kv): """ Order keys by most numerous, then by string name """ return kv[::-1] key_order = [k for k, v in sorted(all.items(), key=key, reverse=True)] self.bar_widgets = VBox([ HBox([ self.bar_texts[key], self.bars[key], self.bar_labels[key] ]) for key in key_order ]) self.widget.children = (self.elapsed_time, self.bar_widgets) def _ipython_display_(self, **kwargs): IOLoop.current().add_callback(self.listen) return self.widget._ipython_display_(**kwargs) def _draw_stop(self, remaining, status, exception=None, key=None, **kwargs): for k, v in remaining.items(): if not v: self.bars[k].bar_style = 'success' """ TODO if status == 'error': self.bars[self.func(key)].bar_style = 'danger' self.elapsed_time.value = '
Warning: the computation terminated due to an error after ' + format_time(self.elapsed) + '
' """ def _draw_bar(self, remaining, all, status, **kwargs): if self.keys and not self.widget.children: self.make_widget(all) for k, ntasks in all.items(): ndone = ntasks - remaining[k] self.elapsed_time.value = '
Elapsed time: ' + format_time(self.elapsed) + '
' self.bars[k].value = ndone / ntasks if ntasks else 1.0 self.bar_texts[k].value = '
%d / %d
' % (ndone, ntasks) def progress(*futures, **kwargs): """ Track progress of futures This operates differently in the notebook and the console * Notebook: This returns immediately, leaving an IPython widget on screen * Console: This blocks until the computation completes Parameters ---------- futures: Futures A list of futures or keys to track notebook: bool (optional) Running in the notebook or not (defaults to guess) multi: bool (optional) Track different functions independently (defaults to True) complete: bool (optional) Track all keys (True) or only keys that have not yet run (False) (defaults to True) Notes ----- In the notebook, the output of `progress` must be the last statement in the cell. Typically, this means calling `progress` at the end of a cell. Examples -------- >>> progress(futures) # doctest: +SKIP [########################################] | 100% Completed | 1.7s """ notebook = kwargs.pop('notebook', None) multi = kwargs.pop('multi', True) complete = kwargs.pop('complete', True) assert not kwargs futures = futures_of(futures) if not isinstance(futures, (set, list)): futures = [futures] if notebook is None: notebook = is_kernel() # often but not always correct assumption if notebook: if multi: bar = MultiProgressWidget(futures, complete=complete) else: bar = ProgressWidget(futures, complete=complete) return bar else: TextProgressBar(futures, complete=complete) distributed-1.14.3/distributed/diagnostics/scheduler.py000066400000000000000000000043441301211143400232760ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import os from time import time try: from cytoolz import valmap except ImportError: from toolz import valmap from toolz import countby, concat, dissoc from ..utils import key_split, log_errors def tasks(s): """ Task and worker status of scheduler """ processing = sum(map(len, s.processing.values())) with log_errors(): return {'processing': processing, 'total': len(s.tasks), 'in-memory': len(s.who_has), 'ready': len(s.ready) + sum(map(len, s.stacks.values())), 'waiting': len(s.waiting), 'failed': len(s.exceptions_blame)} def workers(s): """ Information about workers Examples -------- >>> workers(my_scheduler) # doctest: +SKIP {'127.0.0.1': {'cores': 3, 'cpu': 0.0, 'last-seen': 0.003068, 'latency': 0.01584628690034151, 'ports': ['54871', '50943'], 'processing': {'inc': 2, 'add': 1}, 'disk-read': 1234, 'disk-write': 1234, 'network-send': 1234, 'network-recv': 1234, 'memory': 16701911040, 'memory_percent': 85}} """ hosts = {host: ['%s:%s' % (host, port) for port in d['ports']] for host, d in s.host_info.items()} processing = {host: countby(key_split, concat(s.processing[w] for w in addrs)) for host, addrs in hosts.items()} now = time() result = {} for host, info in s.host_info.items(): info = info.copy() # info = dissoc(info, 'heartbeat', 'heartbeat-port') info['processing'] = processing[host] result[host] = info info['ports'] = list(info['ports']) if 'last-seen' in info: info['last-seen'] = (now - info['last-seen']) return result def processing(s): return {'processing': valmap(len, s.processing), 'stacks': valmap(len, s.stacks), 'ready': len(s.ready), 'waiting': len(s.waiting), 'memory': len(s.who_has), 'ncores': s.ncores} distributed-1.14.3/distributed/diagnostics/tests/000077500000000000000000000000001301211143400221035ustar00rootroot00000000000000distributed-1.14.3/distributed/diagnostics/tests/test_eventstream.py000066400000000000000000000023711301211143400260540ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import pytest from tornado import gen from distributed import Client, Scheduler, Worker from distributed.core import read from distributed.client import _wait from distributed.diagnostics.eventstream import EventStream, eventstream from distributed.utils_test import inc, div, dec, gen_cluster from distributed.worker import dumps_task from time import time, sleep @gen_cluster(client=True) def test_eventstream(c, s, a, b): es = EventStream() s.add_plugin(es) assert es.buffer == [] futures = c.map(div, [1] * 10, range(10)) yield _wait(futures) assert len(es.buffer) == 10 @gen_cluster(client=True) def test_eventstream_remote(c, s, a, b): stream = yield eventstream(s.address, interval=0.010) start = time() while not s.plugins: yield gen.sleep(0.01) assert time() < start + 5 futures = c.map(div, [1] * 10, range(10)) start = time() total = [] while len(total) < 10: msgs = yield read(stream) assert isinstance(msgs, list) total.extend(msgs) assert time() < start + 5 stream.close() start = time() while s.plugins: yield gen.sleep(0.01) assert time() < start + 5 distributed-1.14.3/distributed/diagnostics/tests/test_plugin.py000066400000000000000000000015601301211143400250140ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from distributed.utils_test import inc, gen_cluster from distributed.diagnostics.plugin import SchedulerPlugin @gen_cluster(client=True) def test_diagnostic(c, s, a, b): class Counter(SchedulerPlugin): def start(self, scheduler): self.scheduler = scheduler scheduler.add_plugin(self) self.count = 0 def transition(self, key, start, finish, *args, **kwargs): if start == 'processing' and finish == 'memory': self.count += 1 counter = Counter() counter.start(s) assert counter in s.plugins assert counter.count == 0 x = c.submit(inc, 1) y = c.submit(inc, x) z = c.submit(inc, y) yield z._result() assert counter.count == 3 s.remove_plugin(counter) assert counter not in s.plugins distributed-1.14.3/distributed/diagnostics/tests/test_progress.py000066400000000000000000000121711301211143400253620ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from operator import add import pytest import sys from time import time from toolz import valmap from tornado import gen from tornado.queues import Queue from dask.core import get_deps from distributed.client import _wait from distributed.worker import dumps_task from distributed import Nanny from distributed.utils_test import (gen_cluster, cluster, inc, dec, gen_test, div) from distributed.utils import All, key_split from distributed.diagnostics.progress import (Progress, SchedulerPlugin, AllProgress, MultiProgress, dependent_keys) from distributed.protocol.pickle import dumps def test_dependent_keys(): a, b, c, d, e, f, g = 'abcdefg' who_has = {a: [1], b: [1]} processing = {'alice': {c}} stacks = {'bob': [d]} exceptions = {} dsk = {a: 1, b: 2, c: (add, a, b), d: (inc, a), e: (add, c, d), f: (inc, e)} dependencies, dependents = get_deps(dsk) assert dependent_keys(f, who_has, processing, stacks, dependencies, exceptions, complete=False)[0] == {f, e, c, d} assert dependent_keys(f, who_has, processing, stacks, dependencies, exceptions, complete=True)[0] == {a, b, c, d, e, f} def f(*args): pass def g(*args): pass def h(*args): pass @gen_cluster(client=True) def test_many_Progress(c, s, a, b): x = c.submit(f, 1) y = c.submit(g, x) z = c.submit(h, y) bars = [Progress(keys=[z], scheduler=s) for i in range(10)] yield [b.setup() for b in bars] yield z._result() start = time() while not all(b.status == 'finished' for b in bars): yield gen.sleep(0.1) assert time() < start + 2 @gen_cluster(client=True) def test_multiprogress(c, s, a, b): x1 = c.submit(f, 1) x2 = c.submit(f, x1) x3 = c.submit(f, x2) y1 = c.submit(g, x3) y2 = c.submit(g, y1) p = MultiProgress([y2], scheduler=s, complete=True) yield p.setup() assert p.all_keys == {'f': {f.key for f in [x1, x2, x3]}, 'g': {f.key for f in [y1, y2]}} yield x3._result() assert p.keys['f'] == set() yield y2._result() assert p.keys == {'f': set(), 'g': set()} assert p.status == 'finished' @gen_cluster(client=True) def test_robust_to_bad_plugin(c, s, a, b): class Bad(SchedulerPlugin): def transition(self, key, start, finish, **kwargs): raise Exception() bad = Bad() s.add_plugin(bad) x = c.submit(inc, 1) y = c.submit(inc, x) result = yield y._result() assert result == 3 def check_bar_completed(capsys, width=40): out, err = capsys.readouterr() bar, percent, time = [i.strip() for i in out.split('\r')[-1].split('|')] assert bar == '[' + '#'*width + ']' assert percent == '100% Completed' @gen_cluster(client=True, Worker=Nanny, timeout=None) def test_AllProgress(c, s, a, b): x, y, z = c.map(inc, [1, 2, 3]) xx, yy, zz = c.map(dec, [x, y, z]) yield _wait([x, y, z]) p = AllProgress(s) assert p.all['inc'] == {x.key, y.key, z.key} assert p.state['memory']['inc'] == {x.key, y.key, z.key} assert p.state['released'] == {} assert p.state['erred'] == {} assert 'inc' in p.nbytes assert isinstance(p.nbytes['inc'], int) assert p.nbytes['inc'] > 0 yield _wait([xx, yy, zz]) assert p.all['dec'] == {xx.key, yy.key, zz.key} assert p.state['memory']['dec'] == {xx.key, yy.key, zz.key} assert p.state['released'] == {} assert p.state['erred'] == {} assert p.nbytes['inc'] == p.nbytes['dec'] t = c.submit(sum, [x, y, z]) yield t._result() keys = {x.key, y.key, z.key} del x, y, z import gc; gc.collect() while any(k in s.who_has for k in keys): yield gen.sleep(0.01) assert p.state['released']['inc'] == keys assert p.all['inc'] == keys assert p.all['dec'] == {xx.key, yy.key, zz.key} if 'inc' in p.nbytes: assert p.nbytes['inc'] == 0 xxx = c.submit(div, 1, 0) yield _wait([xxx]) assert p.state['erred'] == {'div': {xxx.key}} tkey = t.key del xx, yy, zz, t import gc; gc.collect() while tkey in s.task_state: yield gen.sleep(0.01) for coll in [p.all, p.nbytes] + list(p.state.values()): assert 'inc' not in coll assert 'dec' not in coll def f(x): return x for i in range(4): future = c.submit(f, i) import gc; gc.collect() yield gen.sleep(1) yield _wait([future]) assert p.state['memory'] == {'f': {future.key}} yield c._restart() for coll in [p.all] + list(p.state.values()): assert not coll x = c.submit(div, 1, 2) yield _wait([x]) assert set(p.all) == {'div'} assert all(set(d) == {'div'} for d in p.state.values()) @gen_cluster(client=True, Worker=Nanny) def test_AllProgress_lost_key(c, s, a, b, timeout=None): p = AllProgress(s) futures = c.map(inc, range(5)) yield _wait(futures) assert len(p.state['memory']['inc']) == 5 yield a._close() yield b._close() start = time() while len(p.state['memory']['inc']) > 0: yield gen.sleep(0.1) assert time() < start + 2 distributed-1.14.3/distributed/diagnostics/tests/test_progress_stream.py000066400000000000000000000107431301211143400267400ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import pytest from tornado import gen from dask import do from distributed.core import read from distributed.client import _wait from distributed.diagnostics.progress_stream import (progress_quads, nbytes_bar, progress_stream, task_stream_append) from distributed.utils_test import inc, div, dec, gen_cluster from distributed.worker import dumps_task from time import time, sleep def test_progress_quads(): msg = {'all': {'inc': 5, 'dec': 1, 'add': 4}, 'memory': {'inc': 2, 'dec': 0, 'add': 1}, 'erred': {'inc': 0, 'dec': 1, 'add': 0}, 'released': {'inc': 1, 'dec': 0, 'add': 1}} d = progress_quads(msg, nrows=2) color = d.pop('color') assert len(set(color)) == 3 assert d == {'name': ['inc', 'add', 'dec'], 'show-name': ['inc', 'add', 'dec'], 'left': [0, 0, 1], 'right': [0.9, 0.9, 1.9], 'top': [0, -1, 0], 'bottom': [-.8, -1.8, -.8], 'all': [5, 4, 1], 'released': [1, 1, 0], 'memory': [2, 1, 0], 'erred': [0, 0, 1], 'done': ['3 / 5', '2 / 4', '1 / 1'], 'released-loc': [.9 * 1 / 5, .25 * 0.9, 1.0], 'memory-loc': [.9 * 3 / 5, .5 * 0.9, 1.0], 'erred-loc': [.9 * 3 / 5, .5 * 0.9, 1.9]} def test_progress_quads_too_many(): keys = ['x-%d' % i for i in range(1000)] msg = {'all': {k: 1 for k in keys}, 'memory': {k: 0 for k in keys}, 'erred': {k: 0 for k in keys}, 'released': {k: 0 for k in keys}} d = progress_quads(msg, nrows=6, ncols=3) assert len(d['name']) == 6 * 3 @gen_cluster(client=True, timeout=None) def test_progress_stream(c, s, a, b): futures = c.map(div, [1] * 10, range(10)) x = 1 for i in range(5): x = do(inc)(x) future = c.compute(x) yield _wait(futures + [future]) stream = yield progress_stream(s.address, interval=0.010) msg = yield read(stream) nbytes = msg.pop('nbytes') assert msg == {'all': {'div': 10, 'inc': 5, 'finalize': 1}, 'erred': {'div': 1}, 'memory': {'div': 9, 'finalize': 1}, 'released': {'inc': 5}} assert set(nbytes) == set(msg['all']) assert all(v > 0 for v in nbytes.values()) assert progress_quads(msg) stream.close() def test_nbytes_bar(): nbytes = {'inc': 1000, 'dec': 3000} expected = {'name': ['dec', 'inc'], 'left': [0, 0.75], 'center': [0.375, 0.875], 'right': [0.75, 1.0], 'percent': [75, 25], 'MB': [0.003, 0.001], 'text': ['dec', 'inc']} result = nbytes_bar(nbytes) color = result.pop('color') assert len(set(color)) == 2 assert result == expected def test_progress_quads_many_functions(): funcnames = ['fn%d' % i for i in range(1000)] msg = {'all': {fn: 1 for fn in funcnames}, 'memory': {fn: 1 for fn in funcnames}, 'erred': {fn: 0 for fn in funcnames}, 'released': {fn: 0 for fn in funcnames}} d = progress_quads(msg, nrows=2) color = d.pop('color') assert len(set(color)) < 100 def test_task_stream_append(): msgs = [{'status': 'OK', 'compute_start': 10, 'compute_stop': 20, 'key':'inc-1', 'thread': 5855, 'worker':'127.0.0.1:9999'}, {'status': 'OK', 'compute_start': 15, 'compute_stop': 25, 'key':'inc-2', 'thread': 6000, 'worker':'127.0.0.1:9999'}, {'status': 'error', 'compute_start': 10, 'compute_stop': 14, 'key':'inc-3', 'thread': 6000, 'worker':'127.0.0.1:9999'}, {'status': 'OK', 'compute_start': 10, 'compute_stop': 30, 'transfer_start': 8, 'transfer_stop': 10, 'key':'add-1', 'thread': 4000, 'worker':'127.0.0.2:9999'}] lists = {name: [] for name in 'start duration key name color worker worker_thread y alpha'.split()} workers = {'127.0.0.1:9999-5855': 0} for msg in msgs: task_stream_append(lists, msg, workers) assert len(lists['start']) == len(msgs) + 1 # +1 for a transfer assert len(workers) == 3 assert set(workers) == set(lists['worker_thread']) assert set(workers.values()) == set(range(len(workers))) assert lists['color'][-1] == '#FF0020' L = lists['color'] assert L[0] == L[1] assert L[2] == 'black' assert L[3] != L[0] distributed-1.14.3/distributed/diagnostics/tests/test_progressbar.py000066400000000000000000000054611301211143400260530ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import pytest from tornado import gen from distributed import Client, Scheduler, Worker from distributed.diagnostics.progressbar import TextProgressBar, progress from distributed.utils_test import (cluster, loop, inc, div, dec, gen_cluster) from distributed.worker import dumps_task from time import time, sleep def test_text_progressbar(capsys, loop): with cluster(nanny=True) as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: futures = c.map(inc, range(10)) p = TextProgressBar(futures, interval=0.01, complete=True) c.gather(futures) start = time() while p.status != 'finished': sleep(0.01) assert time() - start < 5 check_bar_completed(capsys) assert p._last_response == {'all': 10, 'remaining': 0, 'status': 'finished'} assert p.stream.closed() @gen_cluster(client=True) def test_TextProgressBar_error(c, s, a, b): x = c.submit(div, 1, 0) progress = TextProgressBar([x.key], scheduler=(s.ip, s.port), start=False, interval=0.01) yield progress.listen() assert progress.status == 'error' assert progress.stream.closed() progress = TextProgressBar([x.key], scheduler=(s.ip, s.port), start=False, interval=0.01) yield progress.listen() assert progress.status == 'error' assert progress.stream.closed() def test_TextProgressBar_empty(loop, capsys): @gen.coroutine def f(): s = Scheduler(loop=loop) done = s.start(0) a = Worker(s.ip, s.port, loop=loop, ncores=1) b = Worker(s.ip, s.port, loop=loop, ncores=1) yield [a._start(0), b._start(0)] progress = TextProgressBar([], scheduler=(s.ip, s.port), start=False, interval=0.01) yield progress.listen() assert progress.status == 'finished' check_bar_completed(capsys) yield [a._close(), b._close()] s.close() yield done loop.run_sync(f) def check_bar_completed(capsys, width=40): out, err = capsys.readouterr() bar, percent, time = [i.strip() for i in out.split('\r')[-1].split('|')] assert bar == '[' + '#'*width + ']' assert percent == '100% Completed' def test_progress_function(loop, capsys): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: f = c.submit(lambda: 1) g = c.submit(lambda: 2) progress([[f], [[g]]], notebook=False) check_bar_completed(capsys) progress(f) check_bar_completed(capsys) distributed-1.14.3/distributed/diagnostics/tests/test_scheduler_diagnostics.py000066400000000000000000000026431301211143400300660ustar00rootroot00000000000000import json from tornado import gen from distributed.utils_test import gen_cluster, div from distributed.diagnostics.scheduler import tasks, workers from distributed.client import _wait @gen_cluster(client=True) def test_tasks(c, s, a, b): d = tasks(s) assert d['failed'] == 0 assert d['in-memory'] == 0 assert d['ready'] == 0 assert d['total'] == 0 assert d['waiting'] == 0 L = c.map(div, range(10), range(10)) yield _wait(L) d = tasks(s) assert d['failed'] == 1 assert d['in-memory'] == 9 assert d['ready'] == 0 assert d['total'] == 10 assert d['waiting'] == 0 @gen_cluster(client=True) def test_workers(c, s, a, b): d = workers(s) assert json.loads(json.dumps(d)) == d assert 0 <= d[a.ip]['cpu'] <= 100 assert 0 <= d[a.ip]['memory'] assert 0 < d[a.ip]['memory_percent'] < 100 assert set(map(int, d[a.ip]['ports'])) == {a.port, b.port} assert d[a.ip]['processing'] == {} # assert d[a.ip]['last-seen'] > 0 L = c.map(div, range(10), range(10)) yield _wait(L) assert 0 <= d[a.ip]['cpu'] <= 100 assert 0 <= d[a.ip]['memory'] assert 0 < d[a.ip]['memory_percent'] < 100 assert set(map(int, d[a.ip]['ports'])) == {a.port, b.port} assert d[a.ip]['processing'] == {} assert 0 <= d[a.ip]['disk-read'] assert 0 <= d[a.ip]['disk-write'] assert 0 <= d[a.ip]['network-send'] assert 0 <= d[a.ip]['network-recv'] distributed-1.14.3/distributed/diagnostics/tests/test_widgets.py000066400000000000000000000176021301211143400251700ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import pytest pytest.importorskip('ipywidgets') from ipykernel.comm import Comm import ipywidgets as widgets from ipywidgets import Widget ################# # Utility stuff # ################# # Taken from ipywidgets/widgets/tests/test_interaction.py # https://github.com/ipython/ipywidgets # Licensed under Modified BSD. Copyright IPython Development Team. See: # https://github.com/ipython/ipywidgets/blob/master/COPYING.md class DummyComm(Comm): comm_id = 'a-b-c-d' def open(self, *args, **kwargs): pass def send(self, *args, **kwargs): pass def close(self, *args, **kwargs): pass _widget_attrs = {} displayed = [] undefined = object() def setup(): _widget_attrs['_comm_default'] = getattr(Widget, '_comm_default', undefined) Widget._comm_default = lambda self: DummyComm() _widget_attrs['_ipython_display_'] = Widget._ipython_display_ def raise_not_implemented(*args, **kwargs): raise NotImplementedError() Widget._ipython_display_ = raise_not_implemented def teardown(): for attr, value in _widget_attrs.items(): if value is undefined: delattr(Widget, attr) else: setattr(Widget, attr, value) def f(**kwargs): pass def clear_display(): global displayed displayed = [] def record_display(*args): displayed.extend(args) # End code taken from ipywidgets ##################### # Distributed stuff # ##################### from operator import add import re from toolz import valmap from distributed.client import Client, wait from distributed.worker import dumps_task from distributed.utils_test import (cluster, loop, inc, div, dec, throws, gen_cluster) from distributed.utils import sync from distributed.diagnostics.progressbar import (ProgressWidget, MultiProgressWidget, progress) @gen_cluster() def test_progressbar_widget(s, a, b): s.update_graph(tasks=valmap(dumps_task, {'x': (inc, 1), 'y': (inc, 'x'), 'z': (inc, 'y')}), keys=['z'], dependencies={'y': {'x'}, 'z': {'y'}}) progress = ProgressWidget(['z'], scheduler=(s.ip, s.port)) yield progress.listen() assert progress.bar.value == 1.0 assert '3 / 3' in progress.bar_text.value progress = ProgressWidget(['z'], scheduler=(s.ip, s.port)) yield progress.listen() @gen_cluster() def test_multi_progressbar_widget(s, a, b): s.update_graph(tasks=valmap(dumps_task, {'x-1': (inc, 1), 'x-2': (inc, 'x-1'), 'x-3': (inc, 'x-2'), 'y-1': (dec, 'x-3'), 'y-2': (dec, 'y-1'), 'e': (throws, 'y-2'), 'other': (inc, 123)}), keys=['e'], dependencies={'x-2': ['x-1'], 'x-3': ['x-2'], 'y-1': ['x-3'], 'y-2': ['y-1'], 'e': ['y-2']}) p = MultiProgressWidget(['e'], scheduler=(s.ip, s.port)) yield p.listen() assert p.bars['x'].value == 1.0 assert p.bars['y'].value == 1.0 assert p.bars['e'].value == 0.0 assert '3 / 3' in p.bar_texts['x'].value assert '2 / 2' in p.bar_texts['y'].value assert '0 / 1' in p.bar_texts['e'].value assert p.bars['x'].bar_style == 'success' assert p.bars['y'].bar_style == 'success' # assert p.bars['e'].bar_style == 'danger' assert p.status == 'error' capacities = [int(re.search(r'\d+ / \d+', row.children[0].value) .group().split(' / ')[1]) for row in p.bar_widgets.children] assert sorted(capacities, reverse=True) == capacities @gen_cluster() def test_multi_progressbar_widget_after_close(s, a, b): s.update_graph(tasks=valmap(dumps_task, {'x-1': (inc, 1), 'x-2': (inc, 'x-1'), 'x-3': (inc, 'x-2'), 'y-1': (dec, 'x-3'), 'y-2': (dec, 'y-1'), 'e': (throws, 'y-2'), 'other': (inc, 123)}), keys=['e'], dependencies={'x-2': {'x-1'}, 'x-3': {'x-2'}, 'y-1': {'x-3'}, 'y-2': {'y-1'}, 'e': {'y-2'}}) p = MultiProgressWidget(['x-1', 'x-2', 'x-3'], scheduler=(s.ip, s.port)) yield p.listen() assert 'x' in p.bars def test_values(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: L = [c.submit(inc, i) for i in range(5)] wait(L) p = MultiProgressWidget(L) sync(loop, p.listen) assert set(p.bars) == {'inc'} assert p.status == 'finished' assert p.stream.closed() assert '5 / 5' in p.bar_texts['inc'].value assert p.bars['inc'].value == 1.0 x = c.submit(throws, 1) p = MultiProgressWidget([x]) sync(loop, p.listen) assert p.status == 'error' def test_progressbar_done(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: L = [c.submit(inc, i) for i in range(5)] wait(L) p = ProgressWidget(L) sync(loop, p.listen) assert p.status == 'finished' assert p.bar.value == 1.0 assert p.bar.bar_style == 'success' f = c.submit(throws, L) wait([f]) p = ProgressWidget([f]) sync(loop, p.listen) assert p.status == 'error' assert p.bar.value == 0.0 assert p.bar.bar_style == 'danger' def test_progressbar_cancel(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: import time L = [c.submit(lambda: time.sleep(0.3), i) for i in range(5)] p = ProgressWidget(L) sync(loop, p.listen) L[-1].cancel() wait(L[:-1]) assert p.status == 'error' assert p.bar.value == 0 # no tasks finish before cancel is called @gen_cluster() def test_multibar_complete(s, a, b): s.update_graph(tasks=valmap(dumps_task, {'x-1': (inc, 1), 'x-2': (inc, 'x-1'), 'x-3': (inc, 'x-2'), 'y-1': (dec, 'x-3'), 'y-2': (dec, 'y-1'), 'e': (throws, 'y-2'), 'other': (inc, 123)}), keys=['e'], dependencies={'x-2': {'x-1'}, 'x-3': {'x-2'}, 'y-1': {'x-3'}, 'y-2': {'y-1'}, 'e': {'y-2'}}) p = MultiProgressWidget(['e'], scheduler=(s.ip, s.port), complete=True) yield p.listen() assert p._last_response['all'] == {'x': 3, 'y': 2, 'e': 1} assert all(b.value == 1.0 for k, b in p.bars.items() if k != 'e') assert '3 / 3' in p.bar_texts['x'].value assert '2 / 2' in p.bar_texts['y'].value def test_fast(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: L = c.map(inc, range(100)) L2 = c.map(dec, L) L3 = c.map(add, L, L2) p = progress(L3, multi=True, complete=True, notebook=True) sync(loop, p.listen) assert set(p._last_response['all']) == {'inc', 'dec', 'add'} distributed-1.14.3/distributed/hdfs.py000066400000000000000000000107271301211143400177370ustar00rootroot00000000000000""" This file is experimental and may disappear without warning """ from __future__ import print_function, division, absolute_import import logging from math import log import os from warnings import warn import dask.bytes.core from dask.delayed import delayed from toolz import merge from hdfs3 import HDFileSystem from .client import default_client, ensure_default_get logger = logging.getLogger(__name__) def walk_glob(hdfs, path): if '*' not in path and hdfs.info(path)['kind'] == 'directory': return sorted([fn for fn in hdfs.walk(path) if fn[-1] != '/']) else: return sorted(hdfs.glob(path)) def get_block_locations(hdfs, filename): """ Get block locations from a filename or globstring """ return [merge({'filename': fn}, block) for fn in walk_glob(hdfs, filename) for block in hdfs.get_block_locations(fn)] def read_block_from_hdfs(filename, offset, length, host=None, port=None, delimiter=None): from locket import lock_file with lock_file('.lock'): hdfs = HDFileSystem(host=host, port=port) bytes = hdfs.read_block(filename, offset, length, delimiter) return bytes def open_file_write(paths, hdfs=None, **kwargs): """ Open list of files using delayed """ if hdfs is None: hdfs = HDFileSystem(kwargs.get('host'), kwargs.get('port')) out = [delayed(hdfs.open)(path, 'wb') for path in paths] return out def open_file_write_direct(path, hdfs=None, **kwargs): if hdfs is None: hdfs = HDFileSystem(kwargs.get('host'), kwargs.get('port')) return hdfs.open(path, 'wb') dask.bytes.core._open_files_write['hdfs'] = open_file_write_direct def read_bytes(path, client=None, hdfs=None, lazy=True, delimiter=None, not_zero=False, sample=True, blocksize=None, compression=None, **hdfs_auth): """ Convert location in HDFS to a list of distributed futures Parameters ---------- path: string location in HDFS client: Client (optional) defaults to most recently created client hdfs: HDFileSystem (optional) lazy: boolean (optional) If True then return lazily evaluated dask Values delimiter: bytes An optional delimiter, like ``b'\n'`` on which to split blocks of bytes not_zero: force seek of start-of-file delimiter, discarding header **hdfs_auth: keyword arguments Extra keywords to send to ``hdfs3.HDFileSystem`` Returns ------- List of ``distributed.Future`` objects if ``lazy=False`` or ``dask.Value`` objects if ``lazy=True`` """ if compression: raise NotImplementedError("hdfs compression") hdfs = hdfs or HDFileSystem(**hdfs_auth) client = default_client(client) blocks = get_block_locations(hdfs, path) filenames = [d['filename'] for d in blocks] offsets = [d['offset'] for d in blocks] if not_zero: offsets = [max([o, 1]) for o in offsets] lengths = [d['length'] for d in blocks] workers = [[h.decode() for h in d['hosts']] for d in blocks] logger.debug("Read %d blocks of binary bytes from %s", len(blocks), path) if sample is True: sample = 10000 if sample: with hdfs.open(filenames[0], 'rb') as f: sample = f.read(sample) else: sample = b'' f = delayed(read_block_from_hdfs, pure=True) values = [f(fn, offset, length, hdfs.host, hdfs.port, delimiter) for fn, offset, length in zip(filenames, offsets, lengths)] restrictions = {v.key: w for v, w in zip(values, workers)} client._send_to_scheduler({'op': 'update-graph', 'tasks': {}, 'dependencies': [], 'keys': [], 'restrictions': restrictions, 'loose_restrictions': list(restrictions), 'client': client.id}) return sample, values dask.bytes.core._read_bytes['hdfs'] = read_bytes def hdfs_open_file(path, auth): hdfs = HDFileSystem(**auth) return hdfs.open(path, mode='rb') def open_files(path, hdfs=None, lazy=None, **auth): if lazy is not None: raise DeprecationWarning("Lazy keyword has been deprecated. " "Now always lazy") hdfs = hdfs or HDFileSystem(**auth) filenames = sorted(hdfs.glob(path)) myopen = delayed(hdfs_open_file) return [myopen(fn, auth) for fn in filenames] dask.bytes.core._open_files['hdfs'] = open_files distributed-1.14.3/distributed/http/000077500000000000000000000000001301211143400174115ustar00rootroot00000000000000distributed-1.14.3/distributed/http/__init__.py000066400000000000000000000002061301211143400215200ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from .worker import HTTPWorker from .scheduler import HTTPScheduler distributed-1.14.3/distributed/http/core.py000066400000000000000000000045211301211143400207150ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import json import logging import os import socket from toolz import first from tornado import web, gen from tornado.httpclient import AsyncHTTPClient from tornado.httpserver import HTTPServer from ..utils import log_errors logger = logging.getLogger(__name__) class RequestHandler(web.RequestHandler): def initialize(self, server=None): logger.debug("Connection %s", self.request.uri) if server: self.server = server def resource_collect(pid=None): """Gather system usage stats. Returns empty dict `{}` if psutil is not installed. Parameters ---------- pid : None (default) or int process to check - this one if None """ try: import psutil except ImportError: return {} p = psutil.Process(pid or os.getpid()) return {'cpu_percent': psutil.cpu_percent(), 'status': p.status(), 'memory_percent': p.memory_percent(), 'memory_info_ex': p.memory_info_ex(), 'disk_io_counters': psutil.disk_io_counters(), 'net_io_counters': psutil.net_io_counters()} class Resources(RequestHandler): """Served details about this process and machine""" def get(self): self.write(resource_collect()) class Proxy(RequestHandler): """Send REST call to specific worker return its response""" @gen.coroutine def get(self, ip, port, rest): client = AsyncHTTPClient() response = yield client.fetch("http://%s:%s/%s" % (ip, port, rest)) self.write(response.body) # TODO: capture more data of response class MyApp(HTTPServer): @property def port(self): if not hasattr(self, '_port'): try: self._port = first(self._sockets.values()).getsockname()[1] except StopIteration: raise OSError("Server has no port. Please call .listen first") return self._port def listen(self, port): while True: try: super(MyApp, self).listen(port) break except (socket.error, OSError) as e: if port: raise else: logger.info('Randomly assigned port taken for %s. Retrying', type(self).__name__) distributed-1.14.3/distributed/http/scheduler.py000066400000000000000000000073571301211143400217550ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from collections import defaultdict import json import logging from tornado import web, gen from tornado.httpclient import AsyncHTTPClient from .core import RequestHandler, MyApp, Resources, Proxy from ..utils import key_split, log_errors from ..compatibility import unicode logger = logging.getLogger(__name__) def ensure_string(s): if not isinstance(s, unicode): s = s.decode() return s class Info(RequestHandler): """ Basic info about the scheduler """ def get(self): self.write({'ncores': self.server.ncores, 'status': self.server.status}) class Processing(RequestHandler): """ Active tasks on each worker """ def get(self): resp = {addr: [ensure_string(key_split(t)) for t in tasks] for addr, tasks in self.server.processing.items()} self.write(resp) class Stacks(RequestHandler): """ Active tasks on each worker """ def get(self): resp = {addr: [ensure_string(key_split(t)) for t in tasks] for addr, tasks in self.server.stacks.items()} self.write(resp) class Broadcast(RequestHandler): """ Send call to all workers, collate their responses """ @gen.coroutine def get(self, rest): addresses = [addr.split(':') + [d['services']['http']] for addr, d in self.server.worker_info.items() if 'http' in d['services']] client = AsyncHTTPClient() responses = {'%s:%s' % (ip, tcp_port): client.fetch("http://%s:%s/%s" % (ip, http_port, rest)) for ip, tcp_port, http_port in addresses} responses2 = yield responses responses3 = {k: json.loads(v.body.decode()) for k, v in responses2.items()} self.write(responses3) # TODO: capture more data of response class MemoryLoad(RequestHandler): """The total amount of data held in memory by workers""" def get(self): self.write({worker: sum(self.server.nbytes[k] for k in keys) for worker, keys in self.server.has_what.items()}) class MemoryLoadByKey(RequestHandler): """The total amount of data held in memory by workers""" def get(self): out = {} for worker, keys in self.server.has_what.items(): d = defaultdict(lambda: 0) for key in keys: d[key_split(key)] += self.server.nbytes[key] out[worker] = {k: v for k, v in d.items()} self.write(out) class Tasks(RequestHandler): """ Lots of information about all the workers """ def get(self): with log_errors(): from ..diagnostics.scheduler import tasks self.write(tasks(self.server)) class Workers(RequestHandler): """ Lots of information about all the workers """ def get(self): with log_errors(): from ..diagnostics.scheduler import workers self.write(workers(self.server)) def HTTPScheduler(scheduler, **kwargs): application = MyApp(web.Application([ (r'/info.json', Info, {'server': scheduler}), (r'/resources.json', Resources, {'server': scheduler}), (r'/processing.json', Processing, {'server': scheduler}), (r'/stacks.json', Stacks, {'server': scheduler}), (r'/proxy/([\w.-]+):(\d+)/(.+)', Proxy), (r'/broadcast/(.+)', Broadcast, {'server': scheduler}), (r'/tasks.json', Tasks, {'server': scheduler}), (r'/workers.json', Workers, {'server': scheduler}), (r'/memory-load.json', MemoryLoad, {'server': scheduler}), (r'/memory-load-by-key.json', MemoryLoadByKey, {'server': scheduler}), ]), **kwargs) return application distributed-1.14.3/distributed/http/tests/000077500000000000000000000000001301211143400205535ustar00rootroot00000000000000distributed-1.14.3/distributed/http/tests/test_scheduler_http.py000066400000000000000000000133551301211143400252100ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import json import sys from tornado.ioloop import IOLoop from tornado import web, gen from tornado.httpclient import AsyncHTTPClient from tornado.httpserver import HTTPServer from distributed import Scheduler, Client from distributed.client import _wait from distributed.sizeof import getsizeof from distributed.utils_test import gen_cluster, gen_test, inc, div from distributed.http.scheduler import HTTPScheduler from distributed.http.worker import HTTPWorker @gen_cluster() def test_simple(s, a, b): server = HTTPScheduler(s) server.listen(0) client = AsyncHTTPClient() response = yield client.fetch('http://localhost:%d/info.json' % server.port) response = json.loads(response.body.decode()) assert response == {'ncores': s.ncores, 'status': s.status} server.stop() @gen_cluster() def test_processing_stacks(s, a, b): server = HTTPScheduler(s) server.listen(0) client = AsyncHTTPClient() s.processing[a.address][('foo-1', 1)] = 1 response = yield client.fetch('http://localhost:%d/processing.json' % server.port) response = json.loads(response.body.decode()) assert response == {a.address: ['foo'], b.address: []} s.stacks[a.address].append(('foo-1', 2)) response = yield client.fetch('http://localhost:%d/stacks.json' % server.port) response = json.loads(response.body.decode()) assert response == {a.address: ['foo'], b.address: []} server.stop() @gen_cluster() def test_proxy(s, a, b): server = HTTPScheduler(s) server.listen(0) worker = HTTPWorker(a) worker.listen(0) client = AsyncHTTPClient() c_response = yield client.fetch('http://localhost:%d/info.json' % worker.port) s_response = yield client.fetch('http://localhost:%d/proxy/%s:%d/info.json' % (server.port, a.ip, worker.port)) assert s_response.body.decode() == c_response.body.decode() server.stop() worker.stop() @gen_cluster() def test_broadcast(s, a, b): ss = HTTPScheduler(s) ss.listen(0) s.services['http'] = ss aa = HTTPWorker(a) aa.listen(0) a.services['http'] = aa a.service_ports['http'] = aa.port s.worker_info[a.address]['services']['http'] = aa.port bb = HTTPWorker(b) bb.listen(0) b.services['http'] = bb b.service_ports['http'] = bb.port s.worker_info[b.address]['services']['http'] = bb.port client = AsyncHTTPClient() a_response = yield client.fetch('http://localhost:%d/info.json' % aa.port) b_response = yield client.fetch('http://localhost:%d/info.json' % bb.port) s_response = yield client.fetch('http://localhost:%d/broadcast/info.json' % ss.port) assert (json.loads(s_response.body.decode()) == {a.address: json.loads(a_response.body.decode()), b.address: json.loads(b_response.body.decode())}) ss.stop() aa.stop() bb.stop() @gen_test() def test_services(): s = Scheduler(services={'http': HTTPScheduler}) assert isinstance(s.services['http'], HTTPServer) assert s.services['http'].port @gen_test() def test_services_with_port(): s = Scheduler(services={('http', 9999): HTTPScheduler}) assert isinstance(s.services['http'], HTTPServer) assert s.services['http'].port == 9999 @gen_cluster(client=True) def test_with_data(e, s, a, b): ss = HTTPScheduler(s) ss.listen(0) L = e.map(inc, [1, 2, 3]) L2 = yield e._scatter(['Hello', 'world!']) yield _wait(L) client = AsyncHTTPClient() response = yield client.fetch('http://localhost:%d/memory-load.json' % ss.port) out = json.loads(response.body.decode()) assert all(isinstance(v, int) for v in out.values()) assert set(out) == {a.address, b.address} assert sum(out.values()) == sum(map(getsizeof, [1, 2, 3, 'Hello', 'world!'])) response = yield client.fetch('http://localhost:%s/memory-load-by-key.json' % ss.port) out = json.loads(response.body.decode()) assert set(out) == {a.address, b.address} assert all(isinstance(v, dict) for v in out.values()) assert all(k in {'inc', 'data'} for d in out.values() for k in d) assert all(isinstance(v, int) for d in out.values() for v in d.values()) assert sum(v for d in out.values() for v in d.values()) == \ sum(map(getsizeof, [1, 2, 3, 'Hello', 'world!'])) ss.stop() @gen_cluster(client=True) def test_with_status(e, s, a, b): ss = HTTPScheduler(s) ss.listen(0) client = AsyncHTTPClient() response = yield client.fetch('http://localhost:%d/tasks.json' % ss.port) out = json.loads(response.body.decode()) assert out['total'] == 0 assert out['processing'] == 0 assert out['failed'] == 0 assert out['in-memory'] == 0 assert out['ready'] == 0 assert out['waiting'] == 0 L = e.map(div, range(10), range(10)) yield _wait(L) client = AsyncHTTPClient() response = yield client.fetch('http://localhost:%d/tasks.json' % ss.port) out = json.loads(response.body.decode()) assert out['failed'] == 1 assert out['in-memory'] == 9 assert out['ready'] == 0 assert out['total'] == 10 assert out['waiting'] == 0 ss.stop() @gen_cluster(client=True) def test_basic(e, s, a, b): ss = HTTPScheduler(s) ss.listen(0) client = AsyncHTTPClient() for name in ['tasks', 'workers']: response = yield client.fetch('http://localhost:%d/%s.json' % (ss.port, name)) data = json.loads(response.body.decode()) assert isinstance(data, dict) distributed-1.14.3/distributed/http/tests/test_worker_http.py000066400000000000000000000053351301211143400245420ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import json import tornado from tornado.ioloop import IOLoop from tornado import web from tornado.httpclient import AsyncHTTPClient from tornado.httpserver import HTTPServer from distributed.utils_test import gen_cluster, gen_test from distributed import Worker from distributed.http.worker import HTTPWorker from distributed.sizeof import sizeof @gen_cluster() def test_simple(s, a, b): server = HTTPWorker(a) server.listen(0) client = AsyncHTTPClient() response = yield client.fetch('http://localhost:%d/info.json' % server.port) response = json.loads(response.body.decode()) assert response['ncores'] == a.ncores assert response['status'] == a.status response = yield client.fetch('http://localhost:%d/resources.json' % server.port) response = json.loads(response.body.decode()) a.data['x'] = 1 try: import psutil assert 0 < response['memory_percent'] < 100 except ImportError: assert response == {} endpoints = ['/files.json', '/processing.json', '/nbytes.json', '/nbytes-summary.json'] for endpoint in endpoints: response = yield client.fetch(('http://localhost:%d' % server.port) + endpoint) response = json.loads(response.body.decode()) assert response server.stop() @gen_cluster() def test_services(s, a, b): c = Worker(s.ip, s.port, ncores=1, ip='127.0.0.1', services={'http': HTTPWorker}) yield c._start() assert isinstance(c.services['http'], HTTPServer) assert c.service_ports['http'] == c.services['http'].port assert s.worker_info[c.address]['services']['http'] == c.service_ports['http'] yield c._close() @gen_cluster() def test_services_port(s, a, b): c = Worker(s.ip, s.port, ncores=1, ip='127.0.0.1', services={('http', 9898): HTTPWorker}) yield c._start() assert isinstance(c.services['http'], HTTPServer) assert (c.service_ports['http'] == c.services['http'].port == s.worker_info[c.address]['services']['http'] == 9898) c.services['http'].stop() yield c._close() @gen_cluster() def test_nbytes(s, a, b): server = HTTPWorker(a) server.listen(0) client = AsyncHTTPClient() a.data['x-1'] = 1 a.data['x-2'] = 1 nbytes = yield client.fetch('http://localhost:%d/nbytes.json' % server.port) nbytes = json.loads(nbytes.body.decode()) summary = yield client.fetch('http://localhost:%d/nbytes-summary.json' % server.port) summary = json.loads(summary.body.decode()) assert nbytes == {'x-1': sizeof(1), 'x-2': sizeof(2)} assert summary == {'x': sizeof(1) * 2} distributed-1.14.3/distributed/http/worker.py000066400000000000000000000035751301211143400213060ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from collections import defaultdict import logging import os from toolz import keymap, valmap from tornado import web from .core import RequestHandler, MyApp, Resources from ..sizeof import sizeof from ..utils import key_split logger = logging.getLogger(__name__) class Info(RequestHandler): """Basic info about the worker """ def get(self): resp = {'ncores': self.server.ncores, 'nkeys': len(self.server.data), 'status': self.server.status} self.write(resp) class Processing(RequestHandler): """Basic info about the worker """ def get(self): try: resp = {'processing': [str(k) for k in self.server.active]} except Exception as e: resp = str(e) self.write(resp) class NBytes(RequestHandler): """Basic info about the worker """ def get(self): resp = keymap(str, valmap(sizeof, self.server.data)) self.write(resp) class NBytesSummary(RequestHandler): """Basic info about the worker """ def get(self): out = defaultdict(lambda: 0) for k, v in self.server.data.items(): out[key_split(k)] += sizeof(v) self.write(dict(out)) class LocalFiles(RequestHandler): """List the local spill directory""" def get(self): self.write({'files': os.listdir(self.server.local_dir)}) def HTTPWorker(worker, **kwargs): application = MyApp(web.Application([ (r'/info.json', Info, {'server': worker}), (r'/processing.json', Processing, {'server': worker}), (r'/resources.json', Resources, {'server': worker}), (r'/files.json', LocalFiles, {'server': worker}), (r'/nbytes.json', NBytes, {'server': worker}), (r'/nbytes-summary.json', NBytesSummary, {'server': worker}) ]), **kwargs) return application distributed-1.14.3/distributed/joblib.py000066400000000000000000000056311301211143400202520ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from distutils.version import LooseVersion import uuid from tornado import gen from .client import Client, _wait from .utils import ignoring # A user could have installed joblib, sklearn, both, or neither. Further, only # joblib >= 0.10.0 supports backends, so we also need to check for that. This # bit of logic is to ensure that we create and register the backend for all # viable installations of joblib. joblib = sk_joblib = None with ignoring(ImportError): import joblib if LooseVersion(joblib.__version__) < '0.10.2': joblib = None with ignoring(ImportError): import sklearn.externals.joblib as sk_joblib if LooseVersion(sk_joblib.__version__) < '0.10.2': sk_joblib = None if joblib: from joblib.parallel import (ParallelBackendBase, AutoBatchingMixin) elif sk_joblib: from sklearn.externals.joblib.parallel import ( ParallelBackendBase, AutoBatchingMixin) else: raise RuntimeError("Joblib backend requires either `joblib` >= '0.10.2' " " or `sklearn` > '0.17.1'. Please install or upgrade") class DaskDistributedBackend(ParallelBackendBase, AutoBatchingMixin): MIN_IDEAL_BATCH_DURATION = 0.2 MAX_IDEAL_BATCH_DURATION = 1.0 def __init__(self, scheduler_host='127.0.0.1:8786', loop=None): self.client = Client(scheduler_host, loop=loop) self.futures = set() def configure(self, n_jobs=1, parallel=None, **backend_args): return self.effective_n_jobs(n_jobs) def effective_n_jobs(self, n_jobs=1): return sum(self.client.ncores().values()) def apply_async(self, func, *args, **kwargs): callback = kwargs.pop('callback', None) kwargs['key'] = 'joblib-' + str(uuid.uuid4()) future = self.client.submit(func, *args, **kwargs) self.futures.add(future) @gen.coroutine def callback_wrapper(): result = yield _wait([future]) self.futures.remove(future) callback(result) # gets called in separate thread self.client.loop.add_callback(callback_wrapper) future.get = future.result # monkey patch to achieve AsyncResult API return future def abort_everything(self, ensure_ready=True): # Tell the client to cancel any task submitted via this instance # as joblib.Parallel will never access those results. self.client.cancel(self.futures) self.futures.clear() DistributedBackend = DaskDistributedBackend # Register the backend with any available versions of joblib if joblib: joblib.register_parallel_backend('distributed', DaskDistributedBackend) joblib.register_parallel_backend('dask.distributed', DaskDistributedBackend) if sk_joblib: sk_joblib.register_parallel_backend('distributed', DaskDistributedBackend) sk_joblib.register_parallel_backend('dask.distributed', DaskDistributedBackend) distributed-1.14.3/distributed/nanny.py000066400000000000000000000333401301211143400201320ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from datetime import datetime, timedelta import json import logging from multiprocessing import Process, Queue, queues import os import shutil import subprocess import sys import tempfile from time import time, sleep from tornado.ioloop import IOLoop from tornado.iostream import StreamClosedError from tornado import gen from .compatibility import JSONDecodeError from .core import Server, rpc, write, RPCClosed from .protocol import to_serialize from .utils import get_ip, ignoring, log_errors, tmpfile from .worker import _ncores, Worker, run, TOTAL_MEMORY nanny_environment = os.path.dirname(sys.executable) logger = logging.getLogger(__name__) class Nanny(Server): """ A process to manage worker processes The nanny spins up Worker processes, watches then, and kills or restarts them as necessary. """ def __init__(self, scheduler_ip, scheduler_port, ip=None, worker_port=0, ncores=None, loop=None, local_dir=None, services=None, name=None, memory_limit=TOTAL_MEMORY, environment=nanny_environment, quiet=False, **kwargs): self.ip = ip or get_ip() self.worker_port = None self._given_worker_port = worker_port self.ncores = ncores or _ncores if not local_dir: local_dir = tempfile.mkdtemp(prefix='nanny-') self._should_cleanup_local_dir = True def _cleanup_local_dir(): if os.path.exists(local_dir): shutil.rmtree(local_dir) atexit.register(_cleanup_local_dir) else: self._should_cleanup_local_dir = False self.local_dir = local_dir self.worker_dir = '' self.status = None self.process = None self.loop = loop or IOLoop.current() self.scheduler = rpc(ip=scheduler_ip, port=scheduler_port) self.services = services self.name = name self.memory_limit = memory_limit self.environment = environment self.quiet = quiet handlers = {'instantiate': self.instantiate, 'kill': self._kill, 'terminate': self._close, 'monitor_resources': self.monitor_resources, 'run': self.run} super(Nanny, self).__init__(handlers, io_loop=self.loop, **kwargs) def __str__(self): return "" % (self.worker_address, self.ncores) __repr__ = __str__ @gen.coroutine def _start(self, port=0): """ Start nanny, start local process, start watching """ self.listen(port) logger.info(' Start Nanny at: %20s:%d', self.ip, self.port) yield self.instantiate() self.loop.add_callback(self._watch) assert self.worker_port self.status = 'running' def start(self, port=0): self.loop.add_callback(self._start, port) @gen.coroutine def _kill(self, stream=None, timeout=5): """ Kill the local worker process Blocks until both the process is down and the scheduler is properly informed """ while not self.worker_port: yield gen.sleep(0.1) if self.process is not None: process, self.process = self.process, None # avoid race with _watch try: # Ask worker to close with rpc(ip='127.0.0.1', port=self.worker_port) as worker: result = yield gen.with_timeout( timedelta(seconds=min(1, timeout)), worker.terminate(report=False), io_loop=self.loop) except gen.TimeoutError: logger.info("Worker non-responsive. Terminating.") except StreamClosedError: pass except Exception as e: logger.exception(e) try: # Tell scheduler that worker is gone result = yield gen.with_timeout(timedelta(seconds=timeout), self.scheduler.unregister(address=self.worker_address), io_loop=self.loop) if result not in ('OK', 'already-removed'): logger.critical("Unable to unregister with scheduler %s. " "Nanny: %s, Worker: %s", result, self.address_tuple, self.worker_address) else: logger.info("Unregister worker %s:%d from scheduler", self.ip, self.worker_port) except gen.TimeoutError: logger.info("Nanny %s:%d failed to unregister worker %s:%d", self.ip, self.port, self.ip, self.worker_port, exc_info=True) except (StreamClosedError, RPCClosed): pass except Exception as e: logger.exception(e) self.process = process # re-instantiate self.process if self.process: with ignoring(OSError): self.process.terminate() if self.process in processes_to_close: processes_to_close.remove(self.process) start = time() while isalive(self.process) and time() < start + timeout: sleep(0.01) self.process = None self.cleanup() logger.info("Nanny %s:%d kills worker process %s:%d", self.ip, self.port, self.ip, self.worker_port) raise gen.Return('OK') @gen.coroutine def instantiate(self, stream=None, environment=None): """ Start a local worker process Blocks until the process is up and the scheduler is properly informed """ if environment: if not os.path.isabs(environment): environment = os.path.join(self.local_dir, environment) self.environment = environment with log_errors(): if self.process and isalive(self.process): raise ValueError("Existing process still alive. Please kill first") if self.environment != nanny_environment: with tmpfile() as fn: self.process = run_worker_subprocess(self.environment, self.ip, self.scheduler.ip, self.scheduler.port, self.ncores, self.port, self._given_worker_port, self.name, self.memory_limit, self.loop, fn, self.quiet) while not os.path.exists(fn): yield gen.sleep(0.01) while True: try: with open(fn) as f: msg = json.load(f) self.worker_port = msg['port'] self.worker_dir = msg['local_directory'] break except JSONDecodeError: yield gen.sleep(0.01) else: q = Queue() self.process = Process(target=run_worker_fork, args=(q, self.ip, self.scheduler.ip, self.scheduler.port, self.ncores, self.port, self._given_worker_port, self.local_dir, self.services, self.name, self.memory_limit)) self.process.daemon = True self.process.start() while True: try: msg = q.get_nowait() if isinstance(msg, Exception): raise msg self.worker_port = msg['port'] self.worker_dir = msg['dir'] assert self.worker_port break except queues.Empty: yield gen.sleep(0.1) logger.info("Nanny %s:%d starts worker process %s:%d", self.ip, self.port, self.ip, self.worker_port) raise gen.Return('OK') def run(self, *args, **kwargs): return run(self, *args, **kwargs) def cleanup(self): if self.worker_dir and os.path.exists(self.worker_dir): shutil.rmtree(self.worker_dir) self.worker_dir = None if self.process: with ignoring(OSError): self.process.terminate() def __del__(self): if self._should_cleanup_local_dir and os.path.exists(self.local_dir): shutil.rmtree(self.local_dir) self.cleanup() @gen.coroutine def _watch(self, wait_seconds=0.10): """ Watch the local process, if it dies then spin up a new one """ while True: if closing[0] or self.status == 'closed': yield self._close() break elif self.process and not isalive(self.process): logger.warn("Discovered failed worker. Restarting. Status: %s", self.status) self.cleanup() yield self.scheduler.unregister(address=self.worker_address) yield self.instantiate() else: yield gen.sleep(wait_seconds) @gen.coroutine def _close(self, stream=None, timeout=5, report=None): """ Close the nanny process, stop listening """ if self.status == 'closed': raise gen.Return('OK') logger.info("Closing Nanny at %s:%d", self.ip, self.port) self.status = 'closed' yield self._kill(timeout=timeout) self.rpc.close() self.scheduler.close_rpc() self.stop() raise gen.Return('OK') @property def address(self): return '%s:%d' % (self.ip, self.port) @property def address_tuple(self): return (self.ip, self.port) @property def worker_address_tuple(self): return (self.ip, self.worker_port) @property def worker_address(self): return '%s:%d' % (self.ip, self.worker_port) def resource_collect(self): try: import psutil except ImportError: return {} p = psutil.Process(self.process.pid) return {'timestamp': datetime.now().isoformat(), 'cpu_percent': psutil.cpu_percent(), 'status': p.status(), 'memory_percent': p.memory_percent(), 'memory_info_ex': p.memory_info_ex()._asdict(), 'disk_io_counters': psutil.disk_io_counters()._asdict(), 'net_io_counters': psutil.net_io_counters()._asdict()} @gen.coroutine def monitor_resources(self, stream, interval=1): while not stream.closed(): if self.process: yield write(stream, self.resource_collect()) yield gen.sleep(interval) def run_worker_subprocess(environment, ip, scheduler_ip, scheduler_port, ncores, nanny_port, worker_port, name, memory_limit, io_loop, fn, quiet): if environment.endswith('python'): environment = os.path.dirname(environment) if os.path.exists(os.path.join(environment, 'bin')): environment = os.path.join(environment, 'bin') executable = os.path.join(environment, 'python') args = ['-m', 'distributed.cli.dask_worker'] args.extend(['%s:%d' %(scheduler_ip, scheduler_port), '--no-nanny', '--host', ip, '--worker-port', worker_port, '--nanny-port', nanny_port, '--nthreads', ncores, '--nprocs', 1, '--temp-filename', fn]) if name: args.extend(['--name', name]) if memory_limit: args.extend(['--memory-limit', memory_limit]) proc = subprocess.Popen([executable] + list(map(str, args)), stderr=subprocess.PIPE if quiet else None) processes_to_close.append(proc) return proc def run_worker_fork(q, ip, scheduler_ip, scheduler_port, ncores, nanny_port, worker_port, local_dir, services, name, memory_limit): """ Function run by the Nanny when creating the worker """ from distributed import Worker # pragma: no cover from tornado.ioloop import IOLoop # pragma: no cover IOLoop.clear_instance() # pragma: no cover loop = IOLoop() # pragma: no cover loop.make_current() # pragma: no cover worker = Worker(scheduler_ip, scheduler_port, ncores=ncores, ip=ip, service_ports={'nanny': nanny_port}, local_dir=local_dir, services=services, name=name, memory_limit=memory_limit, loop=loop) # pragma: no cover @gen.coroutine # pragma: no cover def start(): try: # pragma: no cover yield worker._start(worker_port) # pragma: no cover except Exception as e: # pragma: no cover logger.exception(e) # pragma: no cover q.put(e) # pragma: no cover else: assert worker.port # pragma: no cover q.put({'port': worker.port, 'dir': worker.local_dir}) # pragma: no cover loop.add_callback(start) # pragma: no cover try: loop.start() # pragma: no cover finally: loop.stop() loop.close(all_fds=True) def isalive(proc): if isinstance(proc, subprocess.Popen): return proc.poll() is None else: return proc.is_alive() import atexit closing = [False] processes_to_close = [] def _closing(): for proc in processes_to_close: try: proc.terminate() except OSError: pass closing[0] = True atexit.register(_closing) distributed-1.14.3/distributed/protocol/000077500000000000000000000000001301211143400202735ustar00rootroot00000000000000distributed-1.14.3/distributed/protocol/__init__.py000066400000000000000000000007611301211143400224100ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from .compression import compressions, default_compression from .core import dumps, loads, maybe_compress, decompress, msgpack from .serialize import (serialize, deserialize, Serialize, Serialized, to_serialize, register_serialization) from ..utils import ignoring with ignoring(ImportError): from . import numpy with ignoring(ImportError): from . import h5py with ignoring(ImportError): from . import netcdf4 distributed-1.14.3/distributed/protocol/compression.py000066400000000000000000000064431301211143400232150ustar00rootroot00000000000000""" Record known compressors Includes utilities for determining whether or not to compress """ from __future__ import print_function, division, absolute_import import logging import random from toolz import identity, partial from ..config import config from ..utils import ignoring, ensure_bytes compressions = {None: {'compress': identity, 'decompress': identity}} default_compression = None logger = logging.getLogger(__file__) with ignoring(ImportError): import zlib compressions['zlib'] = {'compress': zlib.compress, 'decompress': zlib.decompress} with ignoring(ImportError): import snappy compressions['snappy'] = {'compress': snappy.compress, 'decompress': snappy.decompress} default_compression = 'snappy' with ignoring(ImportError): import lz4 compressions['lz4'] = {'compress': lz4.LZ4_compress, 'decompress': lz4.LZ4_uncompress} default_compression = 'lz4' with ignoring(ImportError): import blosc compressions['blosc'] = {'compress': partial(blosc.compress, clevel=5, cname='lz4'), 'decompress': blosc.decompress} default = config.get('compression', 'auto') if default != 'auto': if default in compressions: default_compression = default else: raise ValueError("Default compression '%s' not found.\n" "Choices include auto, %s" % ( default, ', '.join(sorted(map(str, compressions))))) def byte_sample(b, size, n): """ Sample a bytestring from many locations Parameters ---------- b: bytes or memoryview size: int size of each sample to collect n: int number of samples to collect """ starts = [random.randint(0, len(b) - size) for j in range(n)] ends = [] for i, start in enumerate(starts[:-1]): ends.append(min(start + size, starts[i + 1])) ends.append(starts[-1] + size) parts = [b[start:end] for start, end in zip(starts, ends)] return b''.join(map(ensure_bytes, parts)) def maybe_compress(payload, compression=default_compression, min_size=1e4, sample_size=1e4, nsamples=5): """ Maybe compress payload 1. We don't compress small messages 2. We sample the payload in a few spots, compress that, and if it doesn't do any good we return the original 3. We then compress the full original, it it doesn't compress well then we return the original 4. We return the compressed result """ if not compression: return None, payload if len(payload) < min_size: return None, payload if len(payload) > 2**31: return None, payload min_size = int(min_size) sample_size = int(sample_size) compress = compressions[compression]['compress'] # Compress a sample, return original if not very compressed sample = byte_sample(payload, sample_size, nsamples) if len(compress(sample)) > 0.9 * len(sample): # not very compressible return None, payload compressed = compress(ensure_bytes(payload)) if len(compressed) > 0.9 * len(payload): # not very compressible return None, payload else: return compression, compressed distributed-1.14.3/distributed/protocol/core.py000066400000000000000000000121211301211143400215720ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from functools import partial import logging try: import pandas.msgpack as msgpack except ImportError: import msgpack from toolz import get_in from .compression import compressions, maybe_compress from .serialize import (serialize, deserialize, Serialize, Serialized, to_serialize, extract_serialize) from .utils import frame_split_size, merge_frames from ..utils import ignoring _deserialize = deserialize logger = logging.getLogger(__file__) def dumps(msg): """ Transform Python message to bytestream suitable for communication """ try: data = {} # Only lists and dicts can contain serialized values if isinstance(msg, (list, dict)): msg, data, bytestrings = extract_serialize(msg) small_header, small_payload = dumps_msgpack(msg) if not data: # fast path without serialized data return small_header, small_payload pre = {key: (value.header, value.frames) for key, value in data.items() if type(value) is Serialized} data = {key: serialize(value.data) for key, value in data.items() if type(value) is Serialize} header = {'headers': {}, 'keys': [], 'bytestrings': list(bytestrings)} out_frames = [] for key, (head, frames) in data.items(): if 'lengths' not in head: head['lengths'] = list(map(len, frames)) if 'compression' not in head: frames = frame_split_size(frames) if frames: compression, frames = zip(*map(maybe_compress, frames)) else: compression = [] head['compression'] = compression head['count'] = len(frames) header['headers'][key] = head header['keys'].append(key) out_frames.extend(frames) for key, (head, frames) in pre.items(): if 'lengths' not in head: head['lengths'] = list(map(len, frames)) head['count'] = len(frames) header['headers'][key] = head header['keys'].append(key) out_frames.extend(frames) out_frames = [bytes(f) for f in out_frames] return [small_header, small_payload, msgpack.dumps(header, use_bin_type=True)] + out_frames except Exception as e: logger.critical("Failed to Serialize", exc_info=True) raise def loads(frames, deserialize=True): """ Transform bytestream back into Python value """ frames = frames[::-1] # reverse order to improve pop efficiency if not isinstance(frames, list): frames = list(frames) try: small_header = frames.pop() small_payload = frames.pop() msg = loads_msgpack(small_header, small_payload) if not frames: return msg header = frames.pop() header = msgpack.loads(header, encoding='utf8', use_list=False) keys = header['keys'] headers = header['headers'] bytestrings = set(header['bytestrings']) for key in keys: head = headers[key] lengths = head['lengths'] count = head['count'] fs = frames[-count::][::-1] del frames[-count:] if deserialize or key in bytestrings: fs = decompress(head, fs) fs = merge_frames(head, fs) value = _deserialize(head, fs) else: value = Serialized(head, fs) get_in(key[:-1], msg)[key[-1]] = value return msg except Exception as e: logger.critical("Failed to deerialize", exc_info=True) raise def dumps_msgpack(msg): """ Dump msg into header and payload, both bytestrings All of the message must be msgpack encodable See Also: loads_msgpack """ header = {} payload = msgpack.dumps(msg, use_bin_type=True) fmt, payload = maybe_compress(payload) if fmt: header['compression'] = fmt if header: header_bytes = msgpack.dumps(header, use_bin_type=True) else: header_bytes = b'' return [header_bytes, payload] def loads_msgpack(header, payload): """ Read msgpack header and payload back to Python object See Also: dumps_msgpack """ if header: header = msgpack.loads(header, encoding='utf8') else: header = {} if header.get('compression'): try: decompress = compressions[header['compression']]['decompress'] payload = decompress(payload) except KeyError: raise ValueError("Data is compressed as %s but we don't have this" " installed" % str(header['compression'])) return msgpack.loads(payload, encoding='utf8') def decompress(header, frames): """ Decompress frames according to information in the header """ return [compressions[c]['decompress'](frame) for c, frame in zip(header['compression'], frames)] distributed-1.14.3/distributed/protocol/h5py.py000066400000000000000000000020511301211143400215300ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from .serialize import register_serialization from ..utils import log_errors, ensure_bytes def serialize_h5py_file(f): if f.mode != 'r': raise ValueError("Can only serialize read-only h5py files") return {'filename': f.filename}, [] def deserialize_h5py_file(header, frames): import h5py return h5py.File(header['filename'], mode='r') register_serialization('h5py._hl.files.File', serialize_h5py_file, deserialize_h5py_file) def serialize_h5py_dataset(x): header, _ = serialize_h5py_file(x.file) header['name'] = x.name return header, [] def deserialize_h5py_dataset(header, frames): import h5py file = deserialize_h5py_file(header, frames) return file[header['name']] register_serialization('h5py._hl.dataset.Dataset', serialize_h5py_dataset, deserialize_h5py_dataset) register_serialization('h5py._hl.group.Group', serialize_h5py_dataset, deserialize_h5py_dataset) distributed-1.14.3/distributed/protocol/netcdf4.py000066400000000000000000000031401301211143400221720ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from .serialize import register_serialization, serialize, deserialize from ..utils import log_errors, ensure_bytes try: import netCDF4 HAS_NETCDF4 = True except ImportError: HAS_NETCDF4 = False def serialize_netcdf4_dataset(ds): # assume mode is read-only return {'filename': ds.filepath()}, [] def deserialize_netcdf4_dataset(header, frames): import netCDF4 return netCDF4.Dataset(header['filename'], mode='r') if HAS_NETCDF4: register_serialization(netCDF4.Dataset, serialize_netcdf4_dataset, deserialize_netcdf4_dataset) def serialize_netcdf4_variable(x): header, _ = serialize(x.group()) header['parent-type'] = header['type'] header['name'] = x.name return header, [] def deserialize_netcdf4_variable(header, frames): header['type'] = header['parent-type'] parent = deserialize(header, frames) return parent.variables[header['name']] if HAS_NETCDF4: register_serialization(netCDF4.Variable, serialize_netcdf4_variable, deserialize_netcdf4_variable) def serialize_netcdf4_group(g): parent = g while parent.parent: parent = parent.parent header, _ = serialize_netcdf4_dataset(parent) header['path'] = g.path return header, [] def deserialize_netcdf4_group(header, frames): file = deserialize_netcdf4_dataset(header, frames) return file[header['path']] if HAS_NETCDF4: register_serialization(netCDF4.Group, serialize_netcdf4_group, deserialize_netcdf4_group) distributed-1.14.3/distributed/protocol/numpy.py000066400000000000000000000037511301211143400220230ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import sys import numpy as np try: import blosc n = blosc.set_nthreads(2) except ImportError: blosc = False from .utils import frame_split_size from .serialize import register_serialization from . import pickle from ..utils import log_errors, ensure_bytes def itemsize(dt): """ Itemsize of dtype Try to return the itemsize of the base element, return 8 as a fallback """ result = dt.base.itemsize if result > 255: result = 8 return result def serialize_numpy_ndarray(x): if x.dtype.hasobject: header = {'pickle': True} frames = [pickle.dumps(x)] return header, frames size = itemsize(x.dtype) if x.dtype.kind == 'V': dt = x.dtype.descr else: dt = x.dtype.str x = np.ascontiguousarray(x) # np.frombuffer requires this header = {'dtype': dt, 'strides': x.strides, 'shape': x.shape} data = x.view('u1').data if blosc: frames = frame_split_size([data]) if sys.version_info.major == 2: frames = [ensure_bytes(frame) for frame in frames] frames = [blosc.compress(frame, typesize=size, cname='lz4', clevel=5) for frame in frames] header['compression'] = ['blosc'] * len(frames) else: frames = [data] header['lengths'] = [x.nbytes] return header, frames def deserialize_numpy_ndarray(header, frames): with log_errors(): assert len(frames) == 1 if header.get('pickle'): return pickle.loads(frames[0]) dt = header['dtype'] if isinstance(dt, tuple): dt = list(dt) dt = np.dtype(dt) buffer = frames[0] x = np.frombuffer(buffer, dt) x = np.lib.stride_tricks.as_strided(x, header['shape'], header['strides']) return x register_serialization(np.ndarray, serialize_numpy_ndarray, deserialize_numpy_ndarray) distributed-1.14.3/distributed/protocol/pickle.py000066400000000000000000000027461301211143400221250ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import logging import pickle import cloudpickle from ..utils import ignoring logger = logging.getLogger(__file__) pickle_types = [str, bytes] with ignoring(ImportError): import numpy as np pickle_types.append(np.ndarray) with ignoring(ImportError): import pandas as pd pickle_types.append(pd.core.generic.NDFrame) pickle_types = tuple(pickle_types) def dumps(x): """ Manage between cloudpickle and pickle 1. Try pickle 2. If it is short then check if it contains __main__ 3. If it is long, then first check type, then check __main__ """ try: result = pickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL) if len(result) < 1000: if b'__main__' in result: return cloudpickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL) else: return result else: if isinstance(x, pickle_types) or b'__main__' not in result: return result else: return cloudpickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL) except: try: return cloudpickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL) except Exception: logger.info("Failed to serialize %s", x, exc_info=True) raise def loads(x): try: return pickle.loads(x) except Exception: logger.info("Failed to deserialize %s", x[:10000], exc_info=True) raise distributed-1.14.3/distributed/protocol/serialize.py000066400000000000000000000141461301211143400226420ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from functools import partial from dask.base import normalize_token from toolz import valmap, get_in from . import pickle serializers = {} deserializers = {None: lambda header, frames: pickle.loads(b''.join(frames))} def register_serialization(cls, serialize, deserialize): """ Register a new class for custom serialization Parameters ---------- cls: type serialize: function deserialize: function Examples -------- >>> class Human(object): ... def __init__(self, name): ... self.name = name >>> def serialize(human): ... header = {} ... frames = [human.name.encode()] ... return header, frames >>> def deserialize(header, frames): ... return Human(frames[0].decode()) >>> register_serialization(Human, serialize, deserialize) >>> serialize(Human('Alice')) ({}, [b'Alice']) See Also -------- serialize deserialize """ if isinstance(cls, type): name = typename(cls) elif isinstance(cls, str): name = cls serializers[name] = serialize deserializers[name] = deserialize def typename(typ): """ Return name of type Examples -------- >>> from distributed import Scheduler >>> typename(Scheduler) 'distributed.scheduler.Scheduler' """ return typ.__module__ + '.' + typ.__name__ def serialize(x): r""" Convert object to a header and list of bytestrings This takes in an arbitrary Python object and returns a msgpack serializable header and a list of bytes or memoryview objects. By default this uses pickle/cloudpickle but can use special functions if they have been pre-registered. Examples -------- >>> serialize(1) ({}, [b'\x80\x04\x95\x03\x00\x00\x00\x00\x00\x00\x00K\x01.']) >>> serialize(b'123') # some special types get custom treatment ({'type': 'builtins.bytes'}, [b'123']) >>> deserialize(*serialize(1)) 1 Returns ------- header: dictionary containing any msgpack-serializable metadata frames: list of bytes or memoryviews, commonly of length one See Also -------- deserialize: Convert header and frames back to object to_serialize: Mark that data in a message should be serialized register_serialization: Register custom serialization functions """ if isinstance(x, Serialized): return x.header, x.frames name = typename(type(x)) if name in serializers: header, frames = serializers[name](x) header['type'] = name else: header, frames = {}, [pickle.dumps(x)] return header, frames def deserialize(header, frames): """ Convert serialized header and list of bytestrings back to a Python object Parameters ---------- header: dict frames: list of bytes See Also -------- serialize """ f = deserializers[header.get('type')] return f(header, frames) class Serialize(object): """ Mark an object that should be serialized Example ------- >>> msg = {'op': 'update', 'data': to_serialize(123)} >>> msg # doctest: +SKIP {'op': 'update', 'data': } See also -------- distributed.protocol.dumps """ def __init__(self, data): self.data = data def __str__(self): return "" % str(self.data) __repr__ = __str__ to_serialize = Serialize class Serialized(object): """ An object that is already serialized into header and frames Normal serialization operations pass these objects through. This is typically used within the scheduler which accepts messages that contain data without actually unpacking that data. """ def __init__(self, header, frames): self.header = header self.frames = frames def deserialize(self): from .core import decompress frames = decompress(self.header, self.frames) return deserialize(self.header, frames) def container_copy(c): typ = type(c) if typ is list: return list(map(container_copy, c)) if typ is dict: return valmap(container_copy, c) return c def extract_serialize(x): """ Pull out Serialize objects from message This also remove large bytestrings from the message into a second dictionary. Examples -------- >>> from distributed.protocol import to_serialize >>> msg = {'op': 'update', 'data': to_serialize(123)} >>> extract_serialize(msg) ({'op': 'update'}, {('data',): }, set()) """ ser = {} _extract_serialize(x, ser) if ser: x = container_copy(x) for path in ser: t = get_in(path[:-1], x) if isinstance(t, dict): del t[path[-1]] else: t[path[-1]] = None bytestrings = set() for k, v in ser.items(): if type(v) is bytes: ser[k] = to_serialize(v) bytestrings.add(k) return x, ser, bytestrings def _extract_serialize(x, ser, path=()): if type(x) is dict: for k, v in x.items(): typ = type(v) if typ is list or typ is dict: _extract_serialize(v, ser, path + (k,)) elif (typ is Serialize or typ is Serialized or typ is bytes and len(v) > 2**16): ser[path + (k,)] = v elif type(x) is list: for k, v in enumerate(x): typ = type(v) if typ is list or typ is dict: _extract_serialize(v, ser, path + (k,)) elif (typ is Serialize or typ is Serialized or typ is bytes and len(v) > 2**16): ser[path + (k,)] = v @partial(normalize_token.register, Serialized) def normalize_Serialized(o): return [o.header] + o.frames # for dask.base.tokenize # Teach serialize how to handle bytestrings def serialize_bytes(obj): header = {} # no special metadata frames = [obj] return header, frames def deserialize_bytes(header, frames): return frames[0] register_serialization(bytes, serialize_bytes, deserialize_bytes) distributed-1.14.3/distributed/protocol/tests/000077500000000000000000000000001301211143400214355ustar00rootroot00000000000000distributed-1.14.3/distributed/protocol/tests/test_h5py.py000066400000000000000000000050421301211143400237340ustar00rootroot00000000000000import pytest h5py = pytest.importorskip('h5py') from distributed.protocol import serialize, deserialize, dumps, loads from distributed.utils import tmpfile def test_serialize_deserialize_file(): with tmpfile() as fn: with h5py.File(fn, mode='a') as f: f.create_dataset('/x', shape=(2, 2), dtype='i4') with h5py.File(fn, mode='r') as f: g = deserialize(*serialize(f)) assert f.filename == g.filename assert isinstance(g, h5py.File) assert f.mode == g.mode assert g['x'].shape == (2, 2) def test_serialize_deserialize_group(): with tmpfile() as fn: with h5py.File(fn, mode='a') as f: f.create_dataset('/group1/group2/x', shape=(2, 2), dtype='i4') with h5py.File(fn, mode='r') as f: group = f['/group1/group2'] group2 = deserialize(*serialize(group)) assert isinstance(group2, h5py.Group) assert group.file.filename == group2.file.filename assert group2['x'].shape == (2, 2) def test_serialize_deserialize_dataset(): with tmpfile() as fn: with h5py.File(fn, mode='a') as f: x = f.create_dataset('/group1/group2/x', shape=(2, 2), dtype='i4') with h5py.File(fn, mode='r') as f: x = f['group1/group2/x'] y = deserialize(*serialize(x)) assert isinstance(y, h5py.Dataset) assert x.name == y.name assert x.file.filename == y.file.filename assert (x[:] == y[:]).all() def test_raise_error_on_serialize_write_permissions(): with tmpfile() as fn: with h5py.File(fn, mode='a') as f: x = f.create_dataset('/x', shape=(2, 2), dtype='i4') f.flush() with pytest.raises(ValueError): serialize(x) with pytest.raises(ValueError): serialize(f) from distributed.utils_test import gen_cluster from distributed.client import _wait from tornado import gen import dask.array as da @gen_cluster(client=True) def test_h5py_serialize(c, s, a, b): with tmpfile() as fn: with h5py.File(fn, mode='a') as f: x = f.create_dataset('/group/x', shape=(4,), dtype='i4', chunks=(2,)) x[:] = [1, 2, 3, 4] f.flush() with h5py.File(fn, mode='r') as f: dset = f['/group/x'] x = da.from_array(dset, chunks=dset.chunks) y = c.compute(x) y = yield y._result() assert (y[:] == dset[:]).all() distributed-1.14.3/distributed/protocol/tests/test_netcdf4.py000066400000000000000000000056211301211143400244010ustar00rootroot00000000000000import pytest netCDF4 = pytest.importorskip('netCDF4') np = pytest.importorskip('numpy') from distributed.protocol import serialize, deserialize, dumps, loads from distributed.utils import tmpfile import distributed.protocol.netcdf4 def create_test_dataset(fn): with netCDF4.Dataset(fn, mode='w') as ds: ds.createDimension('x', 3) v = ds.createVariable('x', np.int32, ('x',)) v[:] = np.arange(3) g = ds.createGroup('group') g2 = ds.createGroup('group/group1') v2 = ds.createVariable('group/y', np.int32, ('x',)) v2[:] = np.arange(3) + 1 v3 = ds.createVariable('group/group1/z', np.int32, ('x',)) v3[:] = np.arange(3) + 2 def test_serialize_deserialize_dataset(): with tmpfile() as fn: create_test_dataset(fn) with netCDF4.Dataset(fn, mode='r') as f: g = deserialize(*serialize(f)) assert f.filepath() == g.filepath() assert isinstance(g, netCDF4.Dataset) assert g.variables['x'].dimensions == ('x',) assert g.variables['x'].dtype == np.int32 assert (g.variables['x'][:] == np.arange(3)).all() def test_serialize_deserialize_variable(): with tmpfile() as fn: create_test_dataset(fn) with netCDF4.Dataset(fn, mode='r') as f: x = f.variables['x'] y = deserialize(*serialize(x)) assert isinstance(y, netCDF4.Variable) assert y.dimensions == ('x',) assert (x.dtype == y.dtype) assert (x[:] == y[:]).all() def test_serialize_deserialize_group(): with tmpfile() as fn: create_test_dataset(fn) with netCDF4.Dataset(fn, mode='r') as f: for path in ['group', 'group/group1']: g = f[path] h = deserialize(*serialize(g)) assert isinstance(h, netCDF4.Group) assert h.name == g.name assert list(g.groups) == list(h.groups) assert list(g.variables) == list(h.variables) vars = [f.variables['x'], f['group'].variables['y'], f['group/group1'].variables['z']] for x in vars: y = deserialize(*serialize(x)) assert isinstance(y, netCDF4.Variable) assert y.dimensions == ('x',) assert (x.dtype == y.dtype) assert (x[:] == y[:]).all() from distributed.utils_test import gen_cluster from distributed.client import _wait from tornado import gen import dask.array as da @gen_cluster(client=True) def test_netcdf4_serialize(c, s, a, b): with tmpfile() as fn: create_test_dataset(fn) with netCDF4.Dataset(fn, mode='r') as f: dset = f.variables['x'] x = da.from_array(dset, chunks=2) y = c.compute(x) y = yield y._result() assert (y[:] == dset[:]).all() distributed-1.14.3/distributed/protocol/tests/test_numpy.py000066400000000000000000000074301301211143400242220ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import numpy as np import pytest from distributed.protocol import (serialize, deserialize, decompress, dumps, loads, to_serialize) from distributed.protocol.utils import BIG_BYTES_SHARD_SIZE from distributed.utils import tmpfile from distributed.utils_test import slow from distributed.protocol.numpy import itemsize from distributed.protocol.compression import maybe_compress import distributed.protocol.numpy def test_serialize(): x = np.ones((5, 5)) header, frames = serialize(x) assert header['type'] assert len(frames) == 1 if 'compression' in header: frames = decompress(header, frames) result = deserialize(header, frames) assert (result == x).all() @pytest.mark.parametrize('x', [np.ones(5), np.array(5), np.asfortranarray(np.random.random((5, 5))), np.random.random(5).astype('f4'), np.random.random(5).astype('>i8'), np.random.random(5).astype(' 2 result = loads(frames) assert result == {'x': 1, 'data': 123} result2 = loads(frames, deserialize=False) assert result2['x'] == 1 assert isinstance(result2['data'], Serialized) assert any(a is b for a in result2['data'].frames for b in frames) frames2 = dumps(result2) assert all(map(eq_frames, frames, frames2)) result3 = loads(frames2) assert result == result3 distributed-1.14.3/distributed/protocol/tests/test_protocol_utils.py000066400000000000000000000006421301211143400261310ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from distributed.protocol.utils import merge_frames from distributed.utils import ensure_bytes def test_merge_frames(): result = merge_frames({'lengths': [3, 4]}, [b'12', b'34', b'567']) expected = [b'123', b'4567'] assert list(map(ensure_bytes, result)) == expected b = b'123' assert merge_frames({'lengths': [3]}, [b])[0] is b distributed-1.14.3/distributed/protocol/tests/test_serialize.py000066400000000000000000000052671301211143400250470ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import pickle import numpy as np import pytest from toolz import identity from distributed.protocol import (register_serialization, serialize, deserialize, Serialize, Serialized, to_serialize) from distributed.protocol import decompress class MyObj(object): def __init__(self, data): self.data = data def __getstate__(self): raise Exception('Not picklable') def serialize_myobj(x): return {}, [pickle.dumps(x.data)] def deserialize_myobj(header, frames): return MyObj(pickle.loads(frames[0])) register_serialization(MyObj, serialize_myobj, deserialize_myobj) def test_dumps_serialize(): for x in [123, [1, 2, 3]]: header, frames = serialize(x) assert not header assert len(frames) == 1 result = deserialize(header, frames) assert result == x x = MyObj(123) header, frames = serialize(x) assert header['type'] assert len(frames) == 1 result = deserialize(header, frames) assert result.data == x.data def test_serialize_bytes(): b = b'123' header, frames = serialize(b) assert frames[0] is b def test_Serialize(): s = Serialize(123) assert '123' in str(s) assert s.data == 123 s = Serialize((1, 2)) assert str(s) from distributed.utils_test import gen_cluster from dask import delayed @gen_cluster(client=True) def test_object_in_graph(c, s, a, b): o = MyObj(123) v = delayed(o) v2 = delayed(identity)(v) future = c.compute(v2) result = yield future._result() assert isinstance(result, MyObj) assert result.data == 123 @gen_cluster(client=True) def test_scatter(c, s, a, b): o = MyObj(123) [future] = yield c._scatter([o]) yield c._replicate(o) o2 = yield c._gather(future) assert isinstance(o2, MyObj) assert o2.data == 123 @gen_cluster(client=True) def test_inter_worker_comms(c, s, a, b): o = MyObj(123) [future] = yield c._scatter([o], workers=a.address) future2 = c.submit(identity, future, workers=b.address) o2 = yield c._gather(future2) assert isinstance(o2, MyObj) assert o2.data == 123 class Empty(object): def __getstate__(self): raise Exception('Not picklable') def serialize_empty(x): return {}, [] def deserialize_empty(header, frames): return Empty() register_serialization(Empty, serialize_empty, deserialize_empty) def test_empty(): e = Empty() e2 = deserialize(*serialize(e)) assert isinstance(e2, Empty) def test_empty_loads(): from distributed.protocol import loads, dumps e = Empty() e2 = loads(dumps([to_serialize(e)])) assert isinstance(e2[0], Empty) distributed-1.14.3/distributed/protocol/utils.py000066400000000000000000000034111301211143400220040ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from ..utils import ensure_bytes BIG_BYTES_SHARD_SIZE = 2**28 def frame_split_size(frames, n=BIG_BYTES_SHARD_SIZE): """ Split a list of frames into a list of frames of maximum size This helps us to avoid passing around very large bytestrings. Examples -------- >>> frame_split_size([b'12345', b'678'], n=3) # doctest: +SKIP [b'123', b'45', b'678'] """ if not frames: return frames if max(map(len, frames)) <= n: return frames out = [] for frame in frames: if len(frame) > n: if isinstance(frame, bytes): frame = memoryview(frame) for i in range(0, len(frame), n): out.append(frame[i: i + n]) else: out.append(frame) return out def merge_frames(header, frames): """ Merge frames into original lengths Examples -------- >>> merge_frames({'lengths': [3, 3]}, [b'123456']) [b'123', b'456'] >>> merge_frames({'lengths': [6]}, [b'123', b'456']) [b'123456'] """ lengths = list(header['lengths'])[::-1] frames = list(frames)[::-1] if not frames: return frames assert sum(lengths) == sum(map(len, frames)) if all(len(f) == l for f, l in zip(frames, lengths)): return frames out = [] while lengths: l = lengths.pop() L = [] while l: frame = frames.pop() if len(frame) <= l: L.append(frame) l -= len(frame) else: mv = memoryview(frame) L.append(mv[:l]) frames.append(mv[l:]) l = 0 out.append(b''.join(map(ensure_bytes, L))) return out distributed-1.14.3/distributed/scheduler.py000066400000000000000000003412371301211143400207740ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from collections import defaultdict, deque, OrderedDict from datetime import datetime, timedelta import logging import math from math import log import os import pickle import random import socket from time import time from timeit import default_timer try: from cytoolz import frequencies, topk except ImportError: from toolz import frequencies, topk from toolz import memoize, valmap, first, second, keymap, unique, concat, merge from tornado import gen from tornado.gen import Return from tornado.queues import Queue from tornado.ioloop import IOLoop, PeriodicCallback from tornado.iostream import StreamClosedError, IOStream from dask.compatibility import PY3, unicode from dask.core import reverse_dict from dask.order import order from .batched import BatchedSend from .config import config from .core import (rpc, connect, read, write, MAX_BUFFER_SIZE, Server, send_recv, coerce_to_address, error_message) from .utils import (All, ignoring, clear_queue, get_ip, ignore_exceptions, ensure_ip, get_fileno_limit, log_errors, key_split, mean, divide_n_among_bins) from .utils_comm import (scatter_to_workers, gather_from_workers) from .versions import get_versions logger = logging.getLogger(__name__) BANDWIDTH = config.get('bandwidth', 100e6) ALLOWED_FAILURES = config.get('allowed-failures', 3) LOG_PDB = config.get('pdb-on-err') or os.environ.get('DASK_ERROR_PDB', False) class Scheduler(Server): """ Dynamic distributed task scheduler The scheduler tracks the current state of workers, data, and computations. The scheduler listens for events and responds by controlling workers appropriately. It continuously tries to use the workers to execute an ever growing dask graph. All events are handled quickly, in linear time with respect to their input (which is often of constant size) and generally within a millisecond. To accomplish this the scheduler tracks a lot of state. Every operation maintains the consistency of this state. The scheduler communicates with the outside world through Tornado IOStreams It maintains a consistent and valid view of the world even when listening to several clients at once. A Scheduler is typically started either with the ``dask-scheduler`` executable:: $ dask-scheduler Scheduler started at 127.0.0.1:8786 Or within a LocalCluster a Client starts up without connection information:: >>> c = Client() # doctest: +SKIP >>> c.cluster.scheduler # doctest: +SKIP Scheduler(...) Users typically do not interact with the scheduler directly but rather with the client object ``Client``. **State** The scheduler contains the following state variables. Each variable is listed along with what it stores and a brief description. * **tasks:** ``{key: task}``: Dictionary mapping key to a serialized task like the following: ``{'function': b'...', 'args': b'...'}`` or ``{'task': b'...'}`` * **dependencies:** ``{key: {keys}}``: Dictionary showing which keys depend on which others * **dependents:** ``{key: {keys}}``: Dictionary showing which keys are dependent on which others * **task_state:** ``{key: string}``: Dictionary listing the current state of every task among the following: released, waiting, stacks, queue, no-worker, processing, memory, erred * **priority:** ``{key: tuple}``: A score per key that determines its priority * **waiting:** ``{key: {key}}``: Dictionary like dependencies but excludes keys already computed * **waiting_data:** ``{key: {key}}``: Dictionary like dependents but excludes keys already computed * **ready:** ``deque(key)`` Keys that are ready to run, but not yet assigned to a worker * **stacks:** ``{worker: [keys]}``: List of keys waiting to be sent to each worker * **processing:** ``{worker: {key: cost}}``: Set of keys currently in execution on each worker and their expected duration * **stack_durations:** ``{worker: [ints]}``: Expected durations of stacked tasks * **stacks_duration:** ``{worker: int}``: Total duration of all tasks in each workers stack * **rprocessing:** ``{key: {worker}}``: Set of workers currently executing a particular task * **who_has:** ``{key: {worker}}``: Where each key lives. The current state of distributed memory. * **has_what:** ``{worker: {key}}``: What worker has what keys. The transpose of who_has. * **released:** ``{keys}`` Set of keys that are known, but released from memory * **unrunnable:** ``{key}`` Keys that we are unable to run * **restrictions:** ``{key: {hostnames}}``: A set of hostnames per key of where that key can be run. Usually this is empty unless a key has been specifically restricted to only run on certain hosts. These restrictions don't include a worker port. Any worker on that hostname is deemed valid. * **loose_restrictions:** ``{key}``: Set of keys for which we are allow to violate restrictions (see above) if not valid workers are present. * **exceptions:** ``{key: Exception}``: A dict mapping keys to remote exceptions * **tracebacks:** ``{key: list}``: A dict mapping keys to remote tracebacks stored as a list of strings * **exceptions_blame:** ``{key: key}``: A dict mapping a key to another key on which it depends that has failed * **suspicious_tasks:** ``{key: int}`` Number of times a task has been involved in a worker failure * **deleted_keys:** ``{key: {workers}}`` Locations of workers that have keys that should be deleted * **wants_what:** ``{client: {key}}``: What keys are wanted by each client.. The transpose of who_wants. * **who_wants:** ``{key: {client}}``: Which clients want each key. The active targets of computation. * **nbytes:** ``{key: int}``: Number of bytes for a key as reported by workers holding that key. * **stealable:** ``[[key]]`` A list of stacks of stealable keys, ordered by stealability * **ncores:** ``{worker: int}``: Number of cores owned by each worker * **idle:** ``{worker}``: Set of workers that are not fully utilized * **worker_info:** ``{worker: {str: data}}``: Information about each worker * **host_info:** ``{hostname: dict}``: Information about each worker host * **worker_bytes:** ``{worker: int}``: Number of bytes in memory on each worker * **occupancy:** ``{worker: time}`` Expected runtime for all tasks currently processing on a worker * **services:** ``{str: port}``: Other services running on this scheduler, like HTTP * **loop:** ``IOLoop``: The running Tornado IOLoop * **streams:** ``[IOStreams]``: A list of Tornado IOStreams from which we both accept stimuli and report results * **task_duration:** ``{key-prefix: time}`` Time we expect certain functions to take, e.g. ``{'sum': 0.25}`` * **coroutines:** ``[Futures]``: A list of active futures that control operation * **scheduler_queues:** ``[Queues]``: A list of Tornado Queues from which we accept stimuli * **report_queues:** ``[Queues]``: A list of Tornado Queues on which we report results """ default_port = 8786 def __init__(self, center=None, loop=None, max_buffer_size=MAX_BUFFER_SIZE, delete_interval=500, synchronize_worker_interval=60000, ip=None, services=None, allowed_failures=ALLOWED_FAILURES, validate=False, steal=True, **kwargs): # Attributes self.ip = ip or get_ip() self.allowed_failures = allowed_failures self.validate = validate self.status = None self.delete_interval = delete_interval self.synchronize_worker_interval = synchronize_worker_interval self.steal = steal # Communication state self.loop = loop or IOLoop.current() self.scheduler_queues = [Queue()] self.report_queues = [] self.worker_streams = dict() self.streams = dict() self.coroutines = [] self._worker_coroutines = [] self._ipython_kernel = None # Task state self.tasks = dict() self.task_state = dict() self.dependencies = dict() self.dependents = dict() self.generation = 0 self.released = set() self.priority = dict() self.nbytes = dict() self.worker_bytes = dict() self.processing = dict() self.rprocessing = defaultdict(set) self.task_duration = {prefix: 0.00001 for prefix in fast_tasks} self.restrictions = dict() self.loose_restrictions = set() self.suspicious_tasks = defaultdict(lambda: 0) self.stacks = dict() self.stack_durations = dict() self.stack_duration = dict() self.waiting = dict() self.waiting_data = dict() self.ready = deque() self.unrunnable = set() self.idle = set() self.maybe_idle = set() self.who_has = dict() self.has_what = dict() self.who_wants = defaultdict(set) self.wants_what = defaultdict(set) self.deleted_keys = defaultdict(set) self.exceptions = dict() self.tracebacks = dict() self.exceptions_blame = dict() self.datasets = dict() self.stealable = [set() for i in range(12)] self.key_stealable = dict() self.stealable_unknown_durations = defaultdict(set) # Worker state self.ncores = dict() self.worker_info = dict() self.host_info = defaultdict(dict) self.aliases = dict() self.saturated = set() self.occupancy = dict() self.plugins = [] self.transition_log = deque(maxlen=config.get('transition-log-length', 100000)) self.compute_handlers = {'update-graph': self.update_graph, 'update-data': self.update_data, 'missing-data': self.stimulus_missing_data, 'client-releases-keys': self.client_releases_keys, 'restart': self.restart} self.handlers = {'register-client': self.add_client, 'scatter': self.scatter, 'register': self.add_worker, 'unregister': self.remove_worker, 'gather': self.gather, 'cancel': self.stimulus_cancel, 'feed': self.feed, 'terminate': self.close, 'broadcast': self.broadcast, 'ncores': self.get_ncores, 'has_what': self.get_has_what, 'who_has': self.get_who_has, 'stacks': self.get_stacks, 'processing': self.get_processing, 'nbytes': self.get_nbytes, 'versions': self.get_versions, 'add_keys': self.add_keys, 'rebalance': self.rebalance, 'replicate': self.replicate, 'start_ipython': self.start_ipython, 'list_datasets': self.list_datasets, 'get_dataset': self.get_dataset, 'publish_dataset': self.publish_dataset, 'unpublish_dataset': self.unpublish_dataset, 'update_data': self.update_data, 'change_worker_cores': self.change_worker_cores} self.services = {} for k, v in (services or {}).items(): if isinstance(k, tuple): k, port = k else: port = 0 try: service = v(self, io_loop=self.loop) service.listen(port) self.services[k] = service except Exception as e: logger.info("Could not launch service: %s-%d", k, port, exc_info=True) self._transitions = { ('released', 'waiting'): self.transition_released_waiting, ('waiting', 'ready'): self.transition_waiting_ready, ('waiting', 'released'): self.transition_waiting_released, ('queue', 'processing'): self.transition_ready_processing, ('stacks', 'processing'): self.transition_ready_processing, ('processing', 'released'): self.transition_processing_released, ('queue', 'released'): self.transition_ready_released, ('stacks', 'released'): self.transition_ready_released, ('no-worker', 'released'): self.transition_ready_released, ('processing', 'memory'): self.transition_processing_memory, ('processing', 'erred'): self.transition_processing_erred, ('released', 'forgotten'): self.transition_released_forgotten, ('memory', 'forgotten'): self.transition_memory_forgotten, ('erred', 'forgotten'): self.transition_released_forgotten, ('memory', 'released'): self.transition_memory_released, ('released', 'erred'): self.transition_released_erred } connection_limit = get_fileno_limit() / 2 super(Scheduler, self).__init__(handlers=self.handlers, max_buffer_size=max_buffer_size, io_loop=self.loop, connection_limit=connection_limit, deserialize=False, **kwargs) ################## # Administration # ################## def __str__(self): return '' % ( self.ip, self.port, len(self.ncores), sum(self.ncores.values())) __repr__ = __str__ @property def address(self): return '%s:%d' % (self.ip, self.port) @property def address_tuple(self): return (self.ip, self.port) def identity(self, stream): """ Basic information about ourselves and our cluster """ d = {'type': type(self).__name__, 'id': str(self.id), 'workers': list(self.ncores), 'services': {key: v.port for (key, v) in self.services.items()}, 'workers': dict(self.worker_info)} return d def get_versions(self, stream): """ Basic information about ourselves and our cluster """ return get_versions() def start(self, port=8786, start_queues=True): """ Clear out old state and restart all running coroutines """ collections = [self.tasks, self.dependencies, self.dependents, self.waiting, self.waiting_data, self.released, self.priority, self.nbytes, self.restrictions, self.loose_restrictions, self.ready, self.who_wants, self.wants_what] for collection in collections: collection.clear() with ignoring(AttributeError): for c in self._worker_coroutines: c.cancel() self._delete_periodic_callback = \ PeriodicCallback(callback=self.clear_data_from_workers, callback_time=self.delete_interval, io_loop=self.loop) self._delete_periodic_callback.start() self._synchronize_data_periodic_callback = \ PeriodicCallback(callback=self.synchronize_worker_data, callback_time=self.synchronize_worker_interval, io_loop=self.loop) self._synchronize_data_periodic_callback.start() if start_queues: self.loop.add_callback(self.handle_queues, self.scheduler_queues[0], None) for cor in self.coroutines: if cor.done(): exc = cor.exception() if exc: raise exc if self.status != 'running': self.listen(port) self.status = 'running' logger.info(" Scheduler at: %20s:%s", self.ip, self.port) for k, v in self.services.items(): logger.info("%11s at: %20s:%s", k, self.ip, v.port) return self.finished() @gen.coroutine def finished(self): """ Wait until all coroutines have ceased """ while any(not c.done() for c in self.coroutines): yield All(self.coroutines) def close_streams(self): """ Close all active IOStreams """ for stream in self.streams.values(): stream.stream.close() self.rpc.close() @gen.coroutine def close(self, stream=None, fast=False): """ Send cleanup signal to all coroutines then wait until finished See Also -------- Scheduler.cleanup """ self._delete_periodic_callback.stop() self._synchronize_data_periodic_callback.stop() for service in self.services.values(): service.stop() yield self.cleanup() if not fast: yield self.finished() self.close_streams() self.status = 'closed' self.stop() @gen.coroutine def cleanup(self): """ Clean up queues and coroutines, prepare to stop """ if self.status == 'closing': raise gen.Return() self.status = 'closing' logger.debug("Cleaning up coroutines") for w, bstream in list(self.worker_streams.items()): with ignoring(AttributeError): yield bstream.close(ignore_closed=True) for s in self.scheduler_queues[1:]: s.put_nowait({'op': 'close-stream'}) for q in self.report_queues: q.put_nowait({'op': 'close'}) ########### # Stimuli # ########### def add_worker(self, stream=None, address=None, keys=(), ncores=None, name=None, coerce_address=True, nbytes=None, now=None, host_info=None, **info): """ Add a new worker to the cluster """ with log_errors(): local_now = time() now = now or time() info = info or {} host_info = host_info or {} if coerce_address: address = self.coerce_address(address) host, port = address.split(':') self.host_info[host]['last-seen'] = local_now if address not in self.worker_info: self.worker_info[address] = dict() if info: self.worker_info[address].update(info) if host_info: self.host_info[host].update(host_info) delay = time() - now self.worker_info[address]['time-delay'] = delay self.worker_info[address]['last-seen'] = time() if address in self.ncores: return 'OK' name = name or address if name in self.aliases: return 'name taken, %s' % name if coerce_address: if 'ports' not in self.host_info[host]: self.host_info[host].update({'ports': set(), 'cores': 0}) self.host_info[host]['ports'].add(port) self.host_info[host]['cores'] += ncores self.ncores[address] = ncores self.aliases[name] = address self.worker_info[address]['name'] = name if address not in self.processing: self.has_what[address] = set() self.worker_bytes[address] = 0 self.processing[address] = dict() self.occupancy[address] = 0 self.stacks[address] = deque() self.stack_durations[address] = deque() self.stack_duration[address] = 0 if nbytes: self.nbytes.update(nbytes) # for key in keys: # TODO # self.mark_key_in_memory(key, [address]) self.worker_streams[address] = BatchedSend(interval=2, loop=self.loop) self._worker_coroutines.append(self.worker_stream(address)) if self.ncores[address] > len(self.processing[address]): self.idle.add(address) for key in list(self.unrunnable): r = self.restrictions.get(key, []) if address in r or host in r or name in r: self.transitions({key: 'released'}) self.maybe_idle.add(address) self.ensure_occupied() logger.info("Register %s", str(address)) return 'OK' def update_graph(self, client=None, tasks=None, keys=None, dependencies=None, restrictions=None, priority=None, loose_restrictions=None): """ Add new computations to the internal dask graph This happens whenever the Client calls submit, map, get, or compute. """ for k in list(tasks): if tasks[k] is k: del tasks[k] if k in self.tasks: del tasks[k] original_keys = keys keys = set(keys) for k in keys: self.who_wants[k].add(client) self.wants_what[client].add(k) n = 0 while len(tasks) != n: # walk thorough new tasks, cancel any bad deps n = len(tasks) for k, deps in list(dependencies.items()): if any(dep not in self.dependencies and dep not in tasks for dep in deps): # bad key logger.info('User asked for computation on lost data, %s', k) del tasks[k] del dependencies[k] if k in keys: keys.remove(k) self.report({'op': 'cancelled-key', 'key': k}) self.client_releases_keys(keys=[k], client=client) stack = list(keys) touched = set() while stack: k = stack.pop() if k in self.dependencies: continue touched.add(k) if k not in self.tasks and k in tasks: self.tasks[k] = tasks[k] self.dependencies[k] = set(dependencies.get(k, ())) self.released.add(k) self.task_state[k] = 'released' for dep in self.dependencies[k]: if dep not in self.dependents: self.dependents[dep] = set() self.dependents[dep].add(k) if k not in self.dependents: self.dependents[k] = set() stack.extend(self.dependencies[k]) recommendations = OrderedDict() new_priority = priority or order(tasks) # TODO: define order wrt old graph self.generation += 1 # older graph generations take precedence for key in set(new_priority) & touched: if key not in self.priority: self.priority[key] = (self.generation, new_priority[key]) # prefer old if restrictions: restrictions = {k: set(map(self.coerce_address, v)) for k, v in restrictions.items()} self.restrictions.update(restrictions) if loose_restrictions: self.loose_restrictions |= set(loose_restrictions) for key in sorted(touched | keys, key=self.priority.get): if self.task_state[key] == 'released': recommendations[key] = 'waiting' for key in touched | keys: for dep in self.dependencies[key]: if dep in self.exceptions_blame: self.exceptions_blame[key] = self.exceptions_blame[dep] recommendations[key] = 'erred' break self.transitions(recommendations) for plugin in self.plugins[:]: try: plugin.update_graph(self, client=client, tasks=tasks, keys=keys, restrictions=restrictions or {}, dependencies=dependencies, loose_restrictions=loose_restrictions) except Exception as e: logger.exception(e) for key in keys: if self.task_state[key] in ('memory', 'erred'): self.report_on_key(key) self.ensure_occupied() def stimulus_task_finished(self, key=None, worker=None, **kwargs): """ Mark that a task has finished execution on a particular worker """ # logger.debug("Stimulus task finished %s, %s", key, worker) self.maybe_idle.add(worker) if key not in self.task_state: return {} if self.task_state[key] == 'processing': recommendations = self.transition(key, 'memory', worker=worker, **kwargs) else: recommendations = {} if self.task_state[key] == 'memory': self.who_has[key].add(worker) if key not in self.has_what[worker]: self.worker_bytes[worker] += self.nbytes.get(key, 1000) self.has_what[worker].add(key) return recommendations def stimulus_task_erred(self, key=None, worker=None, exception=None, traceback=None, **kwargs): """ Mark that a task has erred on a particular worker """ logger.debug("Stimulus task erred %s, %s", key, worker) self.maybe_idle.add(worker) if key not in self.task_state: return {} if self.task_state[key] == 'processing': recommendations = self.transition(key, 'erred', cause=key, exception=exception, traceback=traceback) else: recommendations = {} return recommendations def stimulus_missing_data(self, keys=None, key=None, worker=None, ensure=True, **kwargs): """ Mark that certain keys have gone missing. Recover. """ logger.debug("Stimulus missing data %s, %s", key, worker) if worker: self.maybe_idle.add(worker) recommendations = OrderedDict() for k in set(keys): if self.task_state.get(k) == 'memory': for w in set(self.who_has[k]): self.has_what[w].remove(k) self.who_has[k].remove(w) self.worker_bytes[w] -= self.nbytes.get(k, 1000) recommendations[k] = 'released' if key: recommendations[key] = 'released' self.transitions(recommendations) if ensure: self.ensure_occupied() return {} def remove_worker(self, stream=None, address=None, safe=False): """ Remove worker from cluster We do this when a worker reports that it plans to leave or when it appears to be unresponsive. This may send its tasks back to a released state. """ with log_errors(pdb=LOG_PDB): address = self.coerce_address(address) logger.info("Remove worker %s", address) if address not in self.processing: return 'already-removed' with ignoring(AttributeError): stream = self.worker_streams[address].stream if not stream.closed(): stream.close() host, port = address.split(':') self.host_info[host]['cores'] -= self.ncores[address] self.host_info[host]['ports'].remove(port) if not self.host_info[host]['ports']: del self.host_info[host] del self.worker_streams[address] del self.ncores[address] del self.aliases[self.worker_info[address]['name']] del self.worker_info[address] if address in self.maybe_idle: self.maybe_idle.remove(address) if address in self.idle: self.idle.remove(address) if address in self.saturated: self.saturated.remove(address) recommendations = OrderedDict() in_flight = set(self.processing.pop(address)) for k in list(in_flight): self.rprocessing[k].remove(address) if not safe: self.suspicious_tasks[k] += 1 if not safe and self.suspicious_tasks[k] > self.allowed_failures: e = pickle.dumps(KilledWorker(k, address)) r = self.transition(k, 'erred', exception=e, cause=k) recommendations.update(r) in_flight.remove(k) elif not self.rprocessing[k]: recommendations[k] = 'released' for k in self.stacks.pop(address): if k in self.tasks: recommendations[k] = 'waiting' del self.stack_durations[address] del self.stack_duration[address] del self.occupancy[address] del self.worker_bytes[address] for key in self.has_what.pop(address): self.who_has[key].remove(address) if not self.who_has[key]: if key in self.tasks: recommendations[key] = 'released' else: recommendations[key] = 'forgotten' self.transitions(recommendations) if not self.stacks: logger.info("Lost all workers") self.ensure_occupied() return 'OK' def stimulus_cancel(self, stream, keys=None, client=None): """ Stop execution on a list of keys """ logger.info("Client %s requests to cancel %d keys", client, len(keys)) for key in keys: self.cancel_key(key, client) def cancel_key(self, key, client, retries=5): """ Cancel a particular key and all dependents """ # TODO: this should be converted to use the transition mechanism if key not in self.who_wants: # no key yet, lets try again in 500ms if retries: self.loop.add_future(gen.sleep(0.2), lambda _: self.cancel_key(key, client, retries - 1)) return if self.who_wants[key] == {client}: # no one else wants this key for dep in list(self.dependents[key]): self.cancel_key(dep, client) logger.debug("Scheduler cancels key %s", key) self.report({'op': 'cancelled-key', 'key': key}) self.client_releases_keys(keys=[key], client=client) def client_releases_keys(self, keys=None, client=None): """ Remove keys from client desired list """ for key in list(keys): if key in self.wants_what[client]: self.wants_what[client].remove(key) s = self.who_wants[key] s.remove(client) if not s: del self.who_wants[key] if key in self.waiting_data and not self.waiting_data[key]: r = self.transition(key, 'released') self.transitions(r) if key in self.dependents and not self.dependents[key]: r = self.transition(key, 'forgotten') self.transitions(r) def client_wants_keys(self, keys=None, client=None): for k in keys: self.who_wants[k].add(client) self.wants_what[client].add(k) ###################################### # Task Validation (currently unused) # ###################################### def validate_released(self, key): assert key in self.dependencies assert self.task_state[key] == 'released' assert key not in self.waiting_data assert key not in self.who_has assert key not in self.rprocessing # assert key not in self.ready assert key not in self.waiting assert not any(key in self.waiting_data.get(dep, ()) for dep in self.dependencies[key]) assert key in self.released def validate_waiting(self, key): assert key in self.waiting assert key in self.waiting_data assert key not in self.who_has assert key not in self.rprocessing assert key not in self.released for dep in self.dependencies[key]: assert (dep in self.who_has) + (dep in self.waiting[key]) == 1 def validate_processing(self, key): assert key not in self.waiting assert key in self.waiting_data assert key in self.rprocessing for w in self.rprocessing[key]: assert key in self.processing[w] assert key not in self.who_has for dep in self.dependencies[key]: assert dep in self.who_has def validate_memory(self, key): assert key in self.who_has assert key not in self.rprocessing assert key not in self.waiting assert key not in self.released for dep in self.dependents[key]: assert (dep in self.who_has) + (dep in self.waiting_data[key]) == 1 def validate_queue(self, key): # assert key in self.ready assert key not in self.released assert key not in self.rprocessing assert key not in self.who_has assert key not in self.waiting for dep in self.dependencies[key]: assert dep in self.who_has def validate_stacks(self, key): # assert any(key in stack for stack in self.stacks.values()) assert key not in self.released assert key not in self.rprocessing assert key not in self.who_has assert key not in self.waiting for dep in self.dependencies[key]: assert dep in self.who_has def validate_key(self, key): try: try: func = getattr(self, 'validate_' + self.task_state[key]) except KeyError: logger.debug("Key lost: %s", key) except AttributeError: logger.info("self.validate_%s not found", self.task_state[key]) else: func(key) except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def validate_state(self, allow_overlap=False, allow_bad_stacks=True): validate_state(self.dependencies, self.dependents, self.waiting, self.waiting_data, self.ready, self.who_has, self.stacks, self.processing, None, self.released, self.who_wants, self.wants_what, tasks=self.tasks, erred=self.exceptions_blame, allow_overlap=allow_overlap, allow_bad_stacks=allow_bad_stacks) if not (set(self.ncores) == \ set(self.has_what) == \ set(self.stacks) == \ set(self.processing) == \ set(self.worker_info) == \ set(self.worker_streams)): raise ValueError("Workers not the same in all collections") assert self.worker_bytes == {w: sum(self.nbytes[k] for k in keys) for w, keys in self.has_what.items()} for w in self.stacks: assert abs(sum(self.stack_durations[w]) - self.stack_duration[w]) < 1e-8 assert len(self.stack_durations[w]) == len(self.stacks[w]) ################### # Manage Messages # ################### def report(self, msg): """ Publish updates to all listening Queues and Streams If the message contains a key then we only send the message to those streams that care about the key. """ for q in self.report_queues: q.put_nowait(msg) if 'key' in msg: streams = [self.streams[c] for c in self.who_wants.get(msg['key'], ()) if c in self.streams] else: streams = self.streams.values() for s in streams: try: s.send(msg) # logger.debug("Scheduler sends message to client %s", msg) except StreamClosedError: logger.critical("Tried writing to closed stream: %s", msg) @gen.coroutine def add_client(self, stream, client=None): """ Add client to network We listen to all future messages from this IOStream. """ logger.info("Receive client connection: %s", client) bstream = BatchedSend(interval=2, loop=self.loop) bstream.start(stream) self.streams[client] = bstream try: yield self.handle_messages(stream, bstream, client=client) finally: if not stream.closed(): bstream.send({'op': 'stream-closed'}) yield bstream.close(ignore_closed=True) del self.streams[client] logger.info("Close client connection: %s", client) def remove_client(self, client=None): """ Remove client from network """ logger.info("Remove client %s", client) self.client_releases_keys(self.wants_what.get(client, ()), client) with ignoring(KeyError): del self.wants_what[client] @gen.coroutine def handle_messages(self, in_queue, report, client=None): """ The master client coroutine. Handles all inbound messages from clients. This runs once per Client IOStream or Queue. See Also -------- Scheduler.worker_stream: The equivalent function for workers """ with log_errors(pdb=LOG_PDB): if isinstance(in_queue, Queue): next_message = in_queue.get elif isinstance(in_queue, IOStream): next_message = lambda: read(in_queue, deserialize=self.deserialize) else: raise NotImplementedError() if isinstance(report, Queue): put = report.put_nowait elif isinstance(report, IOStream): put = lambda msg: write(report, msg) elif isinstance(report, BatchedSend): put = report.send else: put = lambda msg: None put({'op': 'stream-start'}) breakout = False while True: try: msgs = yield next_message() except (StreamClosedError, AssertionError, GeneratorExit): break except Exception as e: logger.exception(e) put(error_message(e, status='scheduler-error')) continue if not isinstance(msgs, list): msgs = [msgs] for msg in msgs: # logger.debug("scheduler receives message %s", msg) try: op = msg.pop('op') except Exception as e: logger.exception(e) put(error_message(e, status='scheduler-error')) if op == 'close-stream': breakout = True break elif op == 'close': breakout = True self.close() break elif op in self.compute_handlers: try: result = self.compute_handlers[op](**msg) if isinstance(result, gen.Future): yield result except Exception as e: logger.exception(e) raise else: logger.warn("Bad message: op=%s, %s", op, msg, exc_info=True) if op == 'close': breakout = True break if breakout: break self.remove_client(client=client) logger.debug('Finished handle_messages coroutine') def handle_queues(self, scheduler_queue, report_queue): """ Register new control and report queues to the Scheduler Queues are not in common use. This may be deprecated in the future. """ self.scheduler_queues.append(scheduler_queue) if report_queue: self.report_queues.append(report_queue) future = self.handle_messages(scheduler_queue, report_queue) self.coroutines.append(future) return future def send_task_to_worker(self, worker, key): """ Send a single computational task to a worker """ msg = {'op': 'compute-task', 'key': key} deps = self.dependencies[key] if deps: msg['who_has'] = {dep: tuple(self.who_has.get(dep, ())) for dep in deps} task = self.tasks[key] if type(task) is dict: msg.update(task) else: msg['task'] = task self.worker_streams[worker].send(msg) @gen.coroutine def worker_stream(self, worker): """ Listen to responses from a single worker This is the main loop for scheduler-worker interaction See Also -------- Scheduler.handle_messages: Equivalent coroutine for clients """ yield gen.sleep(0) ip, port = coerce_to_address(worker, out=tuple) stream = yield connect(ip, port) yield write(stream, {'op': 'compute-stream'}) self.worker_streams[worker].start(stream) logger.info("Starting worker compute stream, %s", worker) try: while True: msgs = yield read(stream) if not isinstance(msgs, list): msgs = [msgs] if worker in self.worker_info: recommendations = OrderedDict() for msg in msgs: # logger.debug("Compute response from worker %s, %s", # worker, msg) if msg == 'OK': # from close break self.correct_time_delay(worker, msg) key = msg['key'] if msg['status'] == 'OK': r = self.stimulus_task_finished(worker=worker, **msg) recommendations.update(r) elif msg['status'] == 'error': r = self.stimulus_task_erred(worker=worker, **msg) recommendations.update(r) elif msg['status'] == 'missing-data': r = self.stimulus_missing_data(worker=worker, ensure=False, **msg) recommendations.update(r) else: logger.warn("Unknown message type, %s, %s", msg['status'], msg) self.transitions(recommendations) if self.validate: logger.info("Messages: %s\nRecommendations: %s", msgs, recommendations) self.ensure_occupied() except (StreamClosedError, IOError, OSError): logger.info("Worker failed from closed stream: %s", worker) except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise finally: if not stream.closed(): stream.close() self.remove_worker(address=worker) def correct_time_delay(self, worker, msg): """ Apply offset time delay in message times. Clocks on different workers differ. We keep track of a relative "now" through periodic heartbeats. We use this known delay to align message times to Scheduler local time. In particular this helps with diagnostics. Operates in place """ if 'time-delay' in self.worker_info[worker]: delay = self.worker_info[worker]['time-delay'] for key in ['transfer_start', 'transfer_stop', 'time', 'compute_start', 'compute_stop', 'disk_load_start', 'disk_load_stop']: if key in msg: msg[key] += delay @gen.coroutine def clear_data_from_workers(self): """ Send delete signals to clear unused data from workers This watches the ``.deleted_keys`` attribute, which stores a set of keys to be deleted from each worker. This function is run periodically by the ``._delete_periodic_callback`` to actually remove the data. This runs every ``self.delete_interval`` milliseconds. """ if self.deleted_keys: d = self.deleted_keys.copy() self.deleted_keys.clear() coroutines = [self.rpc(addr=worker).delete_data( keys=list(keys - self.has_what.get(worker, set())), report=False) for worker, keys in d.items() if keys] for worker, keys in d.items(): logger.debug("Remove %d keys from worker %s", len(keys), worker) yield ignore_exceptions(coroutines, socket.error, StreamClosedError) raise Return('OK') def add_plugin(self, plugin): """ Add external plugin to scheduler See https://distributed.readthedocs.io/en/latest/plugins.html """ self.plugins.append(plugin) def remove_plugin(self, plugin): """ Remove external plugin from scheduler """ self.plugins.remove(plugin) ############################ # Less common interactions # ############################ @gen.coroutine def scatter(self, stream=None, data=None, workers=None, client=None, broadcast=False, timeout=2): """ Send data out to workers See also -------- Scheduler.broadcast: """ start = time() while not self.ncores: yield gen.sleep(0.2) if time() > start + timeout: raise gen.TimeoutError("No workers found") if workers is not None: workers = [self.coerce_address(w) for w in workers] ncores = workers if workers is not None else self.ncores keys, who_has, nbytes = yield scatter_to_workers(ncores, data, report=False, serialize=False) self.update_data(who_has=who_has, nbytes=nbytes, client=client) if broadcast: if broadcast == True: n = len(ncores) else: n = broadcast yield self.replicate(keys=keys, workers=workers, n=n) raise gen.Return(keys) @gen.coroutine def gather(self, stream=None, keys=None): """ Collect data in from workers """ keys = list(keys) who_has = {key: self.who_has.get(key, ()) for key in keys} try: data = yield gather_from_workers(who_has, rpc=self.rpc, close=False) result = {'status': 'OK', 'data': data} except KeyError as e: logger.debug("Couldn't gather keys %s", e) result = {'status': 'error', 'keys': e.args} raise gen.Return(result) @gen.coroutine def restart(self, environment=None): """ Restart all workers. Reset local state. """ n = len(self.ncores) with log_errors(): logger.debug("Send shutdown signal to workers") for q in self.scheduler_queues + self.report_queues: clear_queue(q) nannies = {addr: d['services']['nanny'] for addr, d in self.worker_info.items()} for addr in nannies: self.remove_worker(address=addr) for client, keys in self.wants_what.items(): self.client_releases_keys(keys=keys, client=client) logger.debug("Send kill signal to nannies: %s", nannies) nannies = [rpc(ip=worker_address.split(':')[0], port=n_port) for worker_address, n_port in nannies.items()] try: yield All([nanny.kill() for nanny in nannies]) logger.debug("Received done signal from nannies") while self.ncores: yield gen.sleep(0.01) logger.debug("Workers all removed. Sending startup signal") # All quiet resps = yield All([nanny.instantiate(close=True, environment=environment) for nanny in nannies]) assert all(resp == 'OK' for resp in resps) finally: for nanny in nannies: nanny.close_rpc() self.start() logger.debug("All workers reporting in") self.report({'op': 'restart'}) for plugin in self.plugins[:]: try: plugin.restart(self) except Exception as e: logger.exception(e) @gen.coroutine def broadcast(self, stream=None, msg=None, workers=None, hosts=None, nanny=False): """ Broadcast message to workers, return all results """ if workers is None: if hosts is None: workers = list(self.ncores) else: workers = [] if hosts is not None: for host in hosts: if host in self.host_info: workers.extend([host + ':' + port for port in self.host_info[host]['ports']]) # TODO replace with worker_list if nanny: addresses = [] for addr in workers: ip = addr.split(':')[0] port = self.worker_info[addr]['services']['nanny'] addresses.append('%s:%d' % (ip, port)) else: addresses = workers results = yield All([send_recv(arg=address, close=True, **msg) for address in addresses]) raise Return(dict(zip(workers, results))) @gen.coroutine def rebalance(self, stream=None, keys=None, workers=None): """ Rebalance keys so that each worker stores roughly equal bytes **Policy** This orders the workers by what fraction of bytes of the existing keys they have. It walks down this list from most-to-least. At each worker it sends the largest results it can find and sends them to the least occupied worker until either the sender or the recipient are at the average expected load. """ with log_errors(): keys = set(keys or self.who_has) workers = set(workers or self.ncores) if not keys.issubset(self.who_has): raise Return({'status': 'missing-data', 'keys': list(keys - set(self.who_has))}) workers_by_key = {k: self.who_has.get(k, set()) & workers for k in keys} keys_by_worker = {w: set() for w in workers} for k, v in workers_by_key.items(): for vv in v: keys_by_worker[vv].add(k) worker_bytes = {w: sum(self.nbytes.get(k, 1000) for k in v) for w, v in keys_by_worker.items()} avg = sum(worker_bytes.values()) / len(worker_bytes) sorted_workers = list(map(first, sorted(worker_bytes.items(), key=second, reverse=True))) recipients = iter(reversed(sorted_workers)) recipient = next(recipients) msgs = [] # (sender, recipient, key) for sender in sorted_workers[:len(workers) // 2]: sender_keys = {k: self.nbytes.get(k, 1000) for k in keys_by_worker[sender]} sender_keys = iter(sorted(sender_keys.items(), key=second, reverse=True)) try: while worker_bytes[sender] > avg: while (worker_bytes[recipient] < avg and worker_bytes[sender] > avg): k, nb = next(sender_keys) if k not in keys_by_worker[recipient]: keys_by_worker[recipient].add(k) # keys_by_worker[sender].remove(k) msgs.append((sender, recipient, k)) worker_bytes[sender] -= nb worker_bytes[recipient] += nb if worker_bytes[sender] > avg: recipient = next(recipients) except StopIteration: break to_recipients = defaultdict(lambda: defaultdict(list)) to_senders = defaultdict(list) for sender, recipient, key in msgs: to_recipients[recipient][key].append(sender) to_senders[sender].append(key) result = yield {r: self.rpc(addr=r).gather(who_has=v) for r, v in to_recipients.items()} if not all(r['status'] == 'OK' for r in result.values()): raise Return({'status': 'missing-data', 'keys': sum([r['keys'] for r in result if 'keys' in r], [])}) for sender, recipient, key in msgs: self.who_has[key].add(recipient) self.has_what[recipient].add(key) self.worker_bytes[recipient] += self.nbytes.get(key, 1000) result = yield {r: self.rpc(addr=r).delete_data(keys=v, report=False) for r, v in to_senders.items()} for sender, recipient, key in msgs: self.who_has[key].remove(sender) self.has_what[sender].remove(key) self.worker_bytes[sender] -= self.nbytes.get(key, 1000) raise Return({'status': 'OK'}) @gen.coroutine def replicate(self, stream=None, keys=None, n=None, workers=None, branching_factor=2, delete=True): """ Replicate data throughout cluster This performs a tree copy of the data throughout the network individually on each piece of data. Parameters ---------- keys: Iterable list of keys to replicate n: int Number of replications we expect to see within the cluster branching_factor: int, optional The number of workers that can copy data in each generation See also -------- Scheduler.rebalance """ workers = set(self.workers_list(workers)) if n is None: n = len(workers) n = min(n, len(workers)) keys = set(keys) if n == 0: raise ValueError("Can not use replicate to delete data") if not keys.issubset(self.who_has): raise Return({'status': 'missing-data', 'keys': list(keys - set(self.who_has))}) # Delete extraneous data if delete: del_keys = {k: random.sample(self.who_has[k] & workers, len(self.who_has[k] & workers) - n) for k in keys if len(self.who_has[k] & workers) > n} del_workers = {k: v for k, v in reverse_dict(del_keys).items() if v} yield [self.rpc(addr=worker).delete_data(keys=list(keys), report=False) for worker, keys in del_workers.items()] for worker, keys in del_workers.items(): self.has_what[worker] -= keys for key in keys: self.who_has[key].remove(worker) self.worker_bytes[worker] -= self.nbytes.get(key, 1000) keys = {k for k in keys if len(self.who_has[k] & workers) < n} # Copy not-yet-filled data while keys: gathers = defaultdict(dict) for k in list(keys): missing = workers - self.who_has[k] count = min(max(n - len(self.who_has[k] & workers), 0), branching_factor * len(self.who_has[k])) if not count: keys.remove(k) else: sample = random.sample(missing, count) for w in sample: gathers[w][k] = list(self.who_has[k]) results = yield {w: self.rpc(addr=w).gather(who_has=who_has) for w, who_has in gathers.items()} for w, v in results.items(): if v['status'] == 'OK': self.add_keys(address=w, keys=list(gathers[w])) def workers_to_close(self, memory_ratio=2): if not self.idle or self.ready: return [] limit_bytes = {w: self.worker_info[w]['memory_limit'] for w in self.worker_info} worker_bytes = self.worker_bytes limit = sum(limit_bytes.values()) total = sum(worker_bytes.values()) idle = sorted(self.idle, key=worker_bytes.get, reverse=True) to_close = [] while idle: w = idle.pop() limit -= limit_bytes[w] if limit >= memory_ratio * total: # still plenty of space to_close.append(w) else: break return to_close @gen.coroutine def retire_workers(self, stream=None, workers=None, remove=True): if workers is None: while True: try: workers = self.workers_to_close() if workers: yield self.retire_workers(workers=workers, remove=remove) raise gen.Return(list(workers)) except KeyError: # keys left during replicate pass workers = set(workers) keys = set.union(*[self.has_what[w] for w in workers]) keys = {k for k in keys if self.who_has[k].issubset(workers)} other_workers = set(self.worker_info) - workers if keys: if other_workers: yield self.replicate(keys=keys, workers=other_workers, n=1, delete=False) else: raise gen.Return([]) if remove: for w in workers: self.remove_worker(address=w, safe=True) raise gen.Return(list(workers)) @gen.coroutine def synchronize_worker_data(self, stream=None, worker=None): if worker is None: result = yield {w: self.synchronize_worker_data(worker=w) for w in self.worker_info} result = {k: v for k, v in result.items() if any(v.values())} if result: logger.info("Excess keys found on workers: %s", result) raise Return(result or None) else: keys = yield self.rpc(addr=worker).keys() keys = set(keys) missing = self.has_what[worker] - keys if missing: logger.info("Expected data missing from worker: %s, %s", worker, missing) extra = keys - self.has_what[worker] - self.deleted_keys[worker] if extra: yield gen.sleep(self.synchronize_worker_interval / 1000) # delay keys = yield self.rpc(addr=worker).keys() # check again extra &= set(keys) # make sure the keys are still on worker extra -= self.has_what[worker] # and still unknown to scheduler if extra: # still around? delete them yield self.rpc(addr=worker).delete_data(keys=list(extra), report=False) raise Return({'extra': list(extra), 'missing': list(missing)}) def add_keys(self, stream=None, address=None, keys=()): """ Learn that a worker has certain keys This should not be used in practice and is mostly here for legacy reasons. """ address = coerce_to_address(address) if address not in self.worker_info: return 'not found' for key in keys: if key in self.who_has: if key not in self.has_what[address]: self.worker_bytes[address] += self.nbytes.get(key, 1000) self.has_what[address].add(key) self.who_has[key].add(address) # else: # TODO: delete key from worker return 'OK' def update_data(self, stream=None, who_has=None, nbytes=None, client=None): """ Learn that new data has entered the network from an external source See Also -------- Scheduler.mark_key_in_memory """ with log_errors(): who_has = {k: [self.coerce_address(vv) for vv in v] for k, v in who_has.items()} logger.debug("Update data %s", who_has) if client: self.client_wants_keys(keys=list(who_has), client=client) # for key, workers in who_has.items(): # TODO # self.mark_key_in_memory(key, workers) self.nbytes.update(nbytes) for key, workers in who_has.items(): if key not in self.dependents: self.dependents[key] = set() if key not in self.dependencies: self.dependencies[key] = set() self.task_state[key] = 'memory' self.who_has[key] = set(workers) for w in workers: if key not in self.has_what[w]: self.worker_bytes[w] += self.nbytes.get(key, 1000) self.has_what[w].add(key) self.waiting_data[key] = set() self.report({'op': 'key-in-memory', 'key': key, 'workers': list(workers)}) def report_on_key(self, key): if key not in self.task_state: self.report({'op': 'cancelled-key', 'key': key}) elif self.task_state[key] == 'memory': self.report({'op': 'key-in-memory', 'key': key}) elif self.task_state[key] == 'erred': failing_key = self.exceptions_blame[key] self.report({'op': 'task-erred', 'key': key, 'exception': self.exceptions[failing_key], 'traceback': self.tracebacks.get(failing_key, None)}) @gen.coroutine def feed(self, stream, function=None, setup=None, teardown=None, interval=1, **kwargs): """ Provides a data stream to external requester Caution: this runs arbitrary Python code on the scheduler. This should eventually be phased out. It is mostly used by diagnostics. """ import pickle with log_errors(): if function: function = pickle.loads(function) if setup: setup = pickle.loads(setup) if teardown: teardown = pickle.loads(teardown) state = setup(self) if setup else None if isinstance(state, gen.Future): state = yield state try: while True: if state is None: response = function(self) else: response = function(self, state) yield write(stream, response) yield gen.sleep(interval) except (OSError, IOError, StreamClosedError): if teardown: teardown(self, state) def get_stacks(self, stream=None, workers=None): if workers is not None: workers = set(map(self.coerce_address, workers)) return {w: list(self.stacks[w]) for w in workers} else: return valmap(list, self.stacks) def get_processing(self, stream=None, workers=None): if workers is not None: workers = set(map(self.coerce_address, workers)) return {w: list(self.processing[w]) for w in workers} else: return valmap(list, self.processing) def get_who_has(self, stream=None, keys=None): if keys is not None: return {k: list(self.who_has.get(k, [])) for k in keys} else: return valmap(list, self.who_has) def get_has_what(self, stream=None, workers=None): if workers is not None: workers = map(self.coerce_address, workers) return {w: list(self.has_what.get(w, ())) for w in workers} else: return valmap(list, self.has_what) def get_ncores(self, stream=None, workers=None): if workers is not None: workers = map(self.coerce_address, workers) return {w: self.ncores.get(w, None) for w in workers} else: return self.ncores def get_nbytes(self, stream=None, keys=None, summary=True): with log_errors(): if keys is not None: result = {k: self.nbytes[k] for k in keys} else: result = self.nbytes if summary: out = defaultdict(lambda: 0) for k, v in result.items(): out[key_split(k)] += v result = out return result def publish_dataset(self, stream=None, keys=None, data=None, name=None, client=None): if name in self.datasets: raise KeyError("Dataset %s already exists" % name) self.client_wants_keys(keys, 'published-%s' % name) self.datasets[name] = {'data': data, 'keys': keys} return {'status': 'OK', 'name': name} def unpublish_dataset(self, stream=None, name=None): out = self.datasets.pop(name, {'keys': []}) self.client_releases_keys(out['keys'], 'published-%s' % name) def list_datasets(self, *args): return list(sorted(self.datasets.keys())) def get_dataset(self, stream, name=None, client=None): if name in self.datasets: return self.datasets[name] else: raise KeyError("Dataset '%s' not found" % name) def change_worker_cores(self, stream=None, worker=None, diff=0): """ Add or remove cores from a worker This is used when a worker wants to spin off a long-running task """ self.ncores[worker] += diff self.maybe_idle.add(worker) self.ensure_occupied() ##################### # State Transitions # ##################### def transition_released_waiting(self, key): try: if self.validate: assert key in self.tasks assert key in self.dependencies assert key in self.dependents assert key not in self.waiting # assert key not in self.readyset # assert key not in self.rstacks assert key not in self.who_has assert key not in self.rprocessing # assert all(dep in self.task_state # for dep in self.dependencies[key]) if not all(dep in self.task_state for dep in self.dependencies[key]): return {key: 'forgotten'} self.waiting[key] = set() recommendations = OrderedDict() for dep in self.dependencies[key]: if dep in self.exceptions_blame: self.exceptions_blame[key] = self.exceptions_blame[dep] recommendations[key] = 'erred' return recommendations for dep in self.dependencies[key]: if dep not in self.who_has: self.waiting[key].add(dep) if dep in self.released: recommendations[dep] = 'waiting' else: self.waiting_data[dep].add(key) if not self.waiting[key]: recommendations[key] = 'ready' self.waiting_data[key] = {dep for dep in self.dependents[key] if dep not in self.who_has and dep not in self.released and dep not in self.exceptions_blame} self.task_state[key] = 'waiting' self.released.remove(key) if self.validate: assert key in self.waiting assert key in self.waiting_data return recommendations except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def transition_waiting_ready(self, key): try: if self.validate: assert key in self.waiting assert not self.waiting[key] assert key not in self.who_has assert key not in self.exceptions_blame assert key not in self.rprocessing # assert key not in self.readyset assert key not in self.unrunnable assert all(dep in self.who_has for dep in self.dependencies[key]) del self.waiting[key] if self.dependencies.get(key, None) or key in self.restrictions: new_worker = decide_worker(self.dependencies, self.stacks, self.stack_duration, self.processing, self.who_has, self.has_what, self.restrictions, self.loose_restrictions, self.nbytes, self.ncores, key) if not new_worker: self.unrunnable.add(key) self.task_state[key] = 'no-worker' else: self.stacks[new_worker].append(key) duration = self.task_duration.get(key_split(key), 0.5) self.stack_durations[new_worker].append(duration) self.stack_duration[new_worker] += duration self.maybe_idle.add(new_worker) self.put_key_in_stealable(key) self.task_state[key] = 'stacks' else: self.ready.appendleft(key) self.task_state[key] = 'queue' return {} except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def transition_ready_processing(self, key, worker=None, latency=5e-3): try: if self.validate: assert key not in self.waiting assert key not in self.who_has assert key not in self.exceptions_blame assert self.task_state[key] in ('queue', 'stacks') if self.task_state[key] == 'no-worker': raise ValueError() assert worker duration = self.task_duration.get(key_split(key), latency*100) self.processing[worker][key] = duration self.rprocessing[key].add(worker) self.occupancy[worker] += duration self.task_state[key] = 'processing' self.remove_key_from_stealable(key) # logger.debug("Send job to worker: %s, %s", worker, key) try: self.send_task_to_worker(worker, key) except StreamClosedError: self.remove_worker(worker) return {} except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def transition_processing_memory(self, key, nbytes=None, type=None, worker=None, compute_start=None, compute_stop=None, transfer_start=None, transfer_stop=None, **kwargs): try: if self.validate: assert key in self.rprocessing assert all(key in self.processing[w] for w in self.rprocessing[key]) assert key not in self.waiting assert key not in self.who_has assert key not in self.exceptions_blame # assert all(dep in self.waiting_data[key ] for dep in # self.dependents[key] if self.task_state[dep] in # ['waiting', 'queue', 'stacks']) # assert key not in self.nbytes assert self.task_state[key] == 'processing' if worker not in self.processing: return {key: 'released'} ############################# # Update Timing Information # ############################# if compute_start: # Update average task duration for worker info = self.worker_info[worker] ks = key_split(key) gap = (transfer_start or compute_start) - info.get('last-task', 0) old_duration = self.task_duration.get(ks, 0) new_duration = compute_stop - compute_start if (not old_duration or gap > max(10e-3, info.get('latency', 0), old_duration)): avg_duration = new_duration else: avg_duration = (0.5 * old_duration + 0.5 * new_duration) self.task_duration[ks] = avg_duration if ks in self.stealable_unknown_durations: for k in self.stealable_unknown_durations.pop(ks, ()): if self.task_state.get(k) == 'stacks': self.put_key_in_stealable(k) info['last-task'] = compute_stop ############################ # Update State Information # ############################ if nbytes: self.nbytes[key] = nbytes self.who_has[key] = set() if worker: self.who_has[key].add(worker) self.has_what[worker].add(key) self.worker_bytes[worker] += self.nbytes.get(key, 1000) if nbytes: self.nbytes[key] = nbytes workers = self.rprocessing.pop(key) for worker in workers: self.occupancy[worker] -= self.processing[worker].pop(key) recommendations = OrderedDict() deps = self.dependents.get(key, []) if len(deps) > 1: deps = sorted(deps, key=self.priority.get, reverse=True) for dep in deps: if dep in self.waiting: s = self.waiting[dep] s.remove(key) if not s: # new task ready to run recommendations[dep] = 'ready' for dep in self.dependencies.get(key, []): if dep in self.waiting_data: s = self.waiting_data[dep] s.remove(key) if (not s and dep and dep not in self.who_wants and not self.waiting_data.get(dep)): recommendations[dep] = 'released' if (not self.waiting_data.get(key) and key not in self.who_wants): recommendations[key] = 'released' else: msg = {'op': 'key-in-memory', 'key': key} if type is not None: msg['type'] = type self.report(msg) self.task_state[key] = 'memory' if self.validate: assert key not in self.rprocessing return recommendations except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def transition_memory_released(self, key, safe=False): try: if self.validate: assert key in self.who_has assert key not in self.released # assert key not in self.readyset assert key not in self.waiting assert key not in self.rprocessing if safe: assert not self.waiting_data.get(key) # assert key not in self.who_wants recommendations = OrderedDict() for dep in self.waiting_data.get(key, ()): # lost dependency if self.task_state[dep] == 'waiting': self.waiting[dep].add(key) else: recommendations[dep] = 'waiting' workers = self.who_has.pop(key) for w in workers: if w in self.worker_info: # in case worker has died self.has_what[w].remove(key) self.worker_bytes[w] -= self.nbytes.get(key, 1000) self.deleted_keys[w].add(key) self.released.add(key) self.task_state[key] = 'released' self.report({'op': 'lost-data', 'key': key}) if key not in self.tasks: # pure data recommendations[key] = 'forgotten' elif not all(dep in self.task_state for dep in self.dependencies[key]): recommendations[key] = 'forgotten' elif key in self.who_wants or self.waiting_data.get(key): recommendations[key] = 'waiting' if key in self.waiting_data: del self.waiting_data[key] return recommendations except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def transition_released_erred(self, key): try: if self.validate: with log_errors(pdb=LOG_PDB): assert key in self.exceptions_blame assert key not in self.who_has assert key not in self.waiting assert key not in self.waiting_data recommendations = {} failing_key = self.exceptions_blame[key] for dep in self.dependents[key]: self.exceptions_blame[dep] = failing_key if dep not in self.who_has: recommendations[dep] = 'erred' self.report({'op': 'task-erred', 'key': key, 'exception': self.exceptions[failing_key], 'traceback': self.tracebacks.get(failing_key, None)}) self.task_state[key] = 'erred' self.released.remove(key) # TODO: waiting data? return recommendations except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def transition_waiting_released(self, key): try: if self.validate: assert key in self.waiting assert key in self.waiting_data assert key not in self.who_has assert key not in self.rprocessing recommendations = {} del self.waiting[key] for dep in self.dependencies[key]: if dep in self.waiting_data: if key in self.waiting_data[dep]: self.waiting_data[dep].remove(key) if not self.waiting_data[dep] and dep not in self.who_wants: recommendations[dep] = 'released' assert self.task_state[dep] != 'erred' self.task_state[key] = 'released' self.released.add(key) if self.validate: assert not any(key in self.waiting_data.get(dep, ()) for dep in self.dependencies[key]) if any(dep not in self.task_state for dep in self.dependencies[key]): recommendations[key] = 'forgotten' elif (key not in self.exceptions_blame and (key in self.who_wants or self.waiting_data.get(key))): recommendations[key] = 'waiting' del self.waiting_data[key] return recommendations except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def transition_processing_released(self, key): try: if self.validate: assert key in self.rprocessing assert key not in self.who_has assert self.task_state[key] == 'processing' for w in self.rprocessing.pop(key): self.occupancy[w] -= self.processing[w].pop(key) self.released.add(key) self.task_state[key] = 'released' recommendations = OrderedDict() if any(dep not in self.task_state for dep in self.dependencies[key]): recommendations[key] = 'forgotten' elif self.waiting_data[key] or key in self.who_wants: recommendations[key] = 'waiting' else: for dep in self.dependencies[key]: if dep not in self.released: assert key in self.waiting_data[dep] self.waiting_data[dep].remove(key) if not self.waiting_data[dep] and dep not in self.who_wants: recommendations[dep] = 'released' del self.waiting_data[key] if self.validate: assert key not in self.rprocessing return recommendations except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def transition_ready_released(self, key): try: if self.validate: assert key not in self.who_has assert self.task_state[key] in ('stacks', 'queue', 'no-worker') if self.task_state[key] == 'no-worker': self.unrunnable.remove(key) if self.task_state[key] == 'stacks': # TODO: non-linear for w in self.stacks: if key in self.stacks[w]: for i, k in enumerate(self.stacks[w]): if k == key: del self.stacks[w][i] duration = self.stack_durations[w][i] del self.stack_durations[w][i] self.stack_duration[w] -= duration break self.released.add(key) self.task_state[key] = 'released' for dep in self.dependencies[key]: try: self.waiting_data[dep].remove(key) except KeyError: # dep may also be released pass # TODO: maybe release dep if not about to wait? if self.waiting_data[key] or key in self.who_wants: recommendations = {key: 'waiting'} else: recommendations = {} del self.waiting_data[key] return recommendations except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def transition_processing_erred(self, key, cause=None, exception=None, traceback=None): try: if self.validate: assert cause or key in self.exceptions_blame assert key in self.rprocessing assert key not in self.who_has assert key not in self.waiting # assert key not in self.rstacks # assert key not in self.readyset if exception: self.exceptions[key] = exception if traceback: self.tracebacks[key] = traceback if cause: self.exceptions_blame[key] = cause failing_key = self.exceptions_blame[key] recommendations = {} for dep in self.dependents[key]: self.exceptions_blame[dep] = key recommendations[dep] = 'erred' for dep in self.dependencies.get(key, []): if dep in self.waiting_data: s = self.waiting_data[dep] if key in s: s.remove(key) if (not s and dep and dep not in self.who_wants and not self.waiting_data.get(dep)): recommendations[dep] = 'released' for w in self.rprocessing.pop(key): self.occupancy[w] -= self.processing[w].pop(key) del self.waiting_data[key] # do anything with this? self.task_state[key] = 'erred' self.report({'op': 'task-erred', 'key': key, 'exception': self.exceptions[failing_key], 'traceback': self.tracebacks.get(failing_key)}) if self.validate: assert key not in self.rprocessing return recommendations except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def remove_key(self, key): if key in self.tasks: del self.tasks[key] del self.task_state[key] if key in self.dependencies: del self.dependencies[key] del self.dependents[key] if key in self.restrictions: del self.restrictions[key] if key in self.loose_restrictions: self.loose_restrictions.remove(key) if key in self.priority: del self.priority[key] if key in self.exceptions: del self.exceptions[key] if key in self.exceptions_blame: del self.exceptions_blame[key] if key in self.released: self.released.remove(key) if key in self.waiting_data: del self.waiting_data[key] if key in self.suspicious_tasks: del self.suspicious_tasks[key] if key in self.nbytes: del self.nbytes[key] def transition_memory_forgotten(self, key): try: if self.validate: assert key in self.dependents assert self.task_state[key] == 'memory' assert key in self.waiting_data assert key in self.who_has assert key not in self.rprocessing # assert key not in self.ready assert key not in self.waiting recommendations = {} for dep in self.waiting_data[key]: recommendations[dep] = 'forgotten' for dep in self.dependents[key]: if self.task_state[dep] == 'released': recommendations[dep] = 'forgotten' for dep in self.dependencies.get(key, ()): try: s = self.dependents[dep] s.remove(key) if not s and dep not in self.who_wants: assert dep is not key recommendations[dep] = 'forgotten' except KeyError: pass workers = self.who_has.pop(key) for w in workers: if w in self.worker_info: # in case worker has died self.has_what[w].remove(key) self.worker_bytes[w] -= self.nbytes.get(key, 1000) self.deleted_keys[w].add(key) if self.validate: assert all(key not in self.dependents[dep] for dep in self.dependencies[key] if dep in self.task_state) assert all(key not in self.waiting_data.get(dep, ()) for dep in self.dependencies[key] if dep in self.task_state) self.remove_key(key) self.report_on_key(key) return recommendations except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def transition_released_forgotten(self, key): try: if self.validate: assert key in self.dependencies assert self.task_state[key] in ('released', 'erred') # assert not self.waiting_data[key] if key in self.tasks and self.dependencies[key].issubset(self.task_state): assert key not in self.who_wants assert not self.dependents[key] assert not any(key in self.waiting_data.get(dep, ()) for dep in self.dependencies[key]) assert key not in self.who_has assert key not in self.rprocessing # assert key not in self.ready assert key not in self.waiting recommendations = {} for dep in self.dependencies[key]: try: s = self.dependents[dep] s.remove(key) if not s and dep not in self.who_wants: assert dep is not key recommendations[dep] = 'forgotten' except KeyError: pass for dep in self.dependents[key]: if self.task_state[dep] not in ('memory', 'error'): recommendations[dep] = 'forgotten' for dep in self.dependents[key]: if self.task_state[dep] == 'released': recommendations[dep] = 'forgotten' for dep in self.dependencies[key]: try: self.waiting_data[dep].remove(key) except KeyError: pass if self.validate: assert all(key not in self.dependents[dep] for dep in self.dependencies[key] if dep in self.task_state) assert all(key not in self.waiting_data.get(dep, ()) for dep in self.dependencies[key] if dep in self.task_state) self.remove_key(key) self.report_on_key(key) return recommendations except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def transition(self, key, finish, *args, **kwargs): """ Transition a key from its current state to the finish state Examples -------- >>> self.transition('x', 'waiting') {'x': 'ready'} Returns ------- Dictionary of recommendations for future transitions See Also -------- Scheduler.transitions: transitive version of this function """ try: try: start = self.task_state[key] except KeyError: return {} if start == finish: return {} if (start, finish) in self._transitions: func = self._transitions[start, finish] recommendations = func(key, *args, **kwargs) else: func = self._transitions['released', finish] assert not args and not kwargs a = self.transition(key, 'released') if key in a: func = self._transitions['released', a[key]] b = func(key) a = a.copy() a.update(b) recommendations = a start = 'released' finish2 = self.task_state.get(key, 'forgotten') self.transition_log.append((key, start, finish2, recommendations)) if self.validate: logger.info("Transition %s->%s: %s New: %s", start, finish2, key, recommendations) for plugin in self.plugins: try: plugin.transition(key, start, finish2, *args, **kwargs) except Exception: logger.info("Plugin failed with exception", exc_info=True) return recommendations except Exception as e: logger.exception(e) if LOG_PDB: import pdb; pdb.set_trace() raise def transitions(self, recommendations): """ Process transitions until none are left This includes feedback from previous transitions and continues until we reach a steady state """ keys = set() recommendations = recommendations.copy() while recommendations: key, finish = recommendations.popitem() keys.add(key) new = self.transition(key, finish) recommendations.update(new) if self.validate: for key in keys: self.validate_key(key) def transition_story(self, *keys): """ Get all transitions that touch one of the input keys """ keys = set(keys) return [t for t in self.transition_log if t[0] in keys or keys.intersection(t[3])] ############################## # Assigning Tasks to Workers # ############################## def ensure_occupied(self): """ Run ready tasks on idle workers **Work stealing policy** If some workers are idle but not others, if there are no globally ready tasks, and if there are tasks in worker stacks, then we start to pull preferred tasks from overburdened workers and deploy them back into the global pool in the following manner. We determine the number of tasks to reclaim as the number of all tasks in all stacks times the fraction of idle workers to all workers. We sort the stacks by size and walk through them, reclaiming half of each stack until we have enough task to fill the global pool. We are careful not to reclaim tasks that are restricted to run on certain workers. See also -------- Scheduler.ensure_occupied_queue Scheduler.ensure_occupied_stacks Scheduler.work_steal """ with log_errors(pdb=LOG_PDB): for worker in self.maybe_idle: self.ensure_occupied_stacks(worker) self.maybe_idle.clear() if self.idle and self.ready: if len(self.ready) < len(self.idle): def keyfunc(w): return (-len(self.stacks[w]) - len(self.processing[w]), -len(self.has_what.get(w, ()))) for worker in topk(len(self.ready), self.idle, key=keyfunc): self.ensure_occupied_queue(worker, count=1) else: # Fill up empty cores workers = list(self.idle) free_cores = [self.ncores[w] - len(self.processing[w]) for w in workers] workers2 = [] # Clean out workers that *are* actually full free_cores2 = [] for w, fs in zip(workers, free_cores): if fs > 0: workers2.append(w) free_cores2.append(fs) if workers2: n = min(sum(free_cores2), len(self.ready)) counts = divide_n_among_bins(n, free_cores2) for worker, count in zip(workers2, counts): self.ensure_occupied_queue(worker, count=count) # Fill up unsaturated cores by time workers = list(self.idle) latency = 5e-3 free_time = [latency * self.ncores[w] - self.occupancy[w] for w in workers] workers2 = [] # Clean out workers that *are* actually full free_time2 = [] for w, fs in zip(workers, free_time): if fs > 0: workers2.append(w) free_time2.append(fs) total_free_time = sum(free_time2) if workers2 and total_free_time > 0: tasks = [] while self.ready and total_free_time > 0: task = self.ready.pop() if self.task_state.get(task) != 'queue': continue total_free_time -= self.task_duration.get(key_split(task), 1) tasks.append(task) self.ready.extend(tasks[::-1]) counts = divide_n_among_bins(len(tasks), free_time2) for worker, count in zip(workers2, counts): self.ensure_occupied_queue(worker, count=count) if self.idle and any(self.stealable): thieves = self.work_steal() for worker in thieves: self.ensure_occupied_stacks(worker) def ensure_occupied_stacks(self, worker): """ Send tasks to worker while it has tasks and free cores These tasks may come from the worker's own stacks or from the global ready deque. We update the idle workers set appropriately. See Also -------- Scheduler.ensure_occupied Scheduler.ensure_occupied_queue """ stack = self.stacks[worker] latency = 5e-3 while (stack and (self.ncores[worker] > len(self.processing[worker]) or self.occupancy[worker] < latency * self.ncores[worker])): key = stack.pop() duration = self.stack_durations[worker].pop() self.stack_duration[worker] -= duration if self.task_state.get(key) == 'stacks': r = self.transition(key, 'processing', worker=worker, latency=latency) if stack: self.saturated.add(worker) if worker in self.idle: self.idle.remove(worker) else: if worker in self.saturated: self.saturated.remove(worker) self._check_idle(worker) def put_key_in_stealable(self, key): ratio, loc = self.steal_time_ratio(key) if ratio is not None: self.stealable[loc].add(key) self.key_stealable[key] = loc def remove_key_from_stealable(self, key): loc = self.key_stealable.pop(key, None) if loc is not None: try: self.stealable[loc].remove(key) except: pass def ensure_occupied_queue(self, worker, count): """ Send at most count tasks from the ready queue to the specified worker See also -------- Scheduler.ensure_occupied Scheduler.ensure_occupied_stacks """ for i in range(count): try: key = self.ready.pop() while self.task_state.get(key) != 'queue': key = self.ready.pop() except (IndexError, KeyError): break if self.task_state[key] == 'queue': r = self.transition(key, 'processing', worker=worker) self._check_idle(worker) def work_steal(self): """ Steal tasks from saturated workers to idle workers This moves tasks from the bottom of the stacks of over-occupied workers to the stacks of idling workers. See also -------- Scheduler.ensure_occupied """ if not self.steal: return [] with log_errors(): thieves = set() for level, stealable in enumerate(self.stealable[:-1]): if not stealable: continue if len(self.idle) == len(self.ncores): # no stacks stealable.clear() continue # Enough idleness to continue? ratio = 2 ** (level - 3) n_saturated = len(self.ncores) - len(self.idle) duration_if_hold = len(stealable) / n_saturated duration_if_steal = ratio if level > 1 and duration_if_hold < duration_if_steal: break while stealable and self.idle: for w in list(self.idle): try: key = stealable.pop() except: break else: if self.task_state.get(key, 'stacks'): self.stacks[w].append(key) duration = self.task_duration.get(key_split(key), 0.5) self.stack_durations[w].append(duration) self.stack_duration[w] += duration thieves.add(w) if (self.ncores[w] <= len(self.processing[w]) + len(self.stacks[w])): self.idle.remove(w) if stealable: break logger.debug('Stolen tasks for %d workers', len(thieves)) return thieves def steal_time_ratio(self, key, bandwidth=BANDWIDTH): """ The compute to communication time ratio of a key Returns ------- ratio: The compute/communication time ratio of the task loc: The self.stealable bin into which this key should go """ if key in self.restrictions and key not in self.loose_restrictions: return None, None # don't steal nbytes = sum(self.nbytes.get(k, 1000) for k in self.dependencies[key]) transfer_time = nbytes / bandwidth split = key_split(key) if split in fast_tasks: return None, None try: compute_time = self.task_duration[split] except KeyError: self.stealable_unknown_durations[split].add(key) return None, None else: try: ratio = compute_time / transfer_time except ZeroDivisionError: ratio = 10000 if ratio > 8: loc = 0 elif ratio < 2**-8: loc = -1 else: loc = int(-round(log(ratio) / log(2), 0) + 3) return ratio, loc def issaturated(self, worker, latency=5e-3): """ Determine if a worker has enough work to avoid being idle A worker is saturated if the following criteria are met 1. It is working on at least as many tasks as it has cores 2. The expected time it will take to complete all of its currently assigned tasks is at least a full round-trip time. This is relevant when it has many small tasks """ return (len(self.stacks[worker]) + len(self.processing[worker]) > self.ncores[worker] and self.occupancy[worker] > latency * self.ncores[worker]) def _check_idle(self, worker, latency=5e-3): if not self.issaturated(worker, latency=latency): self.idle.add(worker) elif worker in self.idle: self.idle.remove(worker) ##################### # Utility functions # ##################### def coerce_address(self, addr): """ Coerce possible input addresses to canonical form Handles lists, strings, bytes, tuples, or aliases """ if isinstance(addr, list): addr = tuple(addr) if addr in self.aliases: addr = self.aliases[addr] if isinstance(addr, bytes): addr = addr.decode() if addr in self.aliases: addr = self.aliases[addr] if isinstance(addr, unicode): if ':' in addr: addr = tuple(addr.rsplit(':', 1)) else: addr = ensure_ip(addr) if isinstance(addr, tuple): ip, port = addr if PY3 and isinstance(ip, bytes): ip = ip.decode() ip = ensure_ip(ip) port = int(port) addr = '%s:%d' % (ip, port) return addr def workers_list(self, workers): """ List of qualifying workers Takes a list of worker addresses or hostnames. Returns a list of all worker addresses that match """ if workers is None: return list(self.ncores) out = set() for w in workers: if ':' in w: out.add(w) else: out.update({ww for ww in self.ncores if w in ww}) # TODO: quadratic return list(out) def start_ipython(self, stream=None): """Start an IPython kernel Returns Jupyter connection info dictionary. """ from ._ipython_utils import start_ipython if self._ipython_kernel is None: self._ipython_kernel = start_ipython( ip=self.ip, ns={'scheduler': self}, log=logger, ) return self._ipython_kernel.get_connection_info() def decide_worker(dependencies, stacks, stack_duration, processing, who_has, has_what, restrictions, loose_restrictions, nbytes, ncores, key): """ Decide which worker should take task >>> dependencies = {'c': {'b'}, 'b': {'a'}} >>> stacks = {'alice:8000': ['z'], 'bob:8000': []} >>> processing = {'alice:8000': set(), 'bob:8000': set()} >>> who_has = {'a': {'alice:8000'}} >>> has_what = {'alice:8000': {'a'}} >>> nbytes = {'a': 100} >>> ncores = {'alice:8000': 1, 'bob:8000': 1} >>> restrictions = {} >>> loose_restrictions = set() We choose the worker that has the data on which 'b' depends (alice has 'a') >>> decide_worker(dependencies, stacks, processing, who_has, has_what, ... restrictions, loose_restrictions, nbytes, ncores, 'b') 'alice:8000' If both Alice and Bob have dependencies then we choose the less-busy worker >>> who_has = {'a': {'alice:8000', 'bob:8000'}} >>> has_what = {'alice:8000': {'a'}, 'bob:8000': {'a'}} >>> decide_worker(dependencies, stacks, processing, who_has, has_what, ... restrictions, loose_restrictions, nbytes, ncores, 'b') 'bob:8000' Optionally provide restrictions of where jobs are allowed to occur >>> restrictions = {'b': {'alice', 'charlie'}} >>> decide_worker(dependencies, stacks, processing, who_has, has_what, ... restrictions, loose_restrictions, nbytes, ncores, 'b') 'alice:8000' If the task requires data communication, then we choose to minimize the number of bytes sent between workers. This takes precedence over worker occupancy. >>> dependencies = {'c': {'a', 'b'}} >>> who_has = {'a': {'alice:8000'}, 'b': {'bob:8000'}} >>> has_what = {'alice:8000': {'a'}, 'bob:8000': {'b'}} >>> nbytes = {'a': 1, 'b': 1000} >>> stacks = {'alice:8000': [], 'bob:8000': []} >>> decide_worker(dependencies, stacks, processing, who_has, has_what, ... {}, set(), nbytes, ncores, 'c') 'bob:8000' """ deps = dependencies[key] assert all(d in who_has for d in deps) workers = frequencies([w for dep in deps for w in who_has[dep]]) if not workers: workers = stacks if key in restrictions: r = restrictions[key] workers = {w for w in workers if w in r or w.split(':')[0] in r} # TODO: nonlinear if not workers: workers = {w for w in stacks if w in r or w.split(':')[0] in r} if not workers: if key in loose_restrictions: return decide_worker(dependencies, stacks, stack_duration, processing, who_has, has_what, {}, set(), nbytes, ncores, key) else: return None if not workers or not stacks: return None if len(workers) == 1: return first(workers) # Select worker that will finish task first def objective(w): comm_bytes = sum([nbytes.get(k, 1000) for k in dependencies[key] if w not in who_has[k]]) stack_time = stack_duration[w] / ncores[w] start_time = comm_bytes / BANDWIDTH + stack_time return start_time return min(workers, key=objective) def validate_state(dependencies, dependents, waiting, waiting_data, ready, who_has, stacks, processing, finished_results, released, who_wants, wants_what, tasks=None, allow_overlap=False, allow_bad_stacks=False, erred=None, **kwargs): """ Validate a current runtime state This performs a sequence of checks on the entire graph, running in about linear time. This raises assert errors if anything doesn't check out. """ in_stacks = {k for v in stacks.values() for k in v} in_processing = {k for v in processing.values() for k in v} keys = {key for key in dependents if not dependents[key]} ready_set = set(ready) assert set(waiting).issubset(dependencies), "waiting not subset of deps" assert set(waiting_data).issubset(dependents), "waiting_data not subset" if tasks is not None: assert ready_set.issubset(tasks), "All ready tasks are tasks" assert set(dependents).issubset(set(tasks) | set(who_has)), "all dependents tasks" assert set(dependencies).issubset(set(tasks) | set(who_has)), "all dependencies tasks" for k, v in waiting.items(): assert v, "waiting on empty set" assert v.issubset(dependencies[k]), "waiting set not dependencies" for vv in v: assert vv not in who_has, ("waiting dependency in memory", k, vv) assert vv not in released, ("dependency released", k, vv) for dep in dependencies[k]: assert dep in v or who_has.get(dep), ("dep missing", k, dep) for k, v in waiting_data.items(): for vv in v: if vv in released: raise ValueError('dependent not in play', k, vv) if not (vv in ready_set or vv in waiting or vv in in_stacks or vv in in_processing): raise ValueError('dependent not in play2', k, vv) for v in concat(processing.values()): assert v in dependencies, "all processing keys in dependencies" for key in who_has: assert key in waiting_data or key in who_wants @memoize def check_key(key): """ Validate a single key, recurse downwards """ vals = ([key in waiting, key in ready, key in in_stacks, key in in_processing, not not who_has.get(key), key in released, key in erred]) if ((allow_overlap and sum(vals) < 1) or (not allow_overlap and sum(vals) != 1)): if not (in_stacks and waiting): # known ok state raise ValueError("Key exists in wrong number of places", key, vals) for dep in dependencies[key]: if dep in dependents: check_key(dep) # Recursive case if who_has.get(key): assert not any(key in waiting.get(dep, ()) for dep in dependents.get(key, ())) assert not waiting.get(key) if not allow_bad_stacks and (key in in_stacks or key in in_processing): if not all(who_has.get(dep) for dep in dependencies[key]): raise ValueError("Key in stacks/processing without all deps", key) assert not waiting.get(key) assert key not in ready if finished_results is not None: if key in finished_results: assert who_has.get(key) assert key in keys if key in keys and who_has.get(key): assert key in finished_results for key, s in who_wants.items(): assert s, "empty who_wants" for client in s: assert key in wants_what[client] if key in waiting: assert waiting[key], 'waiting empty' if key in ready: assert key not in waiting return True assert all(map(check_key, keys)) _round_robin = [0] fast_tasks = {'rechunk-split', 'shuffle-split'} class KilledWorker(Exception): pass distributed-1.14.3/distributed/sizeof.py000066400000000000000000000025771301211143400203160ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import sys from .compatibility import singledispatch from .utils import ignoring try: # PyPy does not support sys.getsizeof sys.getsizeof(1) getsizeof = sys.getsizeof except: # Monkey patch getsizeof = lambda x: 100 @singledispatch def sizeof(o): return getsizeof(o) @sizeof.register(list) @sizeof.register(tuple) @sizeof.register(set) @sizeof.register(frozenset) def sizeof_python_collection(seq): return getsizeof(seq) + sum(map(sizeof, seq)) with ignoring(ImportError): import numpy as np @sizeof.register(np.ndarray) def sizeof_numpy_ndarray(x): return int(x.nbytes) with ignoring(ImportError): import pandas as pd @sizeof.register(pd.DataFrame) def sizeof_pandas_dataframe(df): o = getsizeof(df) try: return int(o + df.memory_usage(index=True, deep=True).sum()) except: return int(o + df.memory_usage(index=True).sum()) @sizeof.register(pd.Series) def sizeof_pandas_series(s): try: return int(s.memory_usage(index=True, deep=True)) # new in 0.17.1 except: return int(sizeof(s.values) + sizeof(s.index)) @sizeof.register(pd.Index) def sizeof_pandas_index(i): try: return int(i.memory_usage(deep=True)) except: return int(i.nbytes) distributed-1.14.3/distributed/submit.py000066400000000000000000000054711301211143400203160ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import logging import os import socket import subprocess import tempfile import sys from tornado import gen from tornado.ioloop import IOLoop from distributed import rpc from distributed.compatibility import unicode from distributed.core import Server from distributed.utils import get_ip logger = logging.getLogger('distributed.remote') class RemoteClient(Server): def __init__(self, ip=get_ip(), local_dir=tempfile.mkdtemp(prefix='client-'), loop=None, **kwargs): self.ip = ip self.loop = loop or IOLoop.current() self.local_dir = local_dir handlers = {'upload_file': self.upload_file, 'execute': self.execute} super(RemoteClient, self).__init__(handlers, io_loop=self.loop, **kwargs) @gen.coroutine def _start(self, port=0): self.listen(port) def start(self, port=0): self.loop.add_callback(self._start, port) logger.info("Remote Client is running at {0}:{1}".format(self.ip, port)) @gen.coroutine def execute(self, stream=None, filename=None): script_path = os.path.join(self.local_dir, filename) cmd = '{0} {1}'.format(sys.executable, script_path) process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() return_code = process.returncode raise gen.Return({'stdout': out, 'stderr': err, 'returncode': return_code}) def upload_file(self, stream, filename=None, file_payload=None): out_filename = os.path.join(self.local_dir, filename) if isinstance(file_payload, unicode): file_payload = file_payload.encode() with open(out_filename, 'wb') as f: f.write(file_payload) return {'status': 'OK', 'nbytes': len(file_payload)} @gen.coroutine def _close(self): self.stop() def _remote(host, port, loop=IOLoop.current(), client=RemoteClient): host = host or get_ip() if ':' in host and port == 8788: host, port = host.rsplit(':', 1) port = int(port) ip = socket.gethostbyname(host) remote_client = client(ip=ip, loop=loop) remote_client.start(port=port) loop.start() loop.close() remote_client.stop() logger.info("End remote client at %s:%d", host, port) @gen.coroutine def _submit(remote_client_address, filepath): rc = rpc(addr=remote_client_address) remote_file = os.path.basename(filepath) with open(filepath, 'rb') as f: bytes_read = f.read() yield rc.upload_file(filename=remote_file, file_payload=bytes_read) result = yield rc.execute(filename=remote_file) raise gen.Return((result['stdout'], result['stderr'])) distributed-1.14.3/distributed/sync.py000066400000000000000000000035371301211143400177700ustar00rootroot00000000000000""" Synchronous versions of functions found in core.py These are not relevant for normal operation, but are sometimes useful for demonstration. """ from __future__ import print_function, division, absolute_import import socket import struct from . import protocol def connect_sync(ip, port): """ Connect with a blocking socket This is not typically used. See connect() instead """ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((ip, port)) return s def read_bytes_sync(sock, nbytes): """ Read number of bytes from a blocking socket This is not typically used, see IOStream.read_bytes instead """ frames = [] while nbytes: frame = sock.recv(nbytes) frames.append(frame) nbytes -= len(frame) if len(frames) == 1: return frames[0] else: return b''.join(frames) def read_sync(sock): """ Read message from blocking socket This is not typically used. See read() instead """ n_frames = read_bytes_sync(sock, 8) n_frames = struct.unpack('Q', n_frames)[0] lengths = read_bytes_sync(sock, 8 * n_frames) lengths = struct.unpack('Q' * n_frames, lengths) frames = [] for length in lengths: if length: frame = read_bytes_sync(sock, length) else: frame = b'' frames.append(frame) msg = protocol.loads(frames) return msg def write_sync(sock, msg): """ Write a message synchronously to a socket This is not typically used. See write() instead """ try: frames = protocol.dumps(msg) except Exception as e: logger.exception(e) raise lengths = ([struct.pack('Q', len(frames))] + [struct.pack('Q', len(frame)) for frame in frames]) sock.sendall(b''.join(lengths)) for frame in frames: sock.sendall(frame) distributed-1.14.3/distributed/tests/000077500000000000000000000000001301211143400175745ustar00rootroot00000000000000distributed-1.14.3/distributed/tests/mytestegg-1.0.0-py3.4.egg000066400000000000000000000034371301211143400236040ustar00rootroot00000000000000PKoGӉRsEGG-INFO/PKG-INFOM-ILI,I K-*ϳR03KMR(I-.IMOBJ&UZ)yqy$"KK29QԼb$m.E%``9%iEPKoG[ EGG-INFO/top_level.txt+I-.IMOPKoGI|rEGG-INFO/SOURCES.txtm 0{%7RV=/be1l՛s7s]ۄg2^ʐ Rt2߹oRpM'(d\W,%܋ˌR-)E<7PKoG2EGG-INFO/not-zip-safePKoG2EGG-INFO/dependency_links.txtPKG testegg/__init__.pyKIMSKԴRԒҢ= 3: assert results == list(map(add, range(3), range(4))) def f(x, y=10): return x + y L5 = c.map(f, range(5), y=5) results = yield c._gather(L5) assert results == list(range(5, 10)) y = c.submit(f, 10) L6 = c.map(f, range(5), y=y) results = yield c._gather(L6) assert results == list(range(20, 25)) s.validate_state() @gen_cluster(client=True) def test_map_empty(c, s, a, b): L1 = c.map(inc, [], pure=False) assert len(L1) == 0 results = yield c._gather(L1) assert results == [] @gen_cluster(client=True) def test_map_keynames(c, s, a, b): futures = c.map(inc, range(4), key='INC') assert all(f.key.startswith('INC') for f in futures) assert isdistinct(f.key for f in futures) futures2 = c.map(inc, [5, 6, 7, 8], key='INC') assert [f.key for f in futures] != [f.key for f in futures2] keys = ['inc-1', 'inc-2', 'inc-3', 'inc-4'] futures = c.map(inc, range(4), key=keys) assert [f.key for f in futures] == keys @gen_cluster() def test_compatible_map(s, a, b): e = CompatibleExecutor((s.ip, s.port), start=False) yield e._start() results = e.map(inc, range(5)) assert not isinstance(results, list) # Since this map blocks as it waits for results, # waiting here will block the current IOLoop, # which happens to also be running the test Workers. # So wait on the results in a background thread to avoid blocking. f = gen.Future() def wait_on_results(): f.set_result(list(results)) t = Thread(target=wait_on_results) t.daemon = True t.start() result_list = yield f # getting map results blocks assert result_list == list(map(inc, range(5))) yield e._shutdown() @gen_cluster(client=True) def test_future(c, s, a, b): x = c.submit(inc, 10) assert str(x.key) in repr(x) assert str(x.status) in repr(x) @gen_cluster(client=True) def test_Future_exception(c, s, a, b): x = c.submit(div, 1, 0) result = yield x._exception() assert isinstance(result, ZeroDivisionError) x = c.submit(div, 1, 1) result = yield x._exception() assert result is None def test_Future_exception_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = c.submit(div, 1, 0) assert isinstance(x.exception(), ZeroDivisionError) x = c.submit(div, 1, 1) assert x.exception() is None @gen_cluster(client=True) def test_map_naming(c, s, a, b): L1 = c.map(inc, range(5)) L2 = c.map(inc, range(5)) assert [x.key for x in L1] == [x.key for x in L2] L3 = c.map(inc, [1, 1, 1, 1]) assert len({x.event for x in L3}) == 1 L4 = c.map(inc, [1, 1, 1, 1], pure=False) assert len({x.event for x in L4}) == 4 @gen_cluster(client=True) def test_submit_naming(c, s, a, b): a = c.submit(inc, 1) b = c.submit(inc, 1) assert a.event is b.event c = c.submit(inc, 1, pure=False) assert c.key != a.key @gen_cluster(client=True) def test_exceptions(c, s, a, b): x = c.submit(div, 1, 2) result = yield x._result() assert result == 1 / 2 x = c.submit(div, 1, 0) with pytest.raises(ZeroDivisionError): result = yield x._result() x = c.submit(div, 10, 2) # continues to operate result = yield x._result() assert result == 10 / 2 @gen_cluster() def test_gc(s, a, b): c = Client((s.ip, s.port), start=False) yield c._start() x = c.submit(inc, 10) yield x._result() assert s.who_has[x.key] x.__del__() yield c._shutdown() assert x.key not in s.who_has def test_thread(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = c.submit(inc, 1) assert x.result() == 2 def test_sync_exceptions(loop): with cluster() as (s, [a, b]): c = Client(('127.0.0.1', s['port']), loop=loop) x = c.submit(div, 10, 2) assert x.result() == 5 y = c.submit(div, 10, 0) try: y.result() assert False except ZeroDivisionError: pass z = c.submit(div, 10, 5) assert z.result() == 2 c.shutdown() @gen_cluster(client=True) def test_gather(c, s, a, b): x = c.submit(inc, 10) y = c.submit(inc, x) result = yield c._gather(x) assert result == 11 result = yield c._gather([x]) assert result == [11] result = yield c._gather({'x': x, 'y': [y]}) assert result == {'x': 11, 'y': [12]} def test_gather_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = c.submit(inc, 1) assert c.gather(x) == 2 y = c.submit(div, 1, 0) with pytest.raises(ZeroDivisionError): c.gather([x, y]) [xx] = c.gather([x, y], errors='skip') assert xx == 2 @gen_cluster(client=True) def test_gather_strict(c, s, a, b): x = c.submit(div, 2, 1) y = c.submit(div, 1, 0) with pytest.raises(ZeroDivisionError): yield c._gather([x, y]) [xx] = yield c._gather([x, y], errors='skip') assert xx == 2 @gen_cluster(client=True, timeout=None) def test_get(c, s, a, b): result = yield c._get({'x': (inc, 1)}, 'x') assert result == 2 result = yield c._get({'x': (inc, 1)}, ['x']) assert result == [2] result = yield c._get({}, []) assert result == [] result = yield c._get({('x', 1): (inc, 1), ('x', 2): (inc, ('x', 1))}, ('x', 2)) assert result == 3 def test_get_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: assert c.get({'x': (inc, 1)}, 'x') == 2 def test_get_sync_optimize_graph_passes_through(loop): import dask.bag as db import dask bag = db.range(10, npartitions=3).map(inc) with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: dask.compute(bag.sum(), optimize_graph=False, get=c.get) def test_submit_errors(loop): def f(a, b, c): pass c = Client('127.0.0.1:8787', start=False, loop=loop) with pytest.raises(TypeError): c.submit(1, 2, 3) with pytest.raises(TypeError): c.map([1, 2, 3]) @gen_cluster(client=True) def test_wait(c, s, a, b): x = c.submit(inc, 1) y = c.submit(inc, 1) z = c.submit(inc, 2) done, not_done = yield _wait([x, y, z]) assert done == {x, y, z} assert not_done == set() assert x.status == y.status == 'finished' @gen_cluster(client=True) def test__as_completed(c, s, a, b): x = c.submit(inc, 1) y = c.submit(inc, 1) z = c.submit(inc, 2) from distributed.compatibility import Queue queue = Queue() yield _as_completed([x, y, z], queue) assert queue.qsize() == 3 assert {queue.get(), queue.get(), queue.get()} == {x, y, z} result = yield _first_completed([x, y, z]) assert result in [x, y, z] def test_as_completed(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = c.submit(inc, 1) y = c.submit(inc, 2) z = c.submit(inc, 1) seq = as_completed([x, y, z]) assert isinstance(seq, Iterator) assert set(seq) == {x, y, z} def test_wait_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = c.submit(inc, 1) y = c.submit(inc, 2) done, not_done = wait([x, y]) assert done == {x, y} assert not_done == set() assert x.status == y.status == 'finished' @gen_cluster(client=True) def test_garbage_collection(c, s, a, b): x = c.submit(inc, 1) y = c.submit(inc, 1) assert c.refcount[x.key] == 2 x.__del__() assert c.refcount[x.key] == 1 z = c.submit(inc, y) y.__del__() result = yield z._result() assert result == 3 ykey = y.key y.__del__() assert ykey not in c.futures @gen_cluster(client=True) def test_garbage_collection_with_scatter(c, s, a, b): [a] = yield c._scatter([1]) assert a.key in c.futures assert a.status == 'finished' assert a.event.is_set() assert s.who_wants[a.key] == {c.id} assert c.refcount[a.key] == 1 a.__del__() assert c.refcount[a.key] == 0 start = time() while True: if a.key not in s.who_has: break else: assert time() < start + 3 yield gen.sleep(0.1) @gen_cluster(timeout=1000, client=True) def test_recompute_released_key(c, s, a, b): x = c.submit(inc, 100) result1 = yield x._result() xkey = x.key del x import gc; gc.collect() assert c.refcount[xkey] == 0 # 1 second batching needs a second action to trigger while xkey in s.who_has or xkey in a.data or xkey in b.data: yield gen.sleep(0.1) x = c.submit(inc, 100) assert x.key in c.futures result2 = yield x._result() assert result1 == result2 @slow @gen_cluster(client=True) def test_long_tasks_dont_trigger_timeout(c, s, a, b): from time import sleep x = c.submit(sleep, 3) yield x._result() @gen_cluster(client=True) def test_missing_data_heals(c, s, a, b): x = c.submit(inc, 1) y = c.submit(inc, x) z = c.submit(inc, y) yield _wait([x, y, z]) # Secretly delete y's key if y.key in a.data: del a.data[y.key] if y.key in b.data: del b.data[y.key] w = c.submit(add, y, z) result = yield w._result() assert result == 3 + 4 @slow @gen_cluster() def test_missing_worker(s, a, b): bad = 'bad-host:8788' s.ncores[bad] = 4 s.who_has['b'] = {bad} s.has_what[bad] = {'b'} c = Client((s.ip, s.port), start=False) yield c._start() dsk = {'a': 1, 'b': (inc, 'a'), 'c': (inc, 'b')} result = yield c._get(dsk, 'c') assert result == 3 assert bad not in s.ncores yield c._shutdown() @gen_cluster(client=True) def test_gather_robust_to_missing_data(c, s, a, b): x, y, z = c.map(inc, range(3)) yield _wait([x, y, z]) # everything computed for q in [x, y]: if q.key in a.data: del a.data[q.key] if q.key in b.data: del b.data[q.key] xx, yy, zz = yield c._gather([x, y, z]) assert (xx, yy, zz) == (1, 2, 3) @gen_cluster(client=True) def test_gather_robust_to_nested_missing_data(c, s, a, b): w = c.submit(inc, 1) x = c.submit(inc, w) y = c.submit(inc, x) z = c.submit(inc, y) yield _wait([z]) for worker in [a, b]: for datum in [y, z]: if datum.key in worker.data: del worker.data[datum.key] result = yield c._gather([z]) assert result == [inc(inc(inc(inc(1))))] @gen_cluster(client=True) def test_tokenize_on_futures(c, s, a, b): x = c.submit(inc, 1) y = c.submit(inc, 1) tok = tokenize(x) assert tokenize(x) == tokenize(x) assert tokenize(x) == tokenize(y) c.futures[x.key]['status'] = 'finished' assert tok == tokenize(y) @pytest.mark.skipif(not sys.platform.startswith('linux'), reason="Need 127.0.0.2 to mean localhost") @gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True) def test_restrictions_submit(c, s, a, b): x = c.submit(inc, 1, workers={a.ip}) y = c.submit(inc, x, workers={b.ip}) yield _wait([x, y]) assert s.restrictions[x.key] == {a.ip} assert x.key in a.data assert s.restrictions[y.key] == {b.ip} assert y.key in b.data @gen_cluster(client=True) def test_restrictions_ip_port(c, s, a, b): x = c.submit(inc, 1, workers={a.address}) y = c.submit(inc, x, workers={b.address}) yield _wait([x, y]) assert s.restrictions[x.key] == {a.address} assert x.key in a.data assert s.restrictions[y.key] == {b.address} assert y.key in b.data @pytest.mark.skipif(not sys.platform.startswith('linux'), reason="Need 127.0.0.2 to mean localhost") @gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True) def test_restrictions_map(c, s, a, b): L = c.map(inc, range(5), workers={a.ip}) yield _wait(L) assert set(a.data) == {x.key for x in L} assert not b.data for x in L: assert s.restrictions[x.key] == {a.ip} L = c.map(inc, [10, 11, 12], workers=[{a.ip}, {a.ip, b.ip}, {b.ip}]) yield _wait(L) assert s.restrictions[L[0].key] == {a.ip} assert s.restrictions[L[1].key] == {a.ip, b.ip} assert s.restrictions[L[2].key] == {b.ip} with pytest.raises(ValueError): c.map(inc, [10, 11, 12], workers=[{a.ip}]) @pytest.mark.skipif(not sys.platform.startswith('linux'), reason="Need 127.0.0.2 to mean localhost") @gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True) def test_restrictions_get(c, s, a, b): dsk = {'x': 1, 'y': (inc, 'x'), 'z': (inc, 'y')} restrictions = {'y': {a.ip}, 'z': {b.ip}} result = yield c._get(dsk, ['y', 'z'], restrictions) assert result == [2, 3] assert 'y' in a.data assert 'z' in b.data @gen_cluster(client=True) def dont_test_bad_restrictions_raise_exception(c, s, a, b): z = c.submit(inc, 2, workers={'bad-address'}) try: yield z._result() assert False except ValueError as e: assert 'bad-address' in str(e) assert z.key in str(e) @gen_cluster(client=True, timeout=None) def test_remove_worker(c, s, a, b): L = c.map(inc, range(20)) yield _wait(L) yield b._close() assert b.address not in s.worker_info result = yield c._gather(L) assert result == list(map(inc, range(20))) @gen_cluster(ncores=[('127.0.0.1', 1)], client=True) def test_errors_dont_block(c, s, w): L = [c.submit(inc, 1), c.submit(throws, 1), c.submit(inc, 2), c.submit(throws, 2)] start = time() while not (L[0].status == L[2].status == 'finished'): assert time() < start + 5 yield gen.sleep(0.01) result = yield c._gather([L[0], L[2]]) assert result == [2, 3] @gen_cluster(client=True) def test_submit_quotes(c, s, a, b): def assert_list(x, z=[]): return isinstance(x, list) and isinstance(z, list) x = c.submit(assert_list, [1, 2, 3]) result = yield x._result() assert result x = c.submit(assert_list, [1, 2, 3], z=[4, 5, 6]) result = yield x._result() assert result x = c.submit(inc, 1) y = c.submit(inc, 2) z = c.submit(assert_list, [x, y]) result = yield z._result() assert result @gen_cluster(client=True) def test_map_quotes(c, s, a, b): def assert_list(x, z=[]): return isinstance(x, list) and isinstance(z, list) L = c.map(assert_list, [[1, 2, 3], [4]]) result = yield c._gather(L) assert all(result) L = c.map(assert_list, [[1, 2, 3], [4]], z=[10]) result = yield c._gather(L) assert all(result) L = c.map(assert_list, [[1, 2, 3], [4]], [[]] * 3) result = yield c._gather(L) assert all(result) @gen_cluster() def test_two_consecutive_clients_share_results(s, a, b): from random import randint c = Client((s.ip, s.port), start=False) yield c._start() x = c.submit(randint, 0, 1000, pure=True) xx = yield x._result() f = Client((s.ip, s.port), start=False) yield f._start() y = f.submit(randint, 0, 1000, pure=True) yy = yield y._result() assert xx == yy yield c._shutdown() yield f._shutdown() @gen_cluster(client=True) def test_submit_then_get_with_Future(c, s, a, b): x = c.submit(slowinc, 1) dsk = {'y': (inc, x)} result = yield c._get(dsk, 'y') assert result == 3 @gen_cluster(client=True) def test_aliases(c, s, a, b): x = c.submit(inc, 1) dsk = {'y': x} result = yield c._get(dsk, 'y') assert result == 2 @gen_cluster(client=True) def test__scatter(c, s, a, b): d = yield c._scatter({'y': 20}) assert isinstance(d['y'], Future) assert a.data.get('y') == 20 or b.data.get('y') == 20 assert (a.address in s.who_has['y'] or b.address in s.who_has['y']) assert s.who_has['y'] assert s.nbytes == {'y': sizeof(20)} yy = yield c._gather([d['y']]) assert yy == [20] [x] = yield c._scatter([10]) assert isinstance(x, Future) assert a.data.get(x.key) == 10 or b.data.get(x.key) == 10 xx = yield c._gather([x]) assert s.who_has[x.key] assert (a.address in s.who_has[x.key] or b.address in s.who_has[x.key]) assert s.nbytes == {'y': sizeof(20), x.key: sizeof(10)} assert xx == [10] z = c.submit(add, x, d['y']) # submit works on Future result = yield z._result() assert result == 10 + 20 result = yield c._gather([z, x]) assert result == [30, 10] @gen_cluster(client=True) def test__scatter_types(c, s, a, b): d = yield c._scatter({'x': 1}) assert isinstance(d, dict) assert list(d) == ['x'] for seq in [[1], (1,), {1}, frozenset([1])]: L = yield c._scatter(seq) assert isinstance(L, type(seq)) assert len(L) == 1 seq = yield c._scatter(range(5)) assert isinstance(seq, list) assert len(seq) == 5 @gen_cluster(client=True) def test_scatter_hash(c, s, a, b): [a] = yield c._scatter([1]) [b] = yield c._scatter([1]) assert a.key == b.key @gen_cluster(client=True) def test_get_releases_data(c, s, a, b): [x] = yield c._get({'x': (inc, 1)}, ['x']) import gc; gc.collect() assert c.refcount['x'] == 0 def test_global_clients(loop): assert not _global_client[0] with pytest.raises(ValueError): default_client() with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: assert _global_client == [c] assert default_client() is c with Client(('127.0.0.1', s['port']), loop=loop) as f: assert _global_client == [f] assert default_client() is f assert default_client(c) is c assert default_client(f) is f assert not _global_client[0] @gen_cluster(client=True) def test_exception_on_exception(c, s, a, b): x = c.submit(lambda: 1 / 0) y = c.submit(inc, x) with pytest.raises(ZeroDivisionError): yield y._result() z = c.submit(inc, y) with pytest.raises(ZeroDivisionError): yield z._result() @gen_cluster(client=True) def test_nbytes(c, s, a, b): [x] = yield c._scatter([1]) assert s.nbytes == {x.key: sizeof(1)} y = c.submit(inc, x) yield y._result() assert s.nbytes == {x.key: sizeof(1), y.key: sizeof(2)} @pytest.mark.skipif(not sys.platform.startswith('linux'), reason="Need 127.0.0.2 to mean localhost") @gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True) def test_nbytes_determines_worker(c, s, a, b): x = c.submit(identity, 1, workers=[a.ip]) y = c.submit(identity, tuple(range(100)), workers=[b.ip]) yield c._gather([x, y]) z = c.submit(lambda x, y: None, x, y) yield z._result() assert s.who_has[z.key] == {b.address} @gen_cluster(client=True) def test_if_intermediates_clear_on_error(c, s, a, b): x = delayed(div, pure=True)(1, 0) y = delayed(div, pure=True)(1, 2) z = delayed(add, pure=True)(x, y) f = c.compute(z) with pytest.raises(ZeroDivisionError): yield f._result() s.validate_state() assert not s.who_has @gen_cluster(client=True) def test_pragmatic_move_small_data_to_large_data(c, s, a, b): lists = c.map(lambda n: list(range(n)), [10] * 10, pure=False) sums = c.map(sum, lists) total = c.submit(sum, sums) def f(x, y): return None results = c.map(f, lists, [total] * 10) yield _wait([total]) yield _wait(results) assert sum(s.who_has[l.key] == s.who_has[r.key] for l, r in zip(lists, results)) >= 8 @gen_cluster(client=True) def test_get_with_non_list_key(c, s, a, b): dsk = {('x', 0): (inc, 1), 5: (inc, 2)} x = yield c._get(dsk, ('x', 0)) y = yield c._get(dsk, 5) assert x == 2 assert y == 3 @gen_cluster(client=True) def test_get_with_error(c, s, a, b): dsk = {'x': (div, 1, 0), 'y': (inc, 'x')} with pytest.raises(ZeroDivisionError): yield c._get(dsk, 'y') def test_get_with_error_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: dsk = {'x': (div, 1, 0), 'y': (inc, 'x')} with pytest.raises(ZeroDivisionError): c.get(dsk, 'y') @gen_cluster(client=True) def test_directed_scatter(c, s, a, b): yield c._scatter([1, 2, 3], workers=[a.address]) assert len(a.data) == 3 assert not b.data yield c._scatter([4, 5], workers=[b.name]) assert len(b.data) == 2 def test_directed_scatter_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: futures = c.scatter([1, 2, 3], workers=[('127.0.0.1', b['port'])]) has_what = sync(loop, c.scheduler.has_what) assert len(has_what['127.0.0.1:%d' % b['port']]) == len(futures) assert len(has_what['127.0.0.1:%d' % a['port']]) == 0 def test_iterator_scatter(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: aa = c.scatter([1,2,3]) assert [1,2,3] == c.gather(aa) g = (i for i in range(10)) futures = c.scatter(g) assert isinstance(futures, Iterator) a = next(futures) assert c.gather(a) == 0 futures = list(futures) assert len(futures) == 9 assert c.gather(futures) == [1, 2, 3, 4, 5, 6, 7, 8, 9] def test_queue_scatter(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as ee: from distributed.compatibility import Queue q = Queue() for d in range(10): q.put(d) futures = ee.scatter(q) assert isinstance(futures, Queue) a = futures.get() assert ee.gather(a) == 0 def test_queue_scatter_gather_maxsize(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: from distributed.compatibility import Queue q = Queue(maxsize=3) out = c.scatter(q, maxsize=10) assert out.maxsize == 10 local = c.gather(q) assert not local.maxsize q = Queue() out = c.scatter(q) assert not out.maxsize local = c.gather(out, maxsize=10) assert local.maxsize == 10 q = Queue(maxsize=3) out = c.scatter(q) assert not out.maxsize def test_queue_gather(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as ee: from distributed.compatibility import Queue q = Queue() qin = list(range(10)) for d in qin: q.put(d) futures = ee.scatter(q) assert isinstance(futures, Queue) ff = ee.gather(futures) assert isinstance(ff, Queue) qout = [] for f in range(10): qout.append(ff.get()) assert qout == qin def test_iterator_gather(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as ee: i_in = list(range(10)) g = (d for d in i_in) futures = ee.scatter(g) assert isinstance(futures, Iterator) ff = ee.gather(futures) assert isinstance(ff, Iterator) i_out = list(ff) assert i_out == i_in i_in = ['a', 'b', 'c', StopIteration('f'), StopIteration, 'd', 'c'] g = (d for d in i_in) futures = ee.scatter(g) ff = ee.gather(futures) i_out = list(ff) assert i_out[:3] == i_in[:3] # This is because StopIteration('f') != StopIteration('f') assert isinstance(i_out[3], StopIteration) assert i_out[3].args == i_in[3].args assert i_out[4:] == i_in[4:] @gen_cluster(client=True) def test_many_submits_spread_evenly(c, s, a, b): L = [c.submit(inc, i) for i in range(10)] yield _wait(L) assert a.data and b.data @gen_cluster(client=True) def test_traceback(c, s, a, b): x = c.submit(div, 1, 0) tb = yield x._traceback() if sys.version_info[0] >= 3: assert any('x / y' in line for line in pluck(3, traceback.extract_tb(tb))) @gen_cluster(client=True) def test_get_traceback(c, s, a, b): try: yield c._get({'x': (div, 1, 0)}, 'x') except ZeroDivisionError: exc_type, exc_value, exc_traceback = sys.exc_info() L = traceback.format_tb(exc_traceback) assert any('x / y' in line for line in L) @gen_cluster(client=True) def test_gather_traceback(c, s, a, b): x = c.submit(div, 1, 0) try: yield c._gather(x) except ZeroDivisionError: exc_type, exc_value, exc_traceback = sys.exc_info() L = traceback.format_tb(exc_traceback) assert any('x / y' in line for line in L) def test_traceback_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = c.submit(div, 1, 0) tb = x.traceback() if sys.version_info[0] >= 3: assert any('x / y' in line for line in concat(traceback.extract_tb(tb)) if isinstance(line, str)) y = c.submit(inc, x) tb2 = y.traceback() assert set(pluck(3, traceback.extract_tb(tb2))).issuperset( set(pluck(3, traceback.extract_tb(tb)))) z = c.submit(div, 1, 2) tb = z.traceback() assert tb is None @gen_cluster(client=True) def test_upload_file(c, s, a, b): def g(): import myfile return myfile.f() with tmp_text('myfile.py', 'def f():\n return 123') as fn: yield c._upload_file(fn) sleep(1) # TODO: why is this necessary? x = c.submit(g, pure=False) result = yield x._result() assert result == 123 with tmp_text('myfile.py', 'def f():\n return 456') as fn: yield c._upload_file(fn) y = c.submit(g, pure=False) result = yield y._result() assert result == 456 @gen_cluster(client=True) def test_upload_large_file(c, s, a, b): assert a.local_dir assert b.local_dir with tmp_text('myfile', 'abc') as fn: yield c._upload_large_file(fn, remote_filename='x') yield c._upload_large_file(fn) for w in [a, b]: assert os.path.exists(os.path.join(w.local_dir, 'x')) assert os.path.exists(os.path.join(w.local_dir, 'myfile')) with open(os.path.join(w.local_dir, 'x')) as f: assert f.read() == 'abc' def test_upload_file_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: def g(): import myfile return myfile.x with tmp_text('myfile.py', 'x = 123') as fn: c.upload_file(fn) x = c.submit(g) assert x.result() == 123 @gen_cluster(client=True) def test_upload_file_exception(c, s, a, b): with tmp_text('myfile.py', 'syntax-error!') as fn: with pytest.raises(SyntaxError): yield c._upload_file(fn) def test_upload_file_exception_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: with tmp_text('myfile.py', 'syntax-error!') as fn: with pytest.raises(SyntaxError): c.upload_file(fn) @pytest.mark.xfail @gen_cluster() def test_multiple_clients(s, a, b): a = Client((s.ip, s.port), start=False) yield a._start() b = Client((s.ip, s.port), start=False) yield b._start() x = a.submit(inc, 1) y = b.submit(inc, 2) assert x.client is a assert y.client is b xx = yield x._result() yy = yield y._result() assert xx == 2 assert yy == 3 z = a.submit(add, x, y) assert z.client is a zz = yield z._result() assert zz == 5 yield a._shutdown() yield b._shutdown() @gen_cluster(client=True) def test_async_compute(c, s, a, b): from dask.delayed import delayed x = delayed(1) y = delayed(inc)(x) z = delayed(dec)(x) [yy, zz, aa] = c.compute([y, z, 3], sync=False) assert isinstance(yy, Future) assert isinstance(zz, Future) assert aa == 3 result = yield c._gather([yy, zz]) assert result == [2, 0] assert isinstance(c.compute(y), Future) assert isinstance(c.compute([y]), (tuple, list)) @gen_cluster(client=True) def test_async_compute_with_scatter(c, s, a, b): d = yield c._scatter({('x', 1): 1, ('y', 1): 2}) x, y = d[('x', 1)], d[('y', 1)] from dask.delayed import delayed z = delayed(add)(delayed(inc)(x), delayed(inc)(y)) zz = c.compute(z) [result] = yield c._gather([zz]) assert result == 2 + 3 def test_sync_compute(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = delayed(1) y = delayed(inc)(x) z = delayed(dec)(x) yy, zz = c.compute([y, z], sync=True) assert (yy, zz) == (2, 0) @gen_cluster(client=True) def test_remote_scatter_gather(c, s, a, b): x, y, z = yield c._scatter([1, 2, 3]) assert x.key in a.data or x.key in b.data assert y.key in a.data or y.key in b.data assert z.key in a.data or z.key in b.data xx, yy, zz = yield c._gather([x, y, z]) assert (xx, yy, zz) == (1, 2, 3) @gen_cluster(timeout=1000, client=True) def test_remote_submit_on_Future(c, s, a, b): x = c.submit(lambda x: x + 1, 1) y = c.submit(lambda x: x + 1, x) result = yield y._result() assert result == 3 def test_start_is_idempotent(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: c.start() c.start() c.start() x = c.submit(inc, 1) assert x.result() == 2 @gen_cluster(client=True) def test_client_with_scheduler(c, s, a, b): assert s.ncores == {a.address: a.ncores, b.address: b.ncores} x = c.submit(inc, 1) y = c.submit(inc, 2) z = c.submit(add, x, y) result = yield x._result() assert result == 1 + 1 result = yield z._result() assert result == 1 + 1 + 1 + 2 A, B, C = yield c._scatter([1, 2, 3]) AA, BB, xx = yield c._gather([A, B, x]) assert (AA, BB, xx) == (1, 2, 2) result = yield c._get({'x': (inc, 1), 'y': (add, 'x', 10)}, 'y') assert result == 12 @pytest.mark.skipif(not sys.platform.startswith('linux'), reason="Need 127.0.0.2 to mean localhost") @gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True) def test_allow_restrictions(c, s, a, b): x = c.submit(inc, 1, workers=a.ip) yield x._result() assert s.who_has[x.key] == {a.address} assert not s.loose_restrictions x = c.submit(inc, 2, workers=a.ip, allow_other_workers=True) yield x._result() assert s.who_has[x.key] == {a.address} assert x.key in s.loose_restrictions L = c.map(inc, range(3, 13), workers=a.ip, allow_other_workers=True) yield _wait(L) assert all(s.who_has[f.key] == {a.address} for f in L) assert {f.key for f in L}.issubset(s.loose_restrictions) """ x = c.submit(inc, 14, workers='127.0.0.3') with ignoring(gen.TimeoutError): yield gen.with_timeout(timedelta(seconds=0.1), x._result()) assert False assert not s.who_has[x.key] assert x.key not in s.loose_restrictions """ x = c.submit(inc, 15, workers='127.0.0.3', allow_other_workers=True) yield x._result() assert s.who_has[x.key] assert x.key in s.loose_restrictions L = c.map(inc, range(15, 25), workers='127.0.0.3', allow_other_workers=True) yield _wait(L) assert all(s.who_has[f.key] for f in L) assert {f.key for f in L}.issubset(s.loose_restrictions) with pytest.raises(ValueError): c.submit(inc, 1, allow_other_workers=True) with pytest.raises(ValueError): c.map(inc, [1], allow_other_workers=True) with pytest.raises(TypeError): c.submit(inc, 20, workers='127.0.0.1', allow_other_workers='Hello!') with pytest.raises(TypeError): c.map(inc, [20], workers='127.0.0.1', allow_other_workers='Hello!') @pytest.mark.skipif('True', reason='because') def test_bad_address(): try: Client('123.123.123.123:1234', timeout=0.1) except (IOError, gen.TimeoutError) as e: assert "connect" in str(e).lower() try: Client('127.0.0.1:1234', timeout=0.1) except (IOError, gen.TimeoutError) as e: assert "connect" in str(e).lower() @gen_cluster(client=True) def test_long_error(c, s, a, b): def bad(x): raise ValueError('a' * 100000) x = c.submit(bad, 10) try: yield x._result() except ValueError as e: assert len(str(e)) < 100000 tb = yield x._traceback() assert all(len(line) < 100000 for line in concat(traceback.extract_tb(tb)) if isinstance(line, str)) @gen_cluster(client=True) def test_map_on_futures_with_kwargs(c, s, a, b): def f(x, y=10): return x + y futures = c.map(inc, range(10)) futures2 = c.map(f, futures, y=20) results = yield c._gather(futures2) assert results == [i + 1 + 20 for i in range(10)] future = c.submit(inc, 100) future2 = c.submit(f, future, y=200) result = yield future2._result() assert result == 100 + 1 + 200 class BadlySerializedObject(object): def __getstate__(self): return 1 def __setstate__(self, state): raise TypeError("hello!") class FatallySerializedObject(object): def __getstate__(self): return 1 def __setstate__(self, state): print("This should never have been deserialized, closing") import sys sys.exit(0) @gen_cluster(client=True) def test_badly_serialized_input(c, s, a, b): o = BadlySerializedObject() future = c.submit(inc, o) futures = c.map(inc, range(10)) L = yield c._gather(futures) assert list(L) == list(map(inc, range(10))) assert future.status == 'error' @pytest.mark.skipif('True', reason="") def test_badly_serialized_input_stderr(capsys, loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: o = BadlySerializedObject() future = c.submit(inc, o) start = time() while True: sleep(0.01) out, err = capsys.readouterr() if 'hello!' in err: break assert time() - start < 20 assert future.status == 'error' @gen_cluster(client=True) def test_repr(c, s, a, b): assert s.ip in str(c) assert str(s.port) in repr(c) @gen_cluster(client=True) def test_forget_simple(c, s, a, b): x = c.submit(inc, 1) y = c.submit(inc, 2) z = c.submit(add, x, y, workers=[a.ip], allow_other_workers=True) yield _wait([x, y, z]) assert not s.waiting_data[x.key] assert not s.waiting_data[y.key] assert set(s.tasks) == {x.key, y.key, z.key} s.client_releases_keys(keys=[x.key], client=c.id) assert x.key in s.tasks s.client_releases_keys(keys=[z.key], client=c.id) for coll in [s.tasks, s.dependencies, s.dependents, s.waiting, s.waiting_data, s.who_has, s.restrictions, s.loose_restrictions, s.released, s.priority, s.exceptions, s.who_wants, s.exceptions_blame, s.nbytes, s.task_state]: assert x.key not in coll assert z.key not in coll assert z.key not in s.dependents[y.key] s.client_releases_keys(keys=[y.key], client=c.id) assert not s.tasks @gen_cluster(client=True) def test_forget_complex(e, s, A, B): a, b, c, d = yield e._scatter(list(range(4))) ab = e.submit(add, a, b) cd = e.submit(add, c, d) ac = e.submit(add, a, c) acab = e.submit(add, ac, ab) yield _wait([a,b,c,d,ab,ac,cd,acab]) assert set(s.tasks) == {f.key for f in [ab,ac,cd,acab]} s.client_releases_keys(keys=[ab.key], client=e.id) assert set(s.tasks) == {f.key for f in [ab,ac,cd,acab]} s.client_releases_keys(keys=[b.key], client=e.id) assert set(s.tasks) == {f.key for f in [ac,cd,acab]} s.client_releases_keys(keys=[acab.key], client=e.id) assert set(s.tasks) == {f.key for f in [ac,cd]} assert b.key not in s.who_has start = time() while b.key in A.data or b.key in B.data: yield gen.sleep(0.01) assert time() < start + 10 s.client_releases_keys(keys=[ac.key], client=e.id) assert set(s.tasks) == {f.key for f in [cd]} @gen_cluster(client=True) def test_forget_in_flight(e, s, A, B): delayed2 = partial(delayed, pure=True) a, b, c, d = [delayed2(slowinc)(i) for i in range(4)] ab = delayed2(slowadd)(a, b) cd = delayed2(slowadd)(c, d) ac = delayed2(slowadd)(a, c) acab = delayed2(slowadd)(ac, ab) x, y = e.compute([ac, acab]) s.validate_state() for i in range(5): yield gen.sleep(0.01) s.validate_state() s.client_releases_keys(keys=[y.key], client=e.id) s.validate_state() for k in [acab.key, ab.key, b.key]: assert k not in s.tasks assert k not in s.waiting assert k not in s.who_has @gen_cluster(client=True) def test_forget_errors(c, s, a, b): x = c.submit(div, 1, 0) y = c.submit(inc, x) z = c.submit(inc, y) yield _wait([y]) assert x.key in s.exceptions assert x.key in s.exceptions_blame assert y.key in s.exceptions_blame assert z.key in s.exceptions_blame s.client_releases_keys(keys=[z.key], client=c.id) assert x.key in s.exceptions assert x.key in s.exceptions_blame assert y.key in s.exceptions_blame assert z.key not in s.exceptions_blame s.client_releases_keys(keys=[x.key], client=c.id) assert x.key in s.exceptions assert x.key in s.exceptions_blame assert y.key in s.exceptions_blame assert z.key not in s.exceptions_blame s.client_releases_keys(keys=[y.key], client=c.id) assert x.key not in s.exceptions assert x.key not in s.exceptions_blame assert y.key not in s.exceptions_blame assert z.key not in s.exceptions_blame def test_repr_sync(loop): with cluster(nworkers=3) as (s, [a, b, c]): with Client(('127.0.0.1', s['port']), loop=loop) as c: s = str(c) r = repr(c) assert c.scheduler.ip in s assert str(c.scheduler.port) in r assert str(3) in s # nworkers assert 'cores' in s @gen_cluster(client=True) def test_waiting_data(c, s, a, b): x = c.submit(inc, 1) y = c.submit(inc, 2) z = c.submit(add, x, y, workers=[a.ip], allow_other_workers=True) yield _wait([x, y, z]) assert x.key not in s.waiting_data[x.key] assert y.key not in s.waiting_data[y.key] assert not s.waiting_data[x.key] assert not s.waiting_data[y.key] @gen_cluster() def test_multi_client(s, a, b): c = Client((s.ip, s.port), start=False) yield c._start() f = Client((s.ip, s.port), start=False) yield f._start() assert set(s.streams) == {c.id, f.id} x = c.submit(inc, 1) y = f.submit(inc, 2) y2 = c.submit(inc, 2) assert y.key == y2.key yield _wait([x, y]) assert s.wants_what == {c.id: {x.key, y.key}, f.id: {y.key}} assert s.who_wants == {x.key: {c.id}, y.key: {c.id, f.id}} yield c._shutdown() start = time() while c.id in s.wants_what: yield gen.sleep(0.01) assert time() < start + 5 assert c.id not in s.wants_what assert c.id not in s.who_wants[y.key] assert x.key not in s.who_wants yield f._shutdown() assert not s.tasks def long_running_client_connection(ip, port): c = Client((ip, port)) x = c.submit(lambda x: x + 1, 10) x.result() sleep(100) @gen_cluster() def test_cleanup_after_broken_client_connection(s, a, b): proc = Process(target=long_running_client_connection, args=(s.ip, s.port)) proc.daemon = True proc.start() start = time() while not s.tasks: yield gen.sleep(0.01) assert time() < start + 5 proc.terminate() start = time() while s.tasks: yield gen.sleep(0.01) assert time() < start + 5 @gen_cluster() def test_multi_garbage_collection(s, a, b): c = Client((s.ip, s.port), start=False) yield c._start() f = Client((s.ip, s.port), start=False) yield f._start() x = c.submit(inc, 1) y = f.submit(inc, 2) y2 = c.submit(inc, 2) assert y.key == y2.key yield _wait([x, y]) x.__del__() start = time() while x.key in a.data or x.key in b.data: yield gen.sleep(0.01) assert time() < start + 5 assert s.wants_what == {c.id: {y.key}, f.id: {y.key}} assert s.who_wants == {y.key: {c.id, f.id}} y.__del__() start = time() while x.key in s.wants_what[f.id]: yield gen.sleep(0.01) assert time() < start + 5 yield gen.sleep(0.1) assert y.key in a.data or y.key in b.data assert s.wants_what == {c.id: {y.key}, f.id: set()} assert s.who_wants == {y.key: {c.id}} y2.__del__() start = time() while y.key in a.data or y.key in b.data: yield gen.sleep(0.01) assert time() < start + 5 assert not any(v for v in s.wants_what.values()) assert not s.who_wants yield c._shutdown() yield f._shutdown() @gen_cluster(client=True) def test__broadcast(c, s, a, b): x, y = yield c._scatter([1, 2], broadcast=True) assert a.data == b.data == {x.key: 1, y.key: 2} @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4) def test__broadcast_integer(c, s, *workers): x, y = yield c._scatter([1, 2], broadcast=2) assert len(s.who_has[x.key]) == 2 assert len(s.who_has[y.key]) == 2 @gen_cluster(client=True) def test__broadcast_dict(c, s, a, b): d = yield c._scatter({'x': 1}, broadcast=True) assert a.data == b.data == {'x': 1} def test_broadcast(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x, y = c.scatter([1, 2], broadcast=True) has_what = sync(c.loop, c.scheduler.has_what) assert {k: set(v) for k, v in has_what.items()} == { '127.0.0.1:%d' % a['port']: {x.key, y.key}, '127.0.0.1:%d' % b['port']: {x.key, y.key}} [z] = c.scatter([3], broadcast=True, workers=['127.0.0.1:%d' % a['port']]) has_what = sync(c.loop, c.scheduler.has_what) assert {k: set(v) for k, v in has_what.items()} == { '127.0.0.1:%d' % a['port']: {x.key, y.key, z.key}, '127.0.0.1:%d' % b['port']: {x.key, y.key}} @gen_cluster(client=True) def test__cancel(c, s, a, b): x = c.submit(slowinc, 1) y = c.submit(slowinc, x) while y.key not in s.tasks: yield gen.sleep(0.01) yield c._cancel([x]) assert x.cancelled() assert 'cancel' in str(x) s.validate_state() start = time() while not y.cancelled(): yield gen.sleep(0.01) assert time() < start + 5 assert not s.tasks assert not s.who_has s.validate_state() @gen_cluster(client=True) def test__cancel_tuple_key(c, s, a, b): x = c.submit(inc, 1, key=('x', 0, 1)) result = yield x._result() yield c._cancel(x) with pytest.raises(CancelledError): yield x._result() @gen_cluster() def test__cancel_multi_client(s, a, b): c = Client((s.ip, s.port), start=False) yield c._start() f = Client((s.ip, s.port), start=False) yield f._start() x = c.submit(slowinc, 1) y = f.submit(slowinc, 1) assert x.key == y.key yield c._cancel([x]) assert x.cancelled() assert not y.cancelled() start = time() while y.key not in s.tasks: yield gen.sleep(0.01) assert time() < start + 5 out = yield y._result() assert out == 2 with pytest.raises(CancelledError): yield x._result() yield c._shutdown() yield f._shutdown() @gen_cluster(client=True) def test__cancel_collection(c, s, a, b): import dask.bag as db L = c.map(double, [[1], [2], [3]]) x = db.Bag({('b', i): f for i, f in enumerate(L)}, 'b', 3) yield c._cancel(x) yield c._cancel([x]) assert all(f.cancelled() for f in L) assert not s.tasks assert not s.who_has def test_cancel(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = c.submit(slowinc, 1, key='x') y = c.submit(slowinc, x, key='y') z = c.submit(slowinc, y, key='z') c.cancel([y]) start = time() while not z.cancelled(): sleep(0.01) assert time() < start + 5 assert x.result() == 2 z.cancel() assert z.cancelled() @gen_cluster(client=True) def test_future_type(c, s, a, b): x = c.submit(inc, 1) yield _wait([x]) assert x.type == int assert 'int' in str(x) @gen_cluster(client=True) def test_traceback_clean(c, s, a, b): x = c.submit(div, 1, 0) try: yield x._result() except Exception as e: f = e exc_type, exc_value, tb = sys.exc_info() while tb: assert 'scheduler' not in tb.tb_frame.f_code.co_filename assert 'worker' not in tb.tb_frame.f_code.co_filename tb = tb.tb_next @gen_cluster(client=True) def test_map_queue(c, s, a, b): from distributed.compatibility import Queue, isqueue q_1 = Queue(maxsize=2) q_2 = c.map(inc, q_1) assert isqueue(q_2) assert not q_2.maxsize q_3 = c.map(double, q_2, maxsize=3) assert isqueue(q_3) assert q_3.maxsize == 3 q_4 = yield c._gather(q_3) assert isqueue(q_4) q_1.put(1) f = q_4.get() assert isinstance(f, Future) result = yield f._result() assert result == (1 + 1) * 2 @gen_cluster(client=True) def test_map_iterator_with_return(c, s, a, b): def g(): yield 1 yield 2 raise StopIteration(3) # py2.7 compat. f1 = c.map(lambda x: x, g()) assert isinstance(f1, Iterator) start = time() # ensure that we compute eagerly while not s.tasks: yield gen.sleep(0.01) assert time() < start + 5 g1 = g() try: while True: f = next(f1) n = yield f._result() assert n == next(g1) except StopIteration as e: with pytest.raises(StopIteration) as exc_info: next(g1) assert e.args == exc_info.value.args @gen_cluster(client=True) def test_map_iterator(c, s, a, b): x = iter([1, 2, 3]) y = iter([10, 20, 30]) f1 = c.map(add, x, y) assert isinstance(f1, Iterator) start = time() # ensure that we compute eagerly while not s.tasks: yield gen.sleep(0.01) assert time() < start + 5 f2 = c.map(double, f1) assert isinstance(f2, Iterator) future = next(f2) result = yield future._result() assert result == (1 + 10) * 2 futures = list(f2) results = [] for f in futures: r = yield f._result() results.append(r) assert results == [(2 + 20) * 2, (3 + 30) * 2] items = enumerate(range(10)) futures = c.map(lambda x: x, items) assert isinstance(futures, Iterator) result = yield next(futures)._result() assert result == (0, 0) futures_l = list(futures) results = [] for f in futures_l: r = yield f._result() results.append(r) assert results == [(i, i) for i in range(1,10)] @gen_cluster(client=True) def test_map_infinite_iterators(c, s, a, b): futures = c.map(add, [1, 2], itertools.repeat(10)) assert len(futures) == 2 def test_map_iterator_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: items = enumerate(range(10)) futures = c.map(lambda x: x, items) next(futures).result() == (0, 0) @gen_cluster(client=True) def test_map_differnet_lengths(c, s, a, b): assert len(c.map(add, [1, 2], [1, 2, 3])) == 2 def test_Future_exception_sync_2(loop, capsys): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: ensure_default_get(c) ensure_default_get(c) ensure_default_get(c) ensure_default_get(c) assert _globals['get'] == c.get out, err = capsys.readouterr() assert len(out.strip().split('\n')) == 1 assert _globals.get('get') != c.get @gen_cluster(timeout=60, client=True) def test_async_persist(c, s, a, b): from dask.delayed import delayed, Delayed x = delayed(1) y = delayed(inc)(x) z = delayed(dec)(x) w = delayed(add)(y, z) yy, ww = c.persist([y, w]) assert type(yy) == type(y) assert type(ww) == type(w) assert len(yy.dask) == 1 assert len(ww.dask) == 1 assert len(w.dask) > 1 assert y._keys() == yy._keys() assert w._keys() == ww._keys() while y.key not in s.tasks and w.key not in s.tasks: yield gen.sleep(0.01) assert s.who_wants[y.key] == {c.id} assert s.who_wants[w.key] == {c.id} yyf, wwf = c.compute([yy, ww]) yyy, www = yield c._gather([yyf, wwf]) assert yyy == inc(1) assert www == add(inc(1), dec(1)) assert isinstance(c.persist(y), Delayed) assert isinstance(c.persist([y]), (list, tuple)) @gen_cluster(client=True) def test__persist(c, s, a, b): pytest.importorskip('dask.array') import dask.array as da x = da.ones((10, 10), chunks=(5, 10)) y = 2 * (x + 1) assert len(y.dask) == 6 yy = c.persist(y) assert len(y.dask) == 6 assert len(yy.dask) == 2 assert all(isinstance(v, Future) for v in yy.dask.values()) assert yy._keys() == y._keys() g, h = c.compute([y, yy]) gg, hh = yield c._gather([g, h]) assert (gg == hh).all() def test_persist(loop): pytest.importorskip('dask.array') import dask.array as da with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = da.ones((10, 10), chunks=(5, 10)) y = 2 * (x + 1) assert len(y.dask) == 6 yy = c.persist(y) assert len(y.dask) == 6 assert len(yy.dask) == 2 assert all(isinstance(v, Future) for v in yy.dask.values()) assert yy._keys() == y._keys() zz = yy.compute(get=c.get) z = y.compute(get=c.get) assert (zz == z).all() @gen_cluster(timeout=60, client=True) def test_long_traceback(c, s, a, b): from distributed.protocol.pickle import dumps n = sys.getrecursionlimit() sys.setrecursionlimit(500) try: x = c.submit(deep, 1000) yield _wait([x]) assert len(dumps(c.futures[x.key]['traceback'])) < 10000 assert isinstance(c.futures[x.key]['exception'], RuntimeError) finally: sys.setrecursionlimit(n) @gen_cluster(client=True) def test_wait_on_collections(c, s, a, b): import dask.bag as db L = c.map(double, [[1], [2], [3]]) x = db.Bag({('b', i): f for i, f in enumerate(L)}, 'b', 3) yield _wait(x) assert all(f.key in a.data or f.key in b.data for f in L) @gen_cluster(client=True) def test_futures_of(c, s, a, b): x, y, z = c.map(inc, [1, 2, 3]) assert set(futures_of(0)) == set() assert set(futures_of(x)) == {x} assert set(futures_of([x, y, z])) == {x, y, z} assert set(futures_of([x, [y], [[z]]])) == {x, y, z} assert set(futures_of({'x': x, 'y': [y]})) == {x, y} import dask.bag as db b = db.Bag({('b', i): f for i, f in enumerate([x, y, z])}, 'b', 3) assert set(futures_of(b)) == {x, y, z} @gen_cluster(client=True) def test_futures_of_cancelled_raises(c, s, a, b): x = c.submit(inc, 1) yield c._cancel([x]) with pytest.raises(CancelledError): yield x._result() with pytest.raises(CancelledError): yield c._get({'x': (inc, x), 'y': (inc, 2)}, ['x', 'y']) with pytest.raises(CancelledError): c.submit(inc, x) with pytest.raises(CancelledError): c.submit(add, 1, y=x) with pytest.raises(CancelledError): c.map(add, [1], y=x) assert 'y' not in s.tasks @gen_cluster(ncores=[('127.0.0.1', 1)], client=True) def test_dont_delete_recomputed_results(c, s, w): x = c.submit(inc, 1) # compute first time yield _wait([x]) x.__del__() # trigger garbage collection xx = c.submit(inc, 1) # compute second time start = time() while xx.key not in w.data: # data shows up yield gen.sleep(0.01) assert time() < start + 1 while time() < start + (s.delete_interval + 100) / 1000: # and stays assert xx.key in w.data yield gen.sleep(0.01) @gen_cluster(ncores=[], client=True) def test_fatally_serialized_input(c, s): o = FatallySerializedObject() future = c.submit(inc, o) while not s.tasks: yield gen.sleep(0.01) @gen_cluster(client=True) def test_balance_tasks_by_stacks(c, s, a, b): x = c.submit(inc, 1) yield _wait(x) y = c.submit(inc, 2) yield _wait(y) assert len(a.data) == len(b.data) == 1 @gen_cluster(client=True) def test_run(c, s, a, b): results = yield c._run(inc, 1) assert results == {a.address: 2, b.address: 2} results = yield c._run(inc, 1, workers=[a.address]) assert results == {a.address: 2} results = yield c._run(inc, 1, workers=[]) assert results == {} def test_run_sync(loop): def func(x, y=10): return x + y with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: result = c.run(func, 1, y=2) assert result == {'127.0.0.1:%d' % a['port']: 3, '127.0.0.1:%d' % b['port']: 3} result = c.run(func, 1, y=2, workers=['127.0.0.1:%d' % a['port']]) assert result == {'127.0.0.1:%d' % a['port']: 3} def test_run_exception(loop): def raise_exception(exc_type, exc_msg): raise exc_type(exc_msg) with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: for exc_type in [ValueError, RuntimeError]: with pytest.raises(exc_type) as excinfo: c.run(raise_exception, exc_type, 'informative message') assert 'informative message' in str(excinfo.value) def test_diagnostic_ui(loop): with cluster() as (s, [a, b]): a_addr = '127.0.0.1:%d' % a['port'] b_addr = '127.0.0.1:%d' % b['port'] with Client(('127.0.0.1', s['port']), loop=loop) as c: d = c.ncores() assert d == {a_addr: 1, b_addr: 1} d = c.ncores([a_addr]) assert d == {a_addr: 1} d = c.ncores(a_addr) assert d == {a_addr: 1} d = c.ncores(('127.0.0.1', a['port'])) assert d == {a_addr: 1} x = c.submit(inc, 1) y = c.submit(inc, 2) z = c.submit(inc, 3) wait([x, y, z]) d = c.who_has() assert set(d) == {x.key, y.key, z.key} assert all(w in [a_addr, b_addr] for v in d.values() for w in v) assert all(d.values()) d = c.who_has([x, y]) assert set(d) == {x.key, y.key} d = c.who_has(x) assert set(d) == {x.key} d = c.has_what() assert set(d) == {a_addr, b_addr} assert all(k in [x.key, y.key, z.key] for v in d.values() for k in v) d = c.has_what([a_addr]) assert set(d) == {a_addr} d = c.has_what(a_addr) assert set(d) == {a_addr} d = c.has_what(('127.0.0.1', a['port'])) assert set(d) == {a_addr} def test_diagnostic_nbytes_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: incs = c.map(inc, [1, 2, 3]) doubles = c.map(double, [1, 2, 3]) wait(incs + doubles) assert c.nbytes(summary=False) == {k.key: sizeof(1) for k in incs + doubles} assert c.nbytes(summary=True) == {'inc': sizeof(1) * 3, 'double': sizeof(1) * 3} @gen_cluster(client=True) def test_diagnostic_nbytes(c, s, a, b): incs = c.map(inc, [1, 2, 3]) doubles = c.map(double, [1, 2, 3]) yield _wait(incs + doubles) assert s.get_nbytes(summary=False) == {k.key: sizeof(1) for k in incs + doubles} assert s.get_nbytes(summary=True) == {'inc': sizeof(1) * 3, 'double': sizeof(1) * 3} @gen_test() def test_worker_aliases(): s = Scheduler(validate=True) s.start(0) a = Worker(s.ip, s.port, name='alice') b = Worker(s.ip, s.port, name='bob') yield [a._start(), b._start()] c = Client((s.ip, s.port), start=False) yield c._start() L = c.map(inc, range(10), workers='alice') yield _wait(L) assert len(a.data) == 10 assert len(b.data) == 0 yield c._shutdown() yield [a._close(), b._close()] yield s.close() def test_persist_get_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: dadd = delayed(add) x, y = delayed(1), delayed(2) xx = delayed(add)(x, x) yy = delayed(add)(y, y) xxyy = delayed(add)(xx, yy) xxyy2 = c.persist(xxyy) xxyy3 = delayed(add)(xxyy2, 10) assert xxyy3.compute(get=c.get) == ((1+1) + (2+2)) + 10 @gen_cluster(client=True) def test_persist_get(c, s, a, b): dadd = delayed(add) x, y = delayed(1), delayed(2) xx = delayed(add)(x, x) yy = delayed(add)(y, y) xxyy = delayed(add)(xx, yy) xxyy2 = c.persist(xxyy) xxyy3 = delayed(add)(xxyy2, 10) yield gen.sleep(0.5) result = yield c._get(xxyy3.dask, xxyy3._keys()) assert result[0] == ((1+1) + (2+2)) + 10 result = yield c.compute(xxyy3)._result() assert result == ((1+1) + (2+2)) + 10 result = yield c.compute(xxyy3)._result() assert result == ((1+1) + (2+2)) + 10 result = yield c.compute(xxyy3)._result() assert result == ((1+1) + (2+2)) + 10 @pytest.mark.skipif(sys.platform.startswith('win'), reason="num_fds not supported on windows") def test_client_num_fds(loop): psutil = pytest.importorskip('psutil') with cluster() as (s, [a, b]): proc = psutil.Process() before = proc.num_fds() with Client(('127.0.0.1', s['port']), loop=loop) as c: during = proc.num_fds() after = proc.num_fds() assert before >= after @gen_cluster() def test_startup_shutdown_startup(s, a, b): c = Client((s.ip, s.port), start=False) yield c._start() yield c._shutdown() c = Client((s.ip, s.port), start=False) yield c._start() yield c._shutdown() def test_startup_shutdown_startup_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: pass sleep(0.1) with Client(('127.0.0.1', s['port']), loop=loop) as c: pass with Client(('127.0.0.1', s['port'])) as c: pass sleep(0.1) with Client(('127.0.0.1', s['port'])) as c: pass @gen_cluster(client=True) def test_badly_serialized_exceptions(c, s, a, b): def f(): class BadlySerializedException(Exception): def __reduce__(self): raise TypeError() raise BadlySerializedException('hello world') x = c.submit(f) try: result = yield x._result() except Exception as e: assert 'hello world' in str(e) else: assert False @gen_cluster(client=True) def test_rebalance(c, s, a, b): x, y = yield c._scatter([1, 2], workers=[a.address]) assert len(a.data) == 2 assert len(b.data) == 0 s.validate_state() yield c._rebalance() s.validate_state() assert len(b.data) == 1 assert s.has_what[b.address] == set(b.data) assert b.address in s.who_has[x.key] or b.address in s.who_has[y.key] assert len(a.data) == 1 assert s.has_what[a.address] == set(a.data) assert (a.address not in s.who_has[x.key] or a.address not in s.who_has[y.key]) @gen_cluster(ncores=[('127.0.0.1', 1)] * 4, client=True) def test_rebalance_workers(e, s, a, b, c, d): w, x, y, z = yield e._scatter([1, 2, 3, 4], workers=[a.address]) assert len(a.data) == 4 assert len(b.data) == 0 assert len(c.data) == 0 assert len(d.data) == 0 yield e._rebalance([x, y], workers=[a.address, c.address]) assert len(a.data) == 3 assert len(b.data) == 0 assert len(c.data) == 1 assert len(d.data) == 0 assert c.data == {x.key: 2} or c.data == {y.key: 3} yield e._rebalance() assert len(a.data) == 1 assert len(b.data) == 1 assert len(c.data) == 1 assert len(d.data) == 1 s.validate_state() @gen_cluster(client=True) def test_rebalance_execution(c, s, a, b): futures = c.map(inc, range(10), workers=a.address) yield c._rebalance(futures) assert len(a.data) == len(b.data) == 5 s.validate_state() def test_rebalance_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: futures = c.map(inc, range(10), workers=[('127.0.0.1', a['port'])]) c.rebalance(futures) has_what = c.has_what() assert len(has_what) == 2 assert list(valmap(len, has_what).values()) == [5, 5] @gen_cluster(client=True) def test_receive_lost_key(c, s, a, b): x = c.submit(inc, 1, workers=[a.address]) result = yield x._result() yield a._close() start = time() while x.status == 'finished': assert time() < start + 5 yield gen.sleep(0.01) @pytest.mark.skipif(not sys.platform.startswith('linux'), reason="Need 127.0.0.2 to mean localhost") @gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True) def test_unrunnable_task_runs(c, s, a, b): x = c.submit(inc, 1, workers=[a.ip]) result = yield x._result() yield a._close() start = time() while x.status == 'finished': assert time() < start + 5 yield gen.sleep(0.01) assert x.key in s.unrunnable assert s.task_state[x.key] == 'no-worker' w = Worker(s.ip, s.port, ip=a.ip, loop=s.loop) yield w._start() start = time() while x.status != 'finished': assert time() < start + 2 yield gen.sleep(0.01) assert x.key not in s.unrunnable result = yield x._result() assert result == 2 yield w._close() @gen_cluster(client=True, ncores=[]) def test_add_worker_after_tasks(c, s): futures = c.map(inc, range(10)) n = Nanny(s.ip, s.port, ncores=2, loop=s.loop) n.start(0) result = yield c._gather(futures) yield n._close() @pytest.mark.skipif(not sys.platform.startswith('linux'), reason="Need 127.0.0.2 to mean localhost") @gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True) def test_workers_register_indirect_data(c, s, a, b): [x] = yield c._scatter([1], workers=a.address) y = c.submit(inc, x, workers=b.ip) yield y._result() assert b.data[x.key] == 1 assert s.who_has[x.key] == {a.address, b.address} assert s.has_what[b.address] == {x.key, y.key} s.validate_state() @gen_cluster(client=True) def test_submit_on_cancelled_future(c, s, a, b): x = c.submit(inc, 1) yield x._result() yield c._cancel(x) with pytest.raises(CancelledError): y = c.submit(inc, x) @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10) def test_replicate(c, s, *workers): [a, b] = yield c._scatter([1, 2]) yield s.replicate(keys=[a.key, b.key], n=5) s.validate_state() assert len(s.who_has[a.key]) == 5 assert len(s.who_has[b.key]) == 5 assert sum(a.key in w.data for w in workers) == 5 assert sum(b.key in w.data for w in workers) == 5 @gen_cluster(client=True) def test_replicate_tuple_keys(c, s, a, b): x = delayed(inc)(1, dask_key_name=('x', 1)) f = c.persist(x) yield c._replicate(f, n=5) s.validate_state() assert a.data and b.data yield c._rebalance(f) s.validate_state() @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10) def test_replicate_workers(c, s, *workers): [a, b] = yield c._scatter([1, 2], workers=[workers[0].address]) yield s.replicate(keys=[a.key, b.key], n=5, workers=[w.address for w in workers[:5]]) assert len(s.who_has[a.key]) == 5 assert len(s.who_has[b.key]) == 5 assert sum(a.key in w.data for w in workers[:5]) == 5 assert sum(b.key in w.data for w in workers[:5]) == 5 assert sum(a.key in w.data for w in workers[5:]) == 0 assert sum(b.key in w.data for w in workers[5:]) == 0 yield s.replicate(keys=[a.key, b.key], n=1) assert len(s.who_has[a.key]) == 1 assert len(s.who_has[b.key]) == 1 assert sum(a.key in w.data for w in workers) == 1 assert sum(b.key in w.data for w in workers) == 1 s.validate_state() yield s.replicate(keys=[a.key, b.key], n=None) # all assert len(s.who_has[a.key]) == 10 assert len(s.who_has[b.key]) == 10 s.validate_state() yield s.replicate(keys=[a.key, b.key], n=1, workers=[w.address for w in workers[:5]]) assert sum(a.key in w.data for w in workers[:5]) == 1 assert sum(b.key in w.data for w in workers[:5]) == 1 assert sum(a.key in w.data for w in workers[5:]) == 5 assert sum(b.key in w.data for w in workers[5:]) == 5 s.validate_state() class CountSerialization(object): def __init__(self): self.n = 0 def __setstate__(self, n): self.n = n + 1 def __getstate__(self): return self.n @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10) def test_replicate_tree_branching(c, s, *workers): obj = CountSerialization() [future] = yield c._scatter([obj]) yield s.replicate(keys=[future.key], n=10) max_count = max(w.data[future.key].n for w in workers) assert max_count > 1 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10) def test_client_replicate(c, s, *workers): x = c.submit(inc, 1) y = c.submit(inc, 2) yield c._replicate([x, y], n=5) assert len(s.who_has[x.key]) == 5 assert len(s.who_has[y.key]) == 5 yield c._replicate([x, y], n=3) assert len(s.who_has[x.key]) == 3 assert len(s.who_has[y.key]) == 3 yield c._replicate([x, y]) s.validate_state() assert len(s.who_has[x.key]) == 10 assert len(s.who_has[y.key]) == 10 @pytest.mark.skipif(not sys.platform.startswith('linux'), reason="Need 127.0.0.2 to mean localhost") @gen_cluster(client=True, ncores=[('127.0.0.1', 1), ('127.0.0.2', 1), ('127.0.0.2', 1)], timeout=None) def test_client_replicate_host(e, s, a, b, c): x = e.submit(inc, 1, workers='127.0.0.2') yield _wait([x]) assert (s.who_has[x.key] == {b.address} or s.who_has[x.key] == {c.address}) yield e._replicate([x], workers=['127.0.0.2']) assert s.who_has[x.key] == {b.address, c.address} yield e._replicate([x], workers=['127.0.0.1']) assert s.who_has[x.key] == {a.address, b.address, c.address} def test_client_replicate_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = c.submit(inc, 1) y = c.submit(inc, 2) c.replicate([x, y], n=2) who_has = c.who_has() assert len(who_has[x.key]) == len(who_has[y.key]) == 2 with pytest.raises(ValueError): c.replicate([x], n=0) assert y.result() == 3 @pytest.mark.skipif(sys.platform.startswith('win'), reason="Windows timer too coarse-grained") @gen_cluster(client=True, ncores=[('127.0.0.1', 4)] * 1) def test_task_load_adapts_quickly(c, s, a): future = c.submit(slowinc, 1, delay=0.2) # slow yield _wait(future) assert 0.15 < s.task_duration['slowinc'] < 0.4 futures = c.map(slowinc, range(10), delay=0) # very fast yield _wait(futures) assert 0 < s.task_duration['slowinc'] < 0.1 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2) def test_even_load_after_fast_functions(c, s, a, b): x = c.submit(inc, 1, workers=a.address) # very fast y = c.submit(inc, 2, workers=b.address) # very fast yield _wait([x, y]) futures = c.map(inc, range(2, 11)) yield _wait(futures) assert abs(len(a.data) - len(b.data)) <= 2 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2) def test_even_load_on_startup(c, s, a, b): x, y = c.map(inc, [1, 2]) yield _wait([x, y]) assert len(a.data) == len(b.data) == 1 @gen_cluster(client=True, ncores=[('127.0.0.1', 2)] * 2) def test_contiguous_load(c, s, a, b): w, x, y, z = c.map(inc, [1, 2, 3, 4]) yield _wait([w, x, y, z]) groups = [set(a.data), set(b.data)] assert {w.key, x.key} in groups assert {y.key, z.key} in groups @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4) def test_balanced_with_submit(c, s, *workers): L = [c.submit(slowinc, i) for i in range(4)] yield _wait(L) for w in workers: assert len(w.data) == 1 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4) def test_balanced_with_submit_and_resident_data(c, s, *workers): [x] = yield c._scatter([10], broadcast=True) L = [c.submit(slowinc, x, pure=False) for i in range(4)] yield _wait(L) for w in workers: assert len(w.data) == 2 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2) def test_balanced_with_submit_and_resident_data(c, s, a, b): slow1 = c.submit(slowinc, 1, delay=0.2, workers=a.address) # learn slow slow2 = c.submit(slowinc, 2, delay=0.2, workers=b.address) yield _wait([slow1, slow2]) aa = c.map(inc, range(100), pure=False, workers=a.address) # learn fast bb = c.map(inc, range(100), pure=False, workers=b.address) yield _wait(aa + bb) cc = c.map(slowinc, range(10), delay=0.1) while not all(c.done() for c in cc): assert all(len(p) < 3 for p in s.processing.values()) yield gen.sleep(0.01) @gen_cluster(client=True, ncores=[('127.0.0.1', 20)] * 2) def test_scheduler_saturates_cores(c, s, a, b): for delay in [0, 0.01, 0.1]: futures = c.map(slowinc, range(100), delay=delay) futures = c.map(slowinc, futures, delay=delay / 10) while not s.tasks or s.ready: if s.tasks: assert all(len(p) >= 20 for p in s.processing.values()) yield gen.sleep(0.01) @gen_cluster(client=True, ncores=[('127.0.0.1', 20)] * 2) def test_scheduler_saturates_cores_stacks(c, s, a, b): for delay in [0, 0.01, 0.1]: x = c.map(slowinc, range(100), delay=delay, pure=False, workers=a.address) y = c.map(slowinc, range(100), delay=delay, pure=False, workers=b.address) while not s.tasks or any(s.stacks.values()): if s.tasks: for w, stack in s.stacks.items(): if stack: assert len(s.processing[w]) >= s.ncores[w] yield gen.sleep(0.01) @gen_cluster(client=True, ncores=[('127.0.0.1', 20)] * 2) def test_scheduler_saturates_cores_random(c, s, a, b): for delay in [0, 0.01, 0.1]: futures = c.map(randominc, range(100), scale=0.1) while not s.tasks or s.ready: if s.tasks: assert all(len(p) >= 20 for p in s.processing.values()) yield gen.sleep(0.01) @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4) def test_cancel_clears_processing(c, s, *workers): da = pytest.importorskip('dask.array') x = c.submit(slowinc, 1, delay=0.2) while not s.tasks: yield gen.sleep(0.01) yield c._cancel(x) start = time() while any(v for v in s.processing.values()): assert time() < start + 0.2 yield gen.sleep(0.01) s.validate_state() def test_default_get(loop): with cluster() as (s, [a, b]): pre_get = _globals.get('get') pre_shuffle = _globals.get('shuffle') with Client(('127.0.0.1', s['port']), loop=loop, set_as_default=True) as c: assert _globals['get'] == c.get assert _globals['shuffle'] == 'tasks' assert _globals['get'] is pre_get assert _globals['shuffle'] == pre_shuffle c = Client(('127.0.0.1', s['port']), loop=loop, set_as_default=False) assert _globals['get'] is pre_get assert _globals['shuffle'] == pre_shuffle c.shutdown() c = Client(('127.0.0.1', s['port']), loop=loop, set_as_default=True) assert _globals['shuffle'] == 'tasks' assert _globals['get'] == c.get c.shutdown() assert _globals['get'] is pre_get assert _globals['shuffle'] == pre_shuffle with Client(('127.0.0.1', s['port']), loop=loop) as c: assert _globals['get'] == c.get with Client(('127.0.0.1', s['port']), loop=loop, set_as_default=False) as c: assert _globals['get'] != c.get dask.set_options(get=c.get) assert _globals['get'] == c.get assert _globals['get'] != c.get @gen_cluster(client=True) def test_get_stacks_processing(c, s, a, b): stacks = yield c.scheduler.stacks() assert stacks == valmap(list, s.stacks) processing = yield c.scheduler.processing() assert processing == valmap(list, s.processing) futures = c.map(slowinc, range(10), delay=0.1, workers=[a.address], allow_other_workers=True) yield gen.sleep(0.2) x = yield c.scheduler.stacks() assert x == valmap(list, s.stacks) x = yield c.scheduler.stacks(workers=[a.address]) assert x == {a.address: list(s.stacks[a.address])} x = yield c.scheduler.processing() assert x == valmap(list, s.processing) x = yield c.scheduler.processing(workers=[a.address]) assert x == {a.address: list(s.processing[a.address])} @gen_cluster(client=True) def test_get_foo(c, s, a, b): futures = c.map(inc, range(10)) yield _wait(futures) x = yield c.scheduler.ncores() assert x == s.ncores x = yield c.scheduler.ncores(workers=[a.address]) assert x == {a.address: s.ncores[a.address]} x = yield c.scheduler.has_what() assert x == valmap(list, s.has_what) x = yield c.scheduler.has_what(workers=[a.address]) assert x == {a.address: list(s.has_what[a.address])} x = yield c.scheduler.nbytes(summary=False) assert x == s.nbytes x = yield c.scheduler.nbytes(keys=[futures[0].key], summary=False) assert x == {futures[0].key: s.nbytes[futures[0].key]} x = yield c.scheduler.who_has() assert x == valmap(list, s.who_has) x = yield c.scheduler.who_has(keys=[futures[0].key]) assert x == {futures[0].key: list(s.who_has[futures[0].key])} @slow @gen_cluster(client=True, Worker=Nanny) def test_bad_tasks_fail(c, s, a, b): f = c.submit(sys.exit, 1) with pytest.raises(KilledWorker): yield f._result() def test_get_stacks_processing_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: stacks = c.stacks() processing = c.processing() assert len(stacks) == len(processing) == 2 assert not any(v for v in stacks.values()) assert not any(v for v in processing.values()) futures = c.map(slowinc, range(10), delay=0.1, workers=[('127.0.0.1', a['port'])], allow_other_workers=True) sleep(0.2) aa = '127.0.0.1:%d' % a['port'] bb = '127.0.0.1:%d' % b['port'] stacks = c.stacks() processing = c.processing() assert stacks[aa] assert all(k.startswith('slowinc') for k in stacks[aa]) assert stacks[bb] == [] assert set(c.stacks(aa)) == {aa} assert set(c.stacks([aa])) == {aa} assert set(c.processing(aa)) == {aa} assert set(c.processing([aa])) == {aa} c.cancel(futures) def dont_test_scheduler_falldown(loop): with cluster(worker_kwargs={'heartbeat_interval': 10}) as (s, [a, b]): s['proc'].terminate() s['proc'].join(timeout=2) try: s2 = Scheduler(loop=loop, validate=True) loop.add_callback(s2.start, s['port']) sleep(0.1) with Client(('127.0.0.1', s['port']), loop=loop) as ee: assert len(ee.ncores()) == 2 finally: s2.close() def test_shutdown_idempotent(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: c.shutdown() c.shutdown() c.shutdown() @gen_cluster(client=True) def test_get_returns_early(c, s, a, b): start = time() with ignoring(Exception): result = yield c._get({'x': (throws, 1), 'y': (sleep, 1)}, ['x', 'y']) assert time() < start + 0.5 assert not c.futures start = time() while 'y' in s.tasks: yield gen.sleep(0.01) assert time() < start + 3 x = c.submit(inc, 1) yield x._result() with ignoring(Exception): result = yield c._get({'x': (throws, 1), x.key: (inc, 1)}, ['x', x.key]) assert x.key in s.tasks @slow @gen_cluster(Worker=Nanny, client=True) def test_Client_clears_references_after_restart(c, s, a, b): x = c.submit(inc, 1) assert x.key in c.refcount yield c._restart() assert x.key not in c.refcount key = x.key del x import gc; gc.collect() assert key not in c.refcount def test_get_stops_work_after_error(loop): # TODO: this test uses very old testing machinery loop2 = IOLoop() s = Scheduler(loop=loop2, validate=True) s.start(0) w = Worker(s.ip, s.port, loop=loop2) w.start(0) t = Thread(target=loop2.start) t.daemon = True t.start() with Client(s.address, loop=loop) as c: with pytest.raises(Exception): c.get({'x': (throws, 1), 'y': (sleep, 1)}, ['x', 'y']) start = time() while len(s.tasks): sleep(0.1) assert time() < start + 5 sync(loop2, w._close) sync(loop2, s.close) loop2.add_callback(loop2.stop) while loop2._running: sleep(0.01) loop2.close(all_fds=True) t.join() def test_as_completed_list(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: seq = c.map(inc, iter(range(5))) seq2 = list(as_completed(seq)) assert set(c.gather(seq2)) == {1, 2, 3, 4, 5} @gen_test() def test_status(): s = Scheduler() s.start(0) c = Client((s.ip, s.port), start=False) assert c.status != 'running' with pytest.raises(Exception): x = c.submit(inc, 1) yield c._start() assert c.status == 'running' x = c.submit(inc, 1) yield c._shutdown() assert c.status == 'closed' yield s.close() @gen_cluster(client=True) def test_persist_optimize_graph(c, s, a, b): i = 10 import dask.bag as db for method in [c.persist, c.compute]: b = db.range(i, npartitions=2); i += 1 b2 = b.map(inc) b3 = b2.map(inc) b4 = method(b3, optimize_graph=False) yield _wait(b4) assert set(map(tokey, b3._keys())).issubset(s.tasks) b = db.range(i, npartitions=2); i += 1 b2 = b.map(inc) b3 = b2.map(inc) b4 = method(b3, optimize_graph=True) yield _wait(b4) assert not any(tokey(k) in s.tasks for k in b2._keys()) @gen_cluster(client=True, ncores=[]) def test_scatter_raises_if_no_workers(c, s): with pytest.raises(gen.TimeoutError): yield c._scatter([1]) @gen_test() def test_synchronize_worker_data(): s = Scheduler(synchronize_worker_interval=50) s.start(0) a = Worker(s.ip, s.port, name='alice') yield a._start() a.data['x'] = 1 response = yield s.synchronize_worker_data() assert response == {a.address: {'extra': ['x'], 'missing': []}} assert not a.data yield a._close() s.stop() @gen_test() def test_synchronize_worker_data_race_condition(): s = Scheduler(synchronize_worker_interval=50) s.start(0) a = Worker(s.ip, s.port, name='alice') yield a._start() a.data['x'] = 1 s.loop.add_callback(s.synchronize_worker_data) yield gen.sleep(0.020) s.has_what[a.address].add('x') s.who_has['x'] = {a.address} yield gen.sleep(0.100) assert a.data == {'x': 1} yield a._close() s.stop() @gen_test() def test_synchronize_worker_data_callback(): s = Scheduler(synchronize_worker_interval=50) s.start(0) a = Worker(s.ip, s.port, name='alice') yield a._start() a.data['x'] = 1 yield gen.sleep(0.200) assert not a.data yield a._close() s.stop() @gen_cluster(client=True) def test_synchronize_missing_data_on_one_worker(c, s, a, b): s.synchronize_worker_interval = 10 np = pytest.importorskip('numpy') [f] = yield c._scatter([1], broadcast=True) assert a.data and b.data del a.data[f.key] yield s.synchronize_worker_data() assert s.task_state[f.key] == 'memory' assert b.data # assert set(s.has_what[b.address]) == set(a.data) from distributed.utils_test import popen @slow def test_reconnect(loop): w = Worker('127.0.0.1', 9393, loop=loop) w.start() with popen(['dask-scheduler', '--port', '9393', '--no-bokeh']) as s: c = Client('localhost:9393', loop=loop) start = time() while len(c.ncores()) != 1: sleep(0.1) assert time() < start + 3 x = c.submit(inc, 1) assert x.result() == 2 start = time() while c.status != 'connecting': assert time() < start + 5 sleep(0.01) with pytest.raises(Exception): c.ncores() assert x.status == 'cancelled' with pytest.raises(CancelledError): x.result() with popen(['dask-scheduler', '--port', '9393', '--no-bokeh']) as s: start = time() while c.status != 'running': sleep(0.01) assert time() < start + 5 start = time() while len(c.ncores()) != 1: sleep(0.01) assert time() < start + 5 x = c.submit(inc, 1) assert x.result() == 2 start = time() while True: try: x.result() assert False except StreamClosedError: continue except CancelledError: break assert time() < start + 5 sleep(0.1) c.shutdown() sync(loop, w._close) @slow @pytest.mark.skipif(sys.platform.startswith('win'), reason="num_fds not supported on windows") @pytest.mark.parametrize("worker,count,repeat", [(Worker, 100, 5), (Nanny, 10, 20)]) def test_open_close_many_workers(loop, worker, count, repeat): psutil = pytest.importorskip('psutil') proc = psutil.Process() with cluster(nworkers=0) as (s, []): before = proc.num_fds() @gen.coroutine def start_worker(sleep, duration, repeat=1): for i in range(repeat): yield gen.sleep(sleep) w = worker('127.0.0.1', s['port'], loop=loop) yield w._start() yield gen.sleep(duration) yield w._close() for i in range(count): loop.add_callback(start_worker, random() / 5, random() / 5, repeat=repeat) with Client(('127.0.0.1', s['port']), loop=loop) as c: sleep(1) start = time() while c.ncores(): sleep(0.2) assert time() < start + 10 after = proc.num_fds() assert before >= after @gen_cluster(client=False, timeout=None) def test_idempotence(s, a, b): c = Client((s.ip, s.port), start=False) yield c._start() f = Client((s.ip, s.port), start=False) yield f._start() # Submit x = c.submit(inc, 1) yield x._result() log = list(s.transition_log) len_single_submit = len(log) # see last assert y = f.submit(inc, 1) assert x.key == y.key yield y._result() yield gen.sleep(0.1) log2 = list(s.transition_log) assert log == log2 # Error a = c.submit(div, 1, 0) yield _wait(a) assert a.status == 'error' log = list(s.transition_log) b = f.submit(div, 1, 0) assert a.key == b.key yield _wait(b) yield gen.sleep(0.1) log2 = list(s.transition_log) assert log == log2 s.transition_log.clear() # Simultaneous Submit d = c.submit(inc, 2) e = c.submit(inc, 2) yield _wait([d, e]) assert len(s.transition_log) == len_single_submit yield c._shutdown() yield f._shutdown() def test_scheduler_info(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: info = c.scheduler_info() assert isinstance(info, dict) assert len(info['workers']) == 2 def test_get_versions(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: v = c.get_versions() assert v['scheduler'] is not None assert v['client'] is not None assert len(v['workers']) == 2 for k, v in v['workers'].items(): assert v is not None def test_threaded_get_within_distributed(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: import dask.multiprocessing for get in [dask.async.get_sync, dask.multiprocessing.get, dask.threaded.get]: def f(): return get({'x': (lambda: 1,)}, 'x') future = c.submit(f) assert future.result() == 1 @gen_cluster(client=False) def test_publish_simple(s, a, b): c = Client((s.ip, s.port), start=False) yield c._start() f = Client((s.ip, s.port), start=False) yield f._start() data = yield c._scatter(range(3)) out = yield c._publish_dataset(data=data) assert 'data' in s.datasets with pytest.raises(KeyError) as exc_info: out = yield c._publish_dataset(data=data) assert "exists" in str(exc_info.value) assert "data" in str(exc_info.value) result = yield c.scheduler.list_datasets() assert result == ['data'] result = yield f.scheduler.list_datasets() assert result == ['data'] yield c._shutdown() yield f._shutdown() @gen_cluster(client=False) def test_publish_roundtrip(s, a, b): c = Client((s.ip, s.port), start=False) yield c._start() f = Client((s.ip, s.port), start=False) yield f._start() data = yield c._scatter([0, 1, 2]) yield c._publish_dataset(data=data) assert 'published-data' in s.who_wants[data[0].key] result = yield f._get_dataset(name='data') assert len(result) == len(data) out = yield f._gather(result) assert out == [0, 1, 2] with pytest.raises(KeyError) as exc_info: result = yield f._get_dataset(name='nonexistent') assert "not found" in str(exc_info.value) assert "nonexistent" in str(exc_info.value) yield c._shutdown() yield f._shutdown() @gen_cluster(client=True) def test_unpublish(c, s, a, b): data = yield c._scatter([0, 1, 2]) yield c._publish_dataset(data=data) key = data[0].key del data yield c.scheduler.unpublish_dataset(name='data') assert 'data' not in s.datasets start = time() while key in s.who_wants: yield gen.sleep(0.01) assert time() < start + 5 with pytest.raises(KeyError) as exc_info: result = yield c._get_dataset(name='data') assert "not found" in str(exc_info.value) assert "data" in str(exc_info.value) @gen_cluster(client=True) def test_publish_multiple_datasets(c, s, a, b): x = delayed(inc)(1) y = delayed(inc)(2) yield c._publish_dataset(x=x, y=y) datasets = yield c.scheduler.list_datasets() assert set(datasets) == {'x', 'y'} @gen_cluster(client=False) def test_publish_bag(s, a, b): db = pytest.importorskip('dask.bag') c = Client((s.ip, s.port), start=False) yield c._start() f = Client((s.ip, s.port), start=False) yield f._start() bag = db.from_sequence([0, 1, 2]) bagp = c.persist(bag) assert len(futures_of(bagp)) == 3 keys = {f.key for f in futures_of(bagp)} assert keys == set(bag.dask) yield c._publish_dataset(data=bagp) # check that serialization didn't affect original bag's dask assert len(futures_of(bagp)) == 3 result = yield f._get_dataset('data') assert set(result.dask.keys()) == set(bagp.dask.keys()) assert {f.key for f in result.dask.values()} == {f.key for f in bagp.dask.values()} out = yield f.compute(result)._result() assert out == [0, 1, 2] yield c._shutdown() yield f._shutdown() @gen_cluster(client=True) def test_lose_scattered_data(c, s, a, b): [x] = yield c._scatter([1], workers=a.address) yield a._close() yield gen.sleep(0.1) assert x.status == 'cancelled' assert x.key not in s.task_state @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3) def test_partially_lose_scattered_data(e, s, a, b, c): [x] = yield e._scatter([1], workers=a.address) yield e._replicate(x, n=2) yield a._close() yield gen.sleep(0.1) assert x.status == 'finished' assert s.task_state[x.key] == 'memory' @gen_cluster(client=True) def test_scatter_compute_lose(c, s, a, b): [x] = yield c._scatter([[1, 2, 3, 4]], workers=a.address) y = c.submit(inc, 1) z = c.submit(slowadd, x, y, delay=0.2) yield gen.sleep(0.1) yield a._close() assert x.status == 'cancelled' assert y.status == 'finished' assert z.status == 'cancelled' with pytest.raises(CancelledError): yield _wait(z) @gen_cluster(client=True) def test_scatter_compute_store_lose(c, s, a, b): """ Create irreplaceable data on one machine, cause a dependent computation to occur on another and complete Kill the machine with the irreplaceable data. What happens to the complete result? How about after it GCs and tries to come back? """ [x] = yield c._scatter([1], workers=a.address) xx = c.submit(inc, x, workers=a.address) y = c.submit(inc, 1) z = c.submit(slowadd, xx, y, delay=0.2, workers=b.address) yield _wait(z) yield a._close() start = time() while x.status == 'finished': yield gen.sleep(0.01) assert time() < start + 2 # assert xx.status == 'finished' assert y.status == 'finished' assert z.status == 'finished' zz = c.submit(inc, z) yield _wait(zz) zkey = z.key del z start = time() while s.task_state[zkey] != 'released': yield gen.sleep(0.01) assert time() < start + 2 xxkey = xx.key del xx start = time() while (x.key in s.task_state and zkey not in s.task_state and xxkey not in s.task_state): yield gen.sleep(0.01) assert time() < start + 2 @gen_cluster(client=True) def test_scatter_compute_store_lose_processing(c, s, a, b): """ Create irreplaceable data on one machine, cause a dependent computation to occur on another and complete Kill the machine with the irreplaceable data. What happens to the complete result? How about after it GCs and tries to come back? """ [x] = yield c._scatter([1], workers=a.address) y = c.submit(slowinc, x, delay=0.2) z = c.submit(inc, y) yield gen.sleep(0.1) yield a._close() start = time() while x.status == 'finished': yield gen.sleep(0.01) assert time() < start + 2 assert y.status == 'cancelled' assert z.status == 'cancelled' @gen_cluster(client=False) def test_serialize_future(s, a, b): c = Client((s.ip, s.port), start=False) yield c._start() f = Client((s.ip, s.port), start=False) yield f._start() future = c.submit(lambda: 1) result = yield future._result() with temp_default_client(f): future2 = pickle.loads(pickle.dumps(future)) assert future2.client is f assert tokey(future2.key) in f.futures result2 = yield future2._result() assert result == result2 yield c._shutdown() yield f._shutdown() @gen_cluster(client=False) def test_temp_client(s, a, b): c = Client((s.ip, s.port), start=False) yield c._start() f = Client((s.ip, s.port), start=False) yield f._start() with temp_default_client(c): assert default_client() is c assert default_client(f) is f with temp_default_client(f): assert default_client() is f assert default_client(c) is c yield c._shutdown() yield f._shutdown() @gen_cluster(ncores=[('127.0.0.1', 1)] * 3, client=True) def test_persist_workers(e, s, a, b, c): L1 = [delayed(inc)(i) for i in range(4)] total = delayed(sum)(L1) L2 = [delayed(add)(i, total) for i in L1] out = e.persist(L1 + L2 + [total], workers={tuple(L1): a.address, total: b.address, tuple(L2): [c.address]}, allow_other_workers=L1 + [total]) yield _wait(out) assert all(v.key in a.data for v in L1) assert total.key in b.data assert all(v.key in c.data for v in L2) assert s.loose_restrictions == {total.key} | {v.key for v in L1} @gen_cluster(ncores=[('127.0.0.1', 1)] * 3, client=True) def test_compute_workers(e, s, a, b, c): L1 = [delayed(inc)(i) for i in range(4)] total = delayed(sum)(L1) L2 = [delayed(add)(i, total) for i in L1] out = e.compute(L1 + L2 + [total], workers={tuple(L1): a.address, total: b.address, tuple(L2): [c.address]}, allow_other_workers=L1 + [total]) yield _wait(out) for v in L1: assert s.restrictions[v.key] == {a.address} for v in L2: assert s.restrictions[v.key] == {c.address} assert s.restrictions[total.key] == {b.address} assert s.loose_restrictions == {total.key} | {v.key for v in L1} def test_get_restrictions(): L1 = [delayed(inc)(i) for i in range(4)] total = delayed(sum)(L1) L2 = [delayed(add)(i, total) for i in L1] r1, loose = get_restrictions(L2, '127.0.0.1', False) assert r1 == {d.key: ['127.0.0.1'] for d in L2} assert not loose r1, loose = get_restrictions(L2, ['127.0.0.1'], True) assert r1 == {d.key: ['127.0.0.1'] for d in L2} assert set(loose) == {d.key for d in L2} r1, loose = get_restrictions(L2, {total: '127.0.0.1'}, True) assert r1 == {total.key: ['127.0.0.1']} assert loose == [total.key] r1, loose = get_restrictions(L2, {(total,): '127.0.0.1'}, True) assert r1 == {total.key: ['127.0.0.1']} assert loose == [total.key] @gen_cluster(client=True) def test_scatter_type(c, s, a, b): [future] = yield c._scatter([1]) assert future.type == int d = yield c._scatter({'x': 1.0}) assert d['x'].type == float @gen_cluster(client=True) def test_retire_workers(c, s, a, b): [x] = yield c._scatter([1], workers=a.address) yield s.retire_workers(workers=[a.address]) assert b.data == {x.key: 1} assert s.who_has == {x.key: {b.address}} assert s.has_what == {b.address: {x.key}} assert a.address not in s.worker_info @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10) def test_retire_many_workers(c, s, *workers): futures = yield c._scatter(list(range(100))) yield s.retire_workers(workers=[w.address for w in workers[:7]]) results = yield c._gather(futures) assert results == list(range(100)) assert len(s.has_what) == len(s.ncores) == 3 for w, keys in s.has_what.items(): assert 20 < len(keys) < 50 @gen_cluster(client=True, scheduler_kwargs={'steal': False}, ncores=[('127.0.0.1', 3)] * 2) def test_weight_occupancy_against_data_movement(c, s, a, b): s.task_duration['f'] = 0.01 def f(x, y=0, z=0): sleep(0.01) return x y = yield c._scatter([[1, 2, 3, 4]], workers=[a.address]) z = yield c._scatter([1], workers=[b.address]) futures = c.map(f, [1, 2, 3, 4], y=y, z=z) yield _wait(futures) assert sum(f.key in a.data for f in futures) >= 2 assert sum(f.key in b.data for f in futures) >= 1 @gen_cluster(client=True, scheduler_kwargs={'steal': False}, ncores=[('127.0.0.1', 1), ('127.0.0.1', 10)]) def test_distribute_tasks_by_ncores(c, s, a, b): s.task_duration['f'] = 0.01 def f(x, y=0): sleep(0.01) return x y = yield c._scatter([1], broadcast=True) futures = c.map(f, range(20), y=y) yield _wait(futures) assert len(b.data) > 2 * len(a.data) @gen_cluster(client=True) def test_add_done_callback(c, s, a, b): x = c.submit(inc, 1) L = [] def f(future): L.append(future.key) L.append(future.status) x.add_done_callback(f) yield _wait(x) assert L == [x.key, x.status] @gen_cluster(client=True) def test_normalize_collection(c, s, a, b): x = delayed(inc)(1) y = delayed(inc)(x) z = delayed(inc)(y) yy = c.persist(y) zz = c.normalize_collection(z) assert len(z.dask) == len(y.dask) + 1 assert isinstance(zz.dask[y.key], Future) assert len(zz.dask) < len(z.dask) @gen_cluster(client=True) def test_normalize_collection_dask_array(c, s, a, b): da = pytest.importorskip('dask.array') x = da.ones(10, chunks=(5,)) y = x + 1 yy = c.persist(y) z = y.sum() zdsk = z.dask.copy() zz = c.normalize_collection(z) assert z.dask == zdsk # do not mutate input assert len(z.dask) > len(zz.dask) assert any(isinstance(v, Future) for v in zz.dask.values()) for k, v in yy.dask.items(): assert zz.dask[k].key == v.key result1 = yield c.compute(z)._result() result2 = yield c.compute(zz)._result() assert result1 == result2 @gen_cluster(client=True) def test_auto_normalize_collection(c, s, a, b): da = pytest.importorskip('dask.array') x = da.ones(10, chunks=5) assert len(x.dask) == 2 with dask.set_options(optimizations=[c._optimize_insert_futures]): y = x.map_blocks(slowinc, delay=1, dtype=x.dtype) yy = c.persist(y) yield _wait(yy) start = time() future = c.compute(y.sum()) yield future._result() end = time() assert end - start < 1 start = time() z = c.persist(y + 1) yield _wait(z) end = time() assert end - start < 1 def test_auto_normalize_collection_sync(loop): da = pytest.importorskip('dask.array') with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = da.ones(10, chunks=5) y = x.map_blocks(slowinc, delay=1, dtype=x.dtype) yy = c.persist(y) wait(yy) with dask.set_options(optimizations=[c._optimize_insert_futures]): start = time() y.sum().compute() end = time() assert end - start < 1 distributed-1.14.3/distributed/tests/test_collections.py000066400000000000000000000276361301211143400235410ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from datetime import timedelta import sys import pytest pytest.importorskip('numpy') pytest.importorskip('pandas') import dask import dask.dataframe as dd import dask.bag as db from distributed import Client from distributed.client import _wait from distributed.utils_test import cluster, loop, gen_cluster from distributed.collections import (_futures_to_dask_dataframe, futures_to_dask_dataframe, _futures_to_dask_array, futures_to_dask_array, _futures_to_collection, _futures_to_dask_bag, futures_to_dask_bag, _future_to_dask_array, _futures_to_dask_arrays, futures_to_collection, future_to_dask_array, futures_to_dask_arrays) import numpy as np import pandas as pd import pandas.util.testing as tm from toolz import identity from tornado import gen from tornado.ioloop import IOLoop dfs = [pd.DataFrame({'x': [1, 2, 3]}, index=[0, 10, 20]), pd.DataFrame({'x': [4, 5, 6]}, index=[30, 40, 50]), pd.DataFrame({'x': [7, 8, 9]}, index=[60, 70, 80])] def assert_equal(a, b): assert type(a) == type(b) if isinstance(a, pd.DataFrame): tm.assert_frame_equal(a, b) elif isinstance(a, pd.Series): tm.assert_series_equal(a, b) elif isinstance(a, pd.Index): tm.assert_index_equal(a, b) else: assert a == b @gen_cluster(client=True) def test__futures_to_dask_dataframe(c, s, a, b): remote_dfs = c.map(identity, dfs) ddf = yield _futures_to_dask_dataframe(remote_dfs, divisions=True, client=c) assert isinstance(ddf, dd.DataFrame) assert ddf.divisions == (0, 30, 60, 80) expr = ddf.x.sum() result = yield c._get(expr.dask, expr._keys()) assert result == [sum([df.x.sum() for df in dfs])] @gen_cluster(client=True) def test_no_divisions(c, s, a, b): dfs = c.map(tm.makeTimeDataFrame, range(5, 10)) df = yield _futures_to_dask_dataframe(dfs) assert not df.known_divisions assert list(df.columns) == list(tm.makeTimeDataFrame(5).columns) def test_futures_to_dask_dataframe(loop): with cluster() as (c, [a, b]): with Client(('127.0.0.1', c['port']), loop=loop) as c: remote_dfs = c.map(lambda x: x, dfs) ddf = futures_to_dask_dataframe(remote_dfs, divisions=True) assert isinstance(ddf, dd.DataFrame) assert ddf.x.sum().compute(get=c.get) == sum([df.x.sum() for df in dfs]) ddf2 = futures_to_collection(remote_dfs, divisions=True) assert type(ddf) == type(ddf2) assert ddf.dask == ddf2.dask @gen_cluster(timeout=240, client=True) def test_dataframes(c, s, a, b): df = pd.DataFrame({'x': np.random.random(1000), 'y': np.random.random(1000)}, index=np.arange(1000)) ldf = dd.from_pandas(df, npartitions=10) rdf = c.persist(ldf) assert rdf.divisions == ldf.divisions remote = c.compute(rdf) result = yield remote._result() tm.assert_frame_equal(result, ldf.compute(get=dask.get)) exprs = [lambda df: df.x.mean(), lambda df: df.y.std(), lambda df: df.assign(z=df.x + df.y).drop_duplicates(), lambda df: df.index, lambda df: df.x, lambda df: df.x.cumsum(), lambda df: df.groupby(['x', 'y']).count(), lambda df: df.loc[50:75]] for f in exprs: local = f(ldf).compute(get=dask.get) remote = c.compute(f(rdf)) remote = yield remote._result() assert_equal(local, remote) @gen_cluster(timeout=60, client=True) def test__futures_to_dask_array(c, s, a, b): import dask.array as da remote_arrays = [[[c.submit(np.full, (2, 3, 4), i + j + k) for i in range(2)] for j in range(2)] for k in range(4)] x = yield _futures_to_dask_array(remote_arrays, client=c) assert x.chunks == ((2, 2, 2, 2), (3, 3), (4, 4)) assert x.dtype == np.full((), 0).dtype assert isinstance(x, da.Array) expr = x.sum() result = yield c._get(expr.dask, expr._keys()) assert isinstance(result[0], np.number) @gen_cluster(client=True) def test__future_to_dask_array(c, s, a, b): import dask.array as da f = c.submit(np.ones, (5, 5)) a = yield _future_to_dask_array(f) assert a.shape == (5, 5) assert a.dtype == np.ones(1).dtype assert isinstance(a, da.Array) aa = yield c.compute(a)._result() assert (aa == 1).all() @gen_cluster(client=True) def test__futures_to_dask_arrays(c, s, a, b): import dask.array as da fs = c.map(np.ones, [(5, i) for i in range(1, 6)], pure=False) arrs = yield _futures_to_dask_arrays(fs) assert len(fs) == len(arrs) == 5 assert all(a.shape == (5, i + 1) for i, a in enumerate(arrs)) assert all(a.dtype == np.ones(1).dtype for a in arrs) assert all(isinstance(a, da.Array) for a in arrs) xs = yield c._gather(c.compute(arrs)) assert all((x == 1).all() for x in xs) def test_futures_to_dask_arrays(loop): with cluster() as (c, [a, b]): with Client(('127.0.0.1', c['port']), loop=loop) as c: futures = c.map(np.ones, [(5, i) for i in range(1, 6)]) x = future_to_dask_array(futures[0]) assert x.shape == (5, 1) assert (x.compute(get=c.get) == 1).all() xs = futures_to_dask_arrays(futures) assert [x.shape for x in xs] == [(5, i) for i in range(1, 6)] @gen_cluster(client=True) def test__dask_array_collections(c, s, a, b): import dask.array as da x_dsk = {('x', i, j): np.random.random((3, 3)) for i in range(3) for j in range(2)} y_dsk = {('y', i, j): np.random.random((3, 3)) for i in range(2) for j in range(3)} x_futures = yield c._scatter(x_dsk) y_futures = yield c._scatter(y_dsk) dt = np.random.random(0).dtype x_local = da.Array(x_dsk, 'x', ((3, 3, 3), (3, 3)), dt) y_local = da.Array(y_dsk, 'y', ((3, 3), (3, 3, 3)), dt) x_remote = da.Array(x_futures, 'x', ((3, 3, 3), (3, 3)), dt) y_remote = da.Array(y_futures, 'y', ((3, 3), (3, 3, 3)), dt) exprs = [lambda x, y: x.T + y, lambda x, y: x.mean() + y.mean(), lambda x, y: x.dot(y).std(axis=0), lambda x, y: x - x.mean(axis=1)[:, None]] for expr in exprs: local = expr(x_local, y_local).compute(get=dask.get) remote = c.compute(expr(x_remote, y_remote)) remote = yield remote._result() assert np.all(local == remote) @gen_cluster(client=True) def test__futures_to_dask_bag(c, s, a, b): import dask.bag as db L = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] futures = yield c._scatter(L) rb = yield _futures_to_dask_bag(futures) assert isinstance(rb, db.Bag) assert rb.npartitions == len(L) lb = db.from_sequence([1, 2, 3, 4, 5, 6, 7, 8, 9], npartitions=3) exprs = [lambda x: x.map(lambda x: x + 1).sum(), lambda x: x.filter(lambda x: x % 2)] for expr in exprs: local = expr(lb).compute(get=dask.get) remote = c.compute(expr(rb)) remote = yield remote._result() assert local == remote def test_futures_to_dask_bag(loop): import dask.bag as db with cluster() as (c, [a, b]): with Client(('127.0.0.1', c['port']), loop=loop) as c: data = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] futures = c.scatter(data) b = futures_to_dask_bag(futures) assert isinstance(b, db.Bag) assert b.map(lambda x: x + 1).sum().compute(get=c.get) == sum(range(2, 11)) @pytest.mark.skipif(not sys.platform.startswith('linux'), reason='KQueue error - uncertain cause') def test_futures_to_dask_array(loop): with cluster() as (c, [a, b]): with Client(('127.0.0.1', c['port']), loop=loop) as c: remote_arrays = [[c.submit(np.full, (3, 3), i + j) for i in range(3)] for j in range(3)] x = futures_to_dask_array(remote_arrays, client=c) assert x.chunks == ((3, 3, 3), (3, 3, 3)) assert x.dtype == np.full((), 0).dtype assert x.sum().compute(get=c.get) == 162 assert (x + x.T).sum().compute(get=c.get) == 162 * 2 y = futures_to_collection(remote_arrays, client=c) assert x.dask == y.dask @gen_cluster(client=True) def test__futures_to_collection(c, s, a, b): remote_dfs = c.map(identity, dfs) ddf = yield _futures_to_collection(remote_dfs, divisions=True) ddf2 = yield _futures_to_dask_dataframe(remote_dfs, divisions=True) assert isinstance(ddf, dd.DataFrame) assert ddf.dask == ddf2.dask remote_arrays = c.map(np.arange, range(3, 5)) x = yield _futures_to_collection(remote_arrays) y = yield _futures_to_dask_array(remote_arrays) assert type(x) == type(y) assert x.dask == y.dask remote_lists = yield c._scatter([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) b = yield _futures_to_collection(remote_lists) c = yield _futures_to_dask_bag(remote_lists) assert type(b) == type(c) assert b.dask == b.dask @gen_cluster(client=True) def test_bag_groupby_tasks_default(c, s, a, b): with dask.set_options(get=c.get): b = db.range(100, npartitions=10) b2 = b.groupby(lambda x: x % 13) assert not any('partd' in k[0] for k in b2.dask) def test_dataframe_set_index_sync(loop): with cluster() as (c, [a, b]): with Client(('127.0.0.1', c['port']), loop=loop) as c: with dask.set_options(get=c.get): df = dd.demo.make_timeseries('2000', '2001', {'value': float, 'name': str, 'id': int}, freq='2H', partition_freq='1M', seed=1) df = c.persist(df) df2 = df.set_index('name', shuffle='tasks') df2 = c.persist(df2) df2.head() def test_loc_sync(loop): with cluster() as (c, [a, b]): with Client(('127.0.0.1', c['port']), loop=loop) as c: df = pd.util.testing.makeTimeDataFrame() ddf = dd.from_pandas(df, npartitions=10) ddf.loc['2000-01-17':'2000-01-24'].compute(get=c.get) def test_rolling_sync(loop): with cluster() as (c, [a, b]): with Client(('127.0.0.1', c['port']), loop=loop) as c: df = pd.util.testing.makeTimeDataFrame() ddf = dd.from_pandas(df, npartitions=10) dd.rolling_mean(ddf.A, 2).compute(get=c.get) @gen_cluster(client=True) def test_loc(c, s, a, b): df = pd.util.testing.makeTimeDataFrame() ddf = dd.from_pandas(df, npartitions=10) future = c.compute(ddf.loc['2000-01-17':'2000-01-24']) yield future._result() def test_dataframe_groupby_tasks(loop): df = pd.util.testing.makeTimeDataFrame() df['A'] = df.A // 0.1 df['B'] = df.B // 0.1 ddf = dd.from_pandas(df, npartitions=10) with cluster() as (c, [a, b]): with Client(('127.0.0.1', c['port']), loop=loop) as c: with dask.set_options(get=c.get): for ind in [lambda x: 'A', lambda x: x.A]: a = df.groupby(ind(df)).apply(len) b = ddf.groupby(ind(ddf)).apply(len) assert_equal(a, b.compute(get=dask.get).sort_index()) assert not any('partd' in k[0] for k in b.dask) a = df.groupby(ind(df)).B.apply(len) b = ddf.groupby(ind(ddf)).B.apply(len) assert_equal(a, b.compute(get=dask.get).sort_index()) assert not any('partd' in k[0] for k in b.dask) with pytest.raises(NotImplementedError): ddf.groupby(ddf[['A', 'B']]).apply(len) a = df.groupby(['A', 'B']).apply(len) b = ddf.groupby(['A', 'B']).apply(len) assert_equal(a, b.compute(get=dask.get).sort_index()) distributed-1.14.3/distributed/tests/test_compatibility.py000066400000000000000000000003701301211143400240560ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from distributed.compatibility import gzip_compress, gzip_decompress def test_gzip(): b = b'Hello, world!' c = gzip_compress(b) d = gzip_decompress(c) assert b == d distributed-1.14.3/distributed/tests/test_core.py000066400000000000000000000154271301211143400221460ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from functools import partial from multiprocessing import Process import socket from time import time from tornado import gen, ioloop from tornado.iostream import StreamClosedError import pytest from distributed.core import (read, write, pingpong, Server, rpc, connect, coerce_to_rpc, send_recv, coerce_to_address, ConnectionPool) from distributed.utils_test import slow, loop, gen_test def test_server(loop): @gen.coroutine def f(): server = Server({'ping': pingpong}) with pytest.raises(OSError): server.port server.listen(8887) assert server.port == 8887 stream = yield connect('127.0.0.1', 8887) yield write(stream, {'op': 'ping'}) response = yield read(stream) assert response == b'pong' yield write(stream, {'op': 'ping', 'close': True}) response = yield read(stream) assert response == b'pong' server.stop() loop.run_sync(f) def test_rpc(loop): @gen.coroutine def f(): server = Server({'ping': pingpong}) server.listen(8887) with rpc(ip='127.0.0.1', port=8887) as remote: response = yield remote.ping() assert response == b'pong' response = yield remote.ping(close=True) assert response == b'pong' assert not remote.streams assert remote.status == 'closed' server.stop() loop.run_sync(f) def test_rpc_inputs(): L = [rpc('127.0.0.1:8887'), rpc(b'127.0.0.1:8887'), rpc(('127.0.0.1', 8887)), rpc((b'127.0.0.1', 8887)), rpc(ip='127.0.0.1', port=8887), rpc(ip=b'127.0.0.1', port=8887), rpc(addr='127.0.0.1:8887'), rpc(addr=b'127.0.0.1:8887')] assert all(r.ip == '127.0.0.1' and r.port == 8887 for r in L) for r in L: r.close_rpc() def test_rpc_with_many_connections(loop): remote = rpc(ip='127.0.0.1', port=8887) @gen.coroutine def g(): for i in range(10): yield remote.ping() @gen.coroutine def f(): server = Server({'ping': pingpong}) server.listen(8887) yield [g() for i in range(10)] server.stop() remote.close_streams() assert all(stream.closed() for stream in remote.streams) loop.run_sync(f) def echo(stream, x): return x @slow def test_large_packets(loop): """ tornado has a 100MB cap by default """ @gen.coroutine def f(): server = Server({'echo': echo}) server.listen(8887) data = b'0' * int(200e6) # slightly more than 100MB conn = rpc(ip='127.0.0.1', port=8887) result = yield conn.echo(x=data) assert result == data d = {'x': data} result = yield conn.echo(x=d) assert result == d server.stop() loop.run_sync(f) def test_identity(loop): @gen.coroutine def f(): server = Server({}) server.listen(8887) with rpc(ip='127.0.0.1', port=8887) as remote: a = yield remote.identity() b = yield remote.identity() assert a['type'] == 'Server' assert a['id'] == b['id'] server.stop() loop.run_sync(f) def test_ports(loop): port = 9876 server = Server({}, io_loop=loop) server.listen(port) try: assert server.port == port with pytest.raises((OSError, socket.error)): server2 = Server({}, io_loop=loop) server2.listen(port) finally: server.stop() try: server3 = Server({}, io_loop=loop) server3.listen(0) assert isinstance(server3.port, int) assert server3.port > 1024 finally: server3.stop() def test_errors(loop): s = Server({}) try: s.port except OSError as e: assert '.listen' in str(e) def test_coerce_to_rpc(): with coerce_to_rpc(('127.0.0.1', 8000)) as r: assert (r.ip, r.port) == ('127.0.0.1', 8000) with coerce_to_rpc('127.0.0.1:8000') as r: assert (r.ip, r.port) == ('127.0.0.1', 8000) with coerce_to_rpc('foo:bar:8000') as r: assert (r.ip, r.port) == ('foo:bar', 8000) def stream_div(stream=None, x=None, y=None): return x / y @gen_test() def test_errors(): server = Server({'div': stream_div}) server.listen(0) with rpc(ip='127.0.0.1', port=server.port) as r: with pytest.raises(ZeroDivisionError): yield r.div(x=1, y=0) @gen_test() def test_connect_raises(): with pytest.raises((gen.TimeoutError, StreamClosedError, IOError)): yield connect('127.0.0.1', 58259, timeout=0.01) @gen_test() def test_send_recv_args(): server = Server({'echo': echo}) server.listen(0) result = yield send_recv(arg=('127.0.0.1', server.port), op='echo', x=b'1') assert result == b'1' result = yield send_recv(addr=('127.0.0.1:%d' % server.port).encode(), op='echo', x=b'1') assert result == b'1' result = yield send_recv(ip=b'127.0.0.1', port=server.port, op='echo', x=b'1') assert result == b'1' result = yield send_recv(ip=b'127.0.0.1', port=server.port, op='echo', x=b'1', reply=False) assert result == None server.stop() def test_coerce_to_address(): for arg in [b'127.0.0.1:8786', '127.0.0.1:8786', ('127.0.0.1', 8786), ('127.0.0.1', '8786')]: assert coerce_to_address(arg) == '127.0.0.1:8786' @gen_test() def test_connection_pool(): @gen.coroutine def ping(stream=None, delay=0.1): yield gen.sleep(delay) raise gen.Return('pong') servers = [Server({'ping': ping}) for i in range(10)] for server in servers: server.listen(0) rpc = ConnectionPool(limit=5) # Reuse connections yield [rpc(ip='127.0.0.1', port=s.port).ping() for s in servers[:5]] yield [rpc(ip='127.0.0.1', port=s.port).ping() for s in servers[:5]] yield [rpc(ip='127.0.0.1', port=s.port).ping() for s in servers[:5]] yield [rpc(ip='127.0.0.1', port=s.port).ping() for s in servers[:5]] assert sum(map(len, rpc.available.values())) == 5 assert sum(map(len, rpc.occupied.values())) == 0 assert rpc.active == 0 assert rpc.open == 5 # Clear out connections to make room for more yield [rpc(ip='127.0.0.1', port=s.port).ping() for s in servers[5:]] assert rpc.active == 0 assert rpc.open == 5 s = servers[0] yield [rpc(ip='127.0.0.1', port=s.port).ping(delay=0.1) for i in range(3)] assert len(rpc.available['127.0.0.1', s.port]) == 3 # Explicitly clear out connections rpc.collect() start = time() while any(rpc.available.values()): yield gen.sleep(0.01) assert time() < start + 2 distributed-1.14.3/distributed/tests/test_hdfs.py000066400000000000000000000323461301211143400221410ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import pytest from tornado import gen from dask.delayed import Delayed import dask.bag as db import dask.dataframe as dd from dask import compute, delayed from distributed.compatibility import unicode from distributed.utils_test import gen_cluster, cluster, make_hdfs from distributed.utils import get_ip from distributed.utils_test import loop from distributed import Client from distributed.client import _wait, Future hdfs3 = pytest.importorskip('hdfs3') from distributed.hdfs import read_bytes, get_block_locations try: hdfs = hdfs3.HDFileSystem(host='localhost', port=8020) hdfs.df() del hdfs except: pytestmark = pytest.skip() ip = get_ip() def test_get_block_locations(): with make_hdfs() as hdfs: data = b'a' * int(1e8) # todo: reduce block size to speed up test fn_1 = '/tmp/test/file1' fn_2 = '/tmp/test/file2' with hdfs.open(fn_1, 'wb', replication=1) as f: f.write(data) with hdfs.open(fn_2, 'wb', replication=1) as f: f.write(data) L = get_block_locations(hdfs, '/tmp/test/') aa = get_block_locations(hdfs, fn_1) bb = get_block_locations(hdfs, fn_2) assert (L == aa + bb) or (L == bb + aa) assert L[0]['filename'] == L[1]['filename'] assert L[2]['filename'] == L[3]['filename'] @gen_cluster([(ip, 1)], timeout=60, client=True) def dont_test_dataframes(e, s, a): # slow pytest.importorskip('pandas') n = 3000000 fn = '/tmp/test/file.csv' with make_hdfs() as hdfs: data = (b'name,amount,id\r\n' + b'Alice,100,1\r\nBob,200,2\r\n' * n) with hdfs.open(fn, 'wb') as f: f.write(data) futures = read_bytes(fn, hdfs=hdfs, delimiter=b'\r\n', lazy=False) assert len(futures) > 1 def load(b, **kwargs): assert b from io import BytesIO import pandas as pd bio = BytesIO(b) return pd.read_csv(bio, **kwargs) dfs = e.map(load, futures, names=['name', 'amount', 'id'], skiprows=1) dfs2 = yield e._gather(dfs) assert sum(map(len, dfs2)) == n * 2 - 1 def test_get_block_locations_nested(): with make_hdfs() as hdfs: data = b'a' for i in range(3): hdfs.mkdir('/tmp/test/data-%d' % i) for j in range(2): fn = '/tmp/test/data-%d/file-%d.csv' % (i, j) with hdfs.open(fn, 'wb', replication=1) as f: f.write(data) L = get_block_locations(hdfs, '/tmp/test/') assert len(L) == 6 @gen_cluster([(ip, 1), (ip, 2)], timeout=60, client=True) def test_read_bytes(e, s, a, b): with make_hdfs() as hdfs: data = b'a' * int(1e8) fn = '/tmp/test/file' with hdfs.open(fn, 'wb', replication=1) as f: f.write(data) blocks = hdfs.get_block_locations(fn) assert len(blocks) > 1 sample, values = read_bytes(fn, hdfs=hdfs) assert sample[:5] == b'aaaaa' assert len(values) == len(blocks) while not s.restrictions: yield gen.sleep(0.01) assert not s.tasks assert {v.key for v in values} == set(s.restrictions) assert {v.key for v in values} == set(s.loose_restrictions) futures = e.compute(values) results = yield e._gather(futures) assert b''.join(results) == data assert s.restrictions def test_read_bytes_sync(loop): with cluster(nworkers=3) as (s, [a, b, c]): with make_hdfs() as hdfs: data = b'a' * int(1e3) for fn in ['/tmp/test/file.%d' % i for i in range(100)]: with hdfs.open(fn, 'wb', replication=1) as f: f.write(data) with Client(('127.0.0.1', s['port']), loop=loop) as e: sample, values = read_bytes('/tmp/test/file.*') futures = e.compute(values) results = e.gather(futures) assert b''.join(results) == 100 * data @gen_cluster([(ip, 1), (ip, 2)], timeout=60, client=True) def test_get_block_locations_nested_2(e, s, a, b): with make_hdfs() as hdfs: data = b'a' for i in range(3): hdfs.mkdir('/tmp/test/data-%d' % i) for j in range(2): fn = '/tmp/test/data-%d/file-%d.csv' % (i, j) with hdfs.open(fn, 'wb', replication=1) as f: f.write(data) L = get_block_locations(hdfs, '/tmp/test/') assert len(L) == 6 sample, values = read_bytes('/tmp/test/', hdfs=hdfs) futures = e.compute(values) results = yield e._gather(futures) assert len(results) == 6 assert all(x == b'a' for x in results) @gen_cluster([(ip, 1), (ip, 2)], timeout=60, client=True) def test_lazy_values(e, s, a, b): with make_hdfs() as hdfs: data = b'a' for i in range(3): hdfs.mkdir('/tmp/test/data-%d' % i) for j in range(2): fn = '/tmp/test/data-%d/file-%d.csv' % (i, j) with hdfs.open(fn, 'wb', replication=1) as f: f.write(data) sample, values = read_bytes('/tmp/test/', hdfs=hdfs) assert all(isinstance(v, Delayed) for v in values) while not s.restrictions: yield gen.sleep(0.01) assert not s.tasks results = e.compute(values, sync=False) results = yield e._gather(results) assert len(results) == 6 assert all(x == b'a' for x in results) @gen_cluster([(ip, 1), (ip, 2)], timeout=60, client=True) def test_write_bytes(e, s, a, b): with make_hdfs() as hdfs: data = [b'123', b'456', b'789'] remote_data = yield e._scatter(data) futures = write_bytes('/tmp/test/data/file.*.dat', remote_data, hdfs=hdfs) yield _wait(futures) assert len(hdfs.ls('/tmp/test/data/')) == 3 with hdfs.open('/tmp/test/data/file.1.dat') as f: assert f.read() == b'456' futures = write_bytes('/tmp/test/data2/', remote_data, hdfs=hdfs) yield _wait(futures) assert len(hdfs.ls('/tmp/test/data2/')) == 3 def test_read_csv_sync(loop): import dask.dataframe as dd import pandas as pd with cluster(nworkers=3) as (s, [a, b, c]): with make_hdfs() as hdfs: with hdfs.open('/tmp/test/1.csv', 'wb') as f: f.write(b'name,amount,id\nAlice,100,1\nBob,200,2') with hdfs.open('/tmp/test/2.csv', 'wb') as f: f.write(b'name,amount,id\nCharlie,300,3\nDennis,400,4') with Client(('127.0.0.1', s['port']), loop=loop) as e: values = dd.read_csv('hdfs:///tmp/test/*.csv', lineterminator='\n', collection=False, header=0) futures = e.compute(values) assert all(isinstance(f, Future) for f in futures) L = e.gather(futures) assert isinstance(L[0], pd.DataFrame) assert list(L[0].columns) == ['name', 'amount', 'id'] df = dd.read_csv('hdfs:///tmp/test/*.csv', lineterminator='\n', collection=True, header=0) assert isinstance(df, dd.DataFrame) assert list(df.head().iloc[0]) == ['Alice', 100, 1] def test_read_csv_sync_compute(loop): with cluster(nworkers=1) as (s, [a]): with make_hdfs() as hdfs: with hdfs.open('/tmp/test/1.csv', 'wb') as f: f.write(b'name,amount,id\nAlice,100,1\nBob,200,2') with hdfs.open('/tmp/test/2.csv', 'wb') as f: f.write(b'name,amount,id\nCharlie,300,3\nDennis,400,4') with Client(('127.0.0.1', s['port']), loop=loop) as e: df = dd.read_csv('hdfs:///tmp/test/*.csv', collection=True) assert df.amount.sum().compute(get=e.get) == 1000 @gen_cluster([(ip, 1), (ip, 1)], timeout=60, client=True) def test_read_csv(e, s, a, b): with make_hdfs() as hdfs: with hdfs.open('/tmp/test/1.csv', 'wb') as f: f.write(b'name,amount,id\nAlice,100,1\nBob,200,2') with hdfs.open('/tmp/test/2.csv', 'wb') as f: f.write(b'name,amount,id\nCharlie,300,3\nDennis,400,4') df = dd.read_csv('hdfs:///tmp/test/*.csv', lineterminator='\n') result = e.compute(df.id.sum(), sync=False) result = yield result._result() assert result == 1 + 2 + 3 + 4 @gen_cluster([(ip, 1), (ip, 1)], timeout=60, client=True) def test_read_csv_with_names(e, s, a, b): with make_hdfs() as hdfs: with hdfs.open('/tmp/test/1.csv', 'wb') as f: f.write(b'name,amount,id\nAlice,100,1\nBob,200,2') df = dd.read_csv('hdfs:///tmp/test/*.csv', names=['amount', 'name'], lineterminator='\n') assert list(df.columns) == ['amount', 'name'] @gen_cluster([(ip, 1), (ip, 1)], timeout=60, client=True) def test_read_csv_lazy(e, s, a, b): with make_hdfs() as hdfs: with hdfs.open('/tmp/test/1.csv', 'wb') as f: f.write(b'name,amount,id\nAlice,100,1\nBob,200,2') with hdfs.open('/tmp/test/2.csv', 'wb') as f: f.write(b'name,amount,id\nCharlie,300,3\nDennis,400,4') df = dd.read_csv('hdfs:///tmp/test/*.csv', lineterminator='\n') yield gen.sleep(0.5) assert not s.tasks result = yield e.compute(df.id.sum(), sync=False)._result() assert result == 1 + 2 + 3 + 4 @gen_cluster([(ip, 1), (ip, 1)], timeout=60, client=True) def test__read_text(e, s, a, b): with make_hdfs() as hdfs: with hdfs.open('/tmp/test/text.1.txt', 'wb') as f: f.write('Alice 100\nBob 200\nCharlie 300'.encode()) with hdfs.open('/tmp/test/text.2.txt', 'wb') as f: f.write('Dan 400\nEdith 500\nFrank 600'.encode()) with hdfs.open('/tmp/test/other.txt', 'wb') as f: f.write('a b\nc d'.encode()) b = db.read_text('hdfs:///tmp/test/text.*.txt', collection=True) yield gen.sleep(0.5) assert not s.tasks import dask b.compute(get=dask.get) future = e.compute(b.str.strip().str.split().map(len)) yield gen.sleep(0.5) result = yield future._result() assert result == [2, 2, 2, 2, 2, 2] b = db.read_text('hdfs:///tmp/test/other.txt', collection=True) b = e.persist(b) future = e.compute(b.str.split().concat()) result = yield future._result() assert result == ['a', 'b', 'c', 'd'] L = db.read_text('hdfs:///tmp/test/text.*.txt', collection=False) assert all(isinstance(x, Delayed) for x in L) @gen_cluster([(ip, 1)], timeout=60, client=True) def test__read_text_json_endline(e, s, a): import json with make_hdfs() as hdfs: with hdfs.open('/tmp/test/text.1.txt', 'wb') as f: f.write(b'{"x": 1}\n{"x": 2}\n') b = db.read_text('hdfs:///tmp/test/text.1.txt').map(json.loads) result = yield e.compute(b)._result() assert result == [{"x": 1}, {"x": 2}] @gen_cluster([(ip, 1), (ip, 1)], timeout=60, client=True) def test__read_text_unicode(e, s, a, b): fn = '/tmp/test/data.txt' data = b'abcd\xc3\xa9' with make_hdfs() as hdfs: with hdfs.open(fn, 'wb') as f: f.write(b'\n'.join([data, data])) f = db.read_text('hdfs://' + fn, collection=False) result = yield e.compute(f[0])._result() assert len(result) == 2 assert list(map(unicode.strip, result)) == [data.decode('utf-8')] * 2 assert len(result[0].strip()) == 5 def test_read_text_sync(loop): with make_hdfs() as hdfs: with hdfs.open('/tmp/test/data.txt', 'wb') as f: f.write(b'hello\nworld') with cluster(nworkers=3) as (s, [a, b, c]): with Client(('127.0.0.1', s['port']), loop=loop): b = db.read_text('hdfs:///tmp/test/*.txt') assert list(b.str.strip().str.upper()) == ['HELLO', 'WORLD'] @gen_cluster([(ip, 1), (ip, 2)], timeout=60, client=True) def test_deterministic_key_names(e, s, a, b): with make_hdfs() as hdfs: data = b'abc\n' * int(1e3) fn = '/tmp/test/file' with hdfs.open(fn, 'wb', replication=1) as f: f.write(data) _, x = read_bytes('/tmp/test/', hdfs=hdfs, lazy=True, delimiter=b'\n') _, y = read_bytes('/tmp/test/', hdfs=hdfs, lazy=True, delimiter=b'\n') _, z = read_bytes('/tmp/test/', hdfs=hdfs, lazy=True, delimiter=b'c') assert [f.key for f in x] == [f.key for f in y] assert [f.key for f in x] != [f.key for f in z] @gen_cluster([(ip, 1), (ip, 2)], timeout=60, client=True) def test_write_bytes(e, s, a, b): from dask.bytes.core import write_bytes, read_bytes with make_hdfs() as hdfs: path = 'hdfs:///tmp/test/' data = [b'test data %i' % i for i in range(5)] values = [delayed(d) for d in data] out = write_bytes(values, path, hdfs=hdfs) futures = e.compute(out) results = yield e._gather(futures) assert len(hdfs.ls('/tmp/test/')) == 5 sample, vals = read_bytes('hdfs:///tmp/test/*.part', hdfs=hdfs, lazy=True) futures = e.compute(vals) results = yield e._gather(futures) assert data == results distributed-1.14.3/distributed/tests/test_ipython.py000066400000000000000000000133461301211143400227060ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import mock import pytest from toolz import first from distributed import Client from distributed.utils_test import zmq_ctx, mock_ipython, cluster, loop @pytest.mark.ipython def test_start_ipython_workers(loop, zmq_ctx): from jupyter_client import BlockingKernelClient with cluster(1) as (s, [a]): with Client(('127.0.0.1', s['port']), loop=loop) as e: info_dict = e.start_ipython_workers() info = first(info_dict.values()) key = info.pop('key') kc = BlockingKernelClient(**info) kc.session.key = key kc.start_channels() kc.wait_for_ready(timeout=10) msg_id = kc.execute("worker") reply = kc.get_shell_msg(timeout=10) assert reply['parent_header']['msg_id'] == msg_id assert reply['content']['status'] == 'ok' kc.stop_channels() @pytest.mark.ipython def test_start_ipython_scheduler(loop, zmq_ctx): from jupyter_client import BlockingKernelClient with cluster(1) as (s, [a]): with Client(('127.0.0.1', s['port']), loop=loop) as e: info = e.start_ipython_scheduler() key = info.pop('key') kc = BlockingKernelClient(**info) kc.session.key = key kc.start_channels() msg_id = kc.execute("scheduler") reply = kc.get_shell_msg(timeout=10) kc.stop_channels() @pytest.mark.ipython def test_start_ipython_scheduler_magic(loop, zmq_ctx): with cluster(1) as (s, [a]): with Client(('127.0.0.1', s['port']), loop=loop) as e, mock_ipython() as ip: info = e.start_ipython_scheduler() expected = [ {'magic_kind': 'line', 'magic_name': 'scheduler'}, {'magic_kind': 'cell', 'magic_name': 'scheduler'}, ] call_kwargs_list = [ kwargs for (args, kwargs) in ip.register_magic_function.call_args_list ] assert call_kwargs_list == expected magic = ip.register_magic_function.call_args_list[0][0][0] magic(line="", cell="scheduler") @pytest.mark.ipython def test_start_ipython_workers_magic(loop, zmq_ctx): with cluster(2) as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as e, mock_ipython() as ip: workers = list(e.ncores())[:2] names = [ 'magic%i' % i for i in range(len(workers)) ] info_dict = e.start_ipython_workers(workers, magic_names=names) expected = [ {'magic_kind': 'line', 'magic_name': 'remote'}, {'magic_kind': 'cell', 'magic_name': 'remote'}, {'magic_kind': 'line', 'magic_name': 'magic0'}, {'magic_kind': 'cell', 'magic_name': 'magic0'}, {'magic_kind': 'line', 'magic_name': 'magic1'}, {'magic_kind': 'cell', 'magic_name': 'magic1'}, ] call_kwargs_list = [ kwargs for (args, kwargs) in ip.register_magic_function.call_args_list ] assert call_kwargs_list == expected assert ip.register_magic_function.call_count == 6 magics = [ args[0][0] for args in ip.register_magic_function.call_args_list[2:] ] magics[-1](line="", cell="worker") [ m.client.stop_channels() for m in magics ] @pytest.mark.ipython def test_start_ipython_workers_magic_asterix(loop, zmq_ctx): with cluster(2) as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as e, mock_ipython() as ip: workers = list(e.ncores())[:2] info_dict = e.start_ipython_workers(workers, magic_names='magic_*') expected = [ {'magic_kind': 'line', 'magic_name': 'remote'}, {'magic_kind': 'cell', 'magic_name': 'remote'}, {'magic_kind': 'line', 'magic_name': 'magic_0'}, {'magic_kind': 'cell', 'magic_name': 'magic_0'}, {'magic_kind': 'line', 'magic_name': 'magic_1'}, {'magic_kind': 'cell', 'magic_name': 'magic_1'}, ] call_kwargs_list = [ kwargs for (args, kwargs) in ip.register_magic_function.call_args_list ] assert call_kwargs_list == expected assert ip.register_magic_function.call_count == 6 magics = [ args[0][0] for args in ip.register_magic_function.call_args_list[2:] ] magics[-1](line="", cell="worker") [ m.client.stop_channels() for m in magics ] @pytest.mark.ipython def test_start_ipython_remote(loop, zmq_ctx): from distributed._ipython_utils import remote_magic with cluster(1) as (s, [a]): with Client(('127.0.0.1', s['port']), loop=loop) as e, mock_ipython() as ip: worker = first(e.ncores()) ip.user_ns['info'] = e.start_ipython_workers(worker)[worker] remote_magic('info 1') # line magic remote_magic('info', 'worker') # cell magic expected = [ ((remote_magic,), {'magic_kind': 'line', 'magic_name': 'remote'}), ((remote_magic,), {'magic_kind': 'cell', 'magic_name': 'remote'}), ] assert ip.register_magic_function.call_args_list == expected assert ip.register_magic_function.call_count == 2 @pytest.mark.ipython def test_start_ipython_qtconsole(loop): Popen = mock.Mock() with cluster() as (s, [a, b]): with mock.patch('distributed._ipython_utils.Popen', Popen), Client(('127.0.0.1', s['port']), loop=loop) as e: worker = first(e.ncores()) e.start_ipython_workers(worker, qtconsole=True) e.start_ipython_workers(worker, qtconsole=True, qtconsole_args=['--debug']) assert Popen.call_count == 2 (cmd,), kwargs = Popen.call_args_list[0] assert cmd[:3] == [ 'jupyter', 'qtconsole', '--existing' ] (cmd,), kwargs = Popen.call_args_list[1] assert cmd[-1:] == [ '--debug' ] distributed-1.14.3/distributed/tests/test_joblib.py000066400000000000000000000034751301211143400224570ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import pytest from random import random from time import sleep from distributed.utils_test import inc, cluster, loop backend = pytest.importorskip('distributed.joblib') joblibs = [backend.joblib, backend.sk_joblib] def slow_raise_value_error(condition, duration=0.05): sleep(duration) if condition: raise ValueError("condition evaluated to True") @pytest.mark.parametrize('joblib', joblibs) def test_simple(loop, joblib): if joblib is None: pytest.skip() Parallel = joblib.Parallel delayed = joblib.delayed with cluster() as (s, [a, b]): with joblib.parallel_backend('dask.distributed', loop=loop, scheduler_host=('127.0.0.1', s['port'])): seq = Parallel()(delayed(inc)(i) for i in range(10)) assert seq == [inc(i) for i in range(10)] with pytest.raises(ValueError): Parallel()(delayed(slow_raise_value_error)(i == 3) for i in range(10)) seq = Parallel()(delayed(inc)(i) for i in range(10)) assert seq == [inc(i) for i in range(10)] ba, _ = joblib.parallel.get_active_backend() ba.client.shutdown() def random2(): return random() @pytest.mark.parametrize('joblib', joblibs) def test_dont_assume_function_purity(loop, joblib): if joblib is None: pytest.skip() Parallel = joblib.Parallel delayed = joblib.delayed with cluster() as (s, [a, b]): with joblib.parallel_backend('dask.distributed', loop=loop, scheduler_host=('127.0.0.1', s['port'])): x, y = Parallel()(delayed(random2)() for i in range(2)) assert x != y ba, _ = joblib.parallel.get_active_backend() ba.client.shutdown() distributed-1.14.3/distributed/tests/test_nanny.py000066400000000000000000000074671301211143400223460ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from datetime import datetime import os import sys from time import time import pytest from toolz import valmap from tornado.tcpclient import TCPClient from tornado.iostream import StreamClosedError from tornado import gen from distributed import Nanny, rpc, Scheduler from distributed.core import connect, read, write from distributed.protocol.pickle import dumps, loads from distributed.utils import ignoring from distributed.utils_test import gen_cluster from distributed.nanny import isalive @gen_cluster(ncores=[]) def test_nanny(s): n = Nanny(s.ip, s.port, ncores=2, ip='127.0.0.1', loop=s.loop) yield n._start(0) with rpc(ip=n.ip, port=n.port) as nn: assert isalive(n.process) # alive assert s.ncores[n.worker_address] == 2 assert s.worker_info[n.worker_address]['services']['nanny'] > 1024 yield nn.kill() assert not n.process assert n.worker_address not in s.ncores assert n.worker_address not in s.worker_info yield nn.kill() assert n.worker_address not in s.ncores assert n.worker_address not in s.worker_info assert not n.process yield nn.instantiate() assert isalive(n.process) assert s.ncores[n.worker_address] == 2 assert s.worker_info[n.worker_address]['services']['nanny'] > 1024 yield nn.terminate() assert not n.process yield n._close() @gen_cluster(Worker=Nanny) def test_str(s, a, b): assert a.worker_address in str(a) assert a.worker_address in repr(a) assert str(a.ncores) in str(a) assert str(a.ncores) in repr(a) @gen_cluster(ncores=[], timeout=20) def test_nanny_process_failure(s): n = Nanny(s.ip, s.port, ncores=2, ip='127.0.0.1', loop=s.loop) yield n._start() first_dir = n.worker_dir assert os.path.exists(first_dir) original_process = n.process ww = rpc(ip=n.ip, port=n.worker_port) yield ww.update_data(data=valmap(dumps, {'x': 1, 'y': 2})) with ignoring(StreamClosedError): yield ww.compute(function=dumps(sys.exit), args=dumps((0,)), key='z') start = time() while n.process is original_process: # wait while process dies yield gen.sleep(0.01) assert time() - start < 5 start = time() while not isalive(n.process): # wait while process comes back yield gen.sleep(0.01) assert time() - start < 5 start = time() while n.worker_address not in s.ncores or n.worker_dir is None: yield gen.sleep(0.01) assert time() - start < 5 second_dir = n.worker_dir yield n._close() assert not os.path.exists(second_dir) assert not os.path.exists(first_dir) assert first_dir != n.worker_dir ww.close_rpc() s.stop() @gen_cluster(ncores=[]) def test_monitor_resources(s): pytest.importorskip('psutil') n = Nanny(s.ip, s.port, ncores=2, ip='127.0.0.1', loop=s.loop) yield n._start() assert isalive(n.process) d = n.resource_collect() assert {'cpu_percent', 'memory_percent'}.issubset(d) assert 'timestamp' in d stream = yield connect(ip=n.ip, port=n.port) yield write(stream, {'op': 'monitor_resources', 'interval': 0.01}) for i in range(3): msg = yield read(stream) assert isinstance(msg, dict) assert {'cpu_percent', 'memory_percent'}.issubset(msg) stream.close() yield n._close() s.stop() @gen_cluster(ncores=[]) def test_run(s): pytest.importorskip('psutil') n = Nanny(s.ip, s.port, ncores=2, ip='127.0.0.1', loop=s.loop) yield n._start() with rpc(n.address) as nn: response = yield nn.run(function=dumps(lambda: 1)) assert response['status'] == 'OK' assert response['result'] == 1 yield n._close() distributed-1.14.3/distributed/tests/test_scheduler.py000066400000000000000000001016451301211143400231720ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import cloudpickle from collections import defaultdict, deque from copy import deepcopy from datetime import timedelta from operator import add import sys from time import time, sleep import dask from dask import delayed from dask.core import get_deps from toolz import merge, concat, valmap, first from tornado.queues import Queue from tornado.iostream import StreamClosedError from tornado.gen import TimeoutError from tornado import gen import pytest from distributed import Nanny, Worker from distributed.batched import BatchedStream from distributed.core import connect, read, write, rpc from distributed.scheduler import (validate_state, decide_worker, Scheduler) from distributed.client import _wait from distributed.protocol.pickle import dumps from distributed.worker import dumps_function, dumps_task from distributed.utils_test import (inc, ignoring, dec, gen_cluster, gen_test, loop) from distributed.utils import All from dask.compatibility import apply alice = 'alice:1234' bob = 'bob:1234' stack_duration = defaultdict(lambda: 0) @gen_cluster() def test_administration(s, a, b): assert isinstance(s.address, str) assert s.address_tuple[0] in s.address assert s.address in str(s) assert str(sum(s.ncores.values())) in repr(s) assert str(len(s.ncores)) in repr(s) @gen_cluster(ncores=[]) def test_update_state(s): s.add_worker(address=alice, ncores=1, coerce_address=False) s.update_graph(tasks={'x': 1, 'y': dumps_task((inc, 'x'))}, keys=['y'], dependencies={'y': 'x', 'x': set()}, client='client') s.ensure_occupied() r = s.transition('x', 'memory', nbytes=10, type=dumps(int), compute_start=10, compute_stop=11, worker=alice) s.transitions(r) s.ensure_occupied() assert set(s.processing[alice]) == {'y'} assert set(s.rprocessing['y']) == {alice} assert not s.ready assert s.who_wants == {'y': {'client'}} assert s.wants_what == {'client': {'y'}} s.update_graph(tasks={'a': 1, 'z': dumps_task((add, 'y', 'a'))}, keys=['z'], dependencies={'z': {'y', 'a'}}, client='client') assert s.tasks == {'x': 1, 'y': dumps_task((inc, 'x')), 'a': 1, 'z': dumps_task((add, 'y', 'a'))} assert s.dependencies == {'x': set(), 'a': set(), 'y': {'x'}, 'z': {'a', 'y'}} assert s.dependents == {'z': set(), 'y': {'z'}, 'a': {'z'}, 'x': {'y'}} assert s.waiting == {'z': {'a', 'y'}} assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'a': {'z'}, 'z': set()} assert s.who_wants == {'z': {'client'}, 'y': {'client'}} assert s.wants_what == {'client': {'y', 'z'}} assert 'a' in s.ready or 'a' in s.processing[alice] @gen_cluster(ncores=[]) def test_update_state_with_processing(s): s.add_worker(address=alice, ncores=1, coerce_address=False) s.update_graph(tasks={'x': 1, 'y': dumps_task((inc, 'x')), 'z': dumps_task((inc, 'y'))}, keys=['z'], dependencies={'y': {'x'}, 'x': set(), 'z': {'y'}}, client='client') s.ensure_occupied() r = s.transition('x', 'memory', nbytes=10, type=dumps(int), compute_start=10, compute_stop=11, worker=alice) s.transitions(r) s.ensure_occupied() assert s.waiting == {'z': {'y'}} assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'z': set()} assert list(s.ready) == [] assert s.who_wants == {'z': {'client'}} assert s.wants_what == {'client': {'z'}} assert s.who_has == {'x': {alice}} s.update_graph(tasks={'a': dumps_task((inc, 'x')), 'b': (add,'a','y'), 'c': dumps_task((inc, 'z'))}, keys=['b', 'c'], dependencies={'a': {'x'}, 'b': {'a', 'y'}, 'c': {'z'}}, client='client') assert s.waiting == {'z': {'y'}, 'b': {'a', 'y'}, 'c': {'z'}} assert 'a' in s.stacks[alice] or 'a' in s.processing[alice] assert not s.ready assert s.waiting_data == {'x': {'y', 'a'}, 'y': {'z', 'b'}, 'z': {'c'}, 'a': {'b'}, 'b': set(), 'c': set()} assert s.who_wants == {'b': {'client'}, 'c': {'client'}, 'z': {'client'}} assert s.wants_what == {'client': {'b', 'c', 'z'}} @gen_cluster(client=True, ncores=[('127.0.0.1', 1)]) def test_update_state_respects_data_in_memory(c, s, a): x = delayed(inc)(1) y = delayed(inc)(x) f = c.persist(y) yield _wait([f]) assert s.released == {x.key} assert s.who_has == {y.key: {a.address}} z = delayed(add)(x, y) f2 = c.persist(z) while not f2.key in s.who_has: assert y.key in s.who_has yield gen.sleep(0.0001) @gen_cluster(ncores=[]) def test_update_state_supports_recomputing_released_results(s): s.add_worker(address=alice, ncores=1, coerce_address=False) s.update_graph(tasks={'x': 1, 'y': dumps_task((inc, 'x')), 'z': dumps_task((inc, 'x'))}, keys=['z'], dependencies={'y': {'x'}, 'x': set(), 'z': {'y'}}, client='client') s.ensure_occupied() r = s.transition('x', 'memory', nbytes=10, type=dumps(int), compute_start=10, compute_stop=11, worker=alice) s.transitions(r) s.ensure_occupied() r = s.transition('y', 'memory', nbytes=10, type=dumps(int), compute_start=10, compute_stop=11, worker=alice) s.transitions(r) s.ensure_occupied() r = s.transition('z', 'memory', nbytes=10, type=dumps(int), compute_start=10, compute_stop=11, worker=alice) s.transitions(r) s.ensure_occupied() assert not s.waiting assert not s.ready assert s.waiting_data == {'z': set()} assert s.who_has == {'z': {alice}} s.update_graph(tasks={'x': 1, 'y': dumps_task((inc, 'x'))}, keys=['y'], dependencies={'y': {'x'}, 'x': set()}, client='client') assert s.waiting == {'y': {'x'}} assert s.waiting_data == {'x': {'y'}, 'y': set(), 'z': set()} assert s.who_wants == {'z': {'client'}, 'y': {'client'}} assert s.wants_what == {'client': {'y', 'z'}} assert set(s.processing[alice]) == {'x'} def test_decide_worker_with_many_independent_leaves(): dsk = merge({('y', i): (inc, ('x', i)) for i in range(100)}, {('x', i): i for i in range(100)}) dependencies, dependents = get_deps(dsk) stacks = {alice: [], bob: []} processing = {alice: dict(), bob: dict()} who_has = merge({('x', i * 2): {alice} for i in range(50)}, {('x', i * 2 + 1): {bob} for i in range(50)}) nbytes = {k: 0 for k in who_has} ncores = {alice: 1, bob: 1} for key in dsk: worker = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, {}, set(), nbytes, ncores, key) stacks[worker].append(key) nhits = (len([k for k in stacks[alice] if alice in who_has[('x', k[1])]]) + len([k for k in stacks[bob] if bob in who_has[('x', k[1])]])) assert nhits > 90 def test_decide_worker_with_restrictions(): dependencies = {'x': set()} alice, bob, charlie = 'alice:8000', 'bob:8000', 'charlie:8000' stacks = {alice: [], bob: [], charlie: []} processing = {alice: dict(), bob: dict(), charlie: dict()} who_has = {} restrictions = {'x': {'alice', 'charlie'}} nbytes = {} ncores = {alice: 1, bob: 1, charlie: 1} result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, set(), nbytes, ncores, 'x') assert result in {alice, charlie} stacks = {alice: [1, 2, 3], bob: [], charlie: [4, 5, 6]} result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, set(), nbytes, ncores, 'x') assert result in {alice, charlie} dependencies = {'x': {'y'}} who_has = {'y': {bob}} nbytes = {'y': 0} result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, set(), nbytes, ncores, 'x') assert result in {alice, charlie} def test_decide_worker_with_loose_restrictions(): dependencies = {'x': set()} alice, bob, charlie = 'alice:8000', 'bob:8000', 'charlie:8000' stacks = {alice: [1, 2, 3], bob: [], charlie: [1]} stack_duration = {alice: 3, bob: 0, charlie: 1} processing = {alice: dict(), bob: dict(), charlie: dict()} who_has = {} nbytes = {} ncores = {alice: 1, bob: 1, charlie: 1} restrictions = {'x': {'alice', 'charlie'}} result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, set(), nbytes, ncores, 'x') assert result == charlie result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, {'x'}, nbytes, ncores, 'x') assert result == charlie restrictions = {'x': {'david', 'ethel'}} result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, set(), nbytes, ncores, 'x') assert result is None restrictions = {'x': {'david', 'ethel'}} result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, {'x'}, nbytes, ncores, 'x') assert result == bob def test_decide_worker_without_stacks(): assert not decide_worker({'x': []}, {}, {}, {}, {}, {}, {}, set(), {}, {}, 'x') def test_validate_state(): dsk = {'x': 1, 'y': (inc, 'x')} dependencies = {'x': set(), 'y': {'x'}} waiting = {'y': {'x'}} ready = deque(['x']) dependents = {'x': {'y'}, 'y': set()} waiting_data = {'x': {'y'}} who_has = dict() stacks = {alice: [], bob: []} processing = {alice: dict(), bob: dict()} finished_results = set() released = set() in_play = {'x', 'y'} who_wants = {'y': {'client'}} wants_what = {'client': {'y'}} erred = {} validate_state(**locals()) who_has['x'] = {alice} with pytest.raises(Exception): validate_state(**locals()) ready.remove('x') with pytest.raises(Exception): validate_state(**locals()) waiting['y'].remove('x') with pytest.raises(Exception): validate_state(**locals()) del waiting['y'] ready.appendleft('y') validate_state(**locals()) stacks[alice].append('y') with pytest.raises(Exception): validate_state(**locals()) ready.remove('y') validate_state(**locals()) stacks[alice].pop() with pytest.raises(Exception): validate_state(**locals()) processing[alice]['y'] = 1 validate_state(**locals()) del processing[alice]['y'] with pytest.raises(Exception): validate_state(**locals()) who_has['y'] = {alice} with pytest.raises(Exception): validate_state(**locals()) finished_results.add('y') with pytest.raises(Exception): validate_state(**locals()) waiting_data.pop('x') who_has.pop('x') released.add('x') validate_state(**locals()) def div(x, y): return x / y @gen_cluster() def test_scheduler(s, a, b): stream = yield connect(s.ip, s.port) yield write(stream, {'op': 'register-client', 'client': 'ident'}) stream = BatchedStream(stream, 10) msg = yield read(stream) assert msg['op'] == 'stream-start' # Test update graph yield write(stream, {'op': 'update-graph', 'tasks': valmap(dumps_task, {'x': (inc, 1), 'y': (inc, 'x'), 'z': (inc, 'y')}), 'dependencies': {'x': [], 'y': ['x'], 'z': ['y']}, 'keys': ['x', 'z'], 'client': 'ident'}) while True: msg = yield read(stream) if msg['op'] == 'key-in-memory' and msg['key'] == 'z': break assert a.data.get('x') == 2 or b.data.get('x') == 2 # Test erring tasks yield write(stream, {'op': 'update-graph', 'tasks': valmap(dumps_task, {'a': (div, 1, 0), 'b': (inc, 'a')}), 'dependencies': {'a': [], 'b': ['a']}, 'keys': ['a', 'b'], 'client': 'ident'}) while True: msg = yield read(stream) if msg['op'] == 'task-erred' and msg['key'] == 'b': break # Test missing data yield write(stream, {'op': 'missing-data', 'keys': ['z']}) s.ensure_occupied() while True: msg = yield read(stream) if msg['op'] == 'key-in-memory' and msg['key'] == 'z': break # Test missing data without being informed for w in [a, b]: if 'z' in w.data: del w.data['z'] yield write(stream, {'op': 'update-graph', 'tasks': {'zz': dumps_task((inc, 'z'))}, 'dependencies': {'zz': ['z']}, 'keys': ['zz'], 'client': 'ident'}) while True: msg = yield read(stream) if msg['op'] == 'key-in-memory' and msg['key'] == 'zz': break write(stream, {'op': 'close'}) stream.close() @gen_cluster() def test_multi_queues(s, a, b): sched, report = Queue(), Queue() s.handle_queues(sched, report) msg = yield report.get() assert msg['op'] == 'stream-start' # Test update graph sched.put_nowait({'op': 'update-graph', 'tasks': valmap(dumps_task, {'x': (inc, 1), 'y': (inc, 'x'), 'z': (inc, 'y')}), 'dependencies': {'x': [], 'y': ['x'], 'z': ['y']}, 'keys': ['z']}) while True: msg = yield report.get() if msg['op'] == 'key-in-memory' and msg['key'] == 'z': break slen, rlen = len(s.scheduler_queues), len(s.report_queues) sched2, report2 = Queue(), Queue() s.handle_queues(sched2, report2) assert slen + 1 == len(s.scheduler_queues) assert rlen + 1 == len(s.report_queues) sched2.put_nowait({'op': 'update-graph', 'tasks': {'a': dumps_task((inc, 10))}, 'dependencies': {'a': []}, 'keys': ['a']}) for q in [report, report2]: while True: msg = yield q.get() if msg['op'] == 'key-in-memory' and msg['key'] == 'a': break @gen_cluster() def test_server(s, a, b): stream = yield connect('127.0.0.1', s.port) yield write(stream, {'op': 'register-client', 'client': 'ident'}) stream = BatchedStream(stream, 0) stream.send({'op': 'update-graph', 'tasks': {'x': dumps_task((inc, 1)), 'y': dumps_task((inc, 'x'))}, 'dependencies': {'x': [], 'y': ['x']}, 'keys': ['y'], 'client': 'ident'}) while True: msg = yield read(stream) if msg['op'] == 'key-in-memory' and msg['key'] == 'y': break stream.send({'op': 'close-stream'}) msg = yield read(stream) assert msg == {'op': 'stream-closed'} assert stream.closed() stream.close() @gen_cluster() def test_remove_client(s, a, b): s.add_client(client='ident') s.update_graph(tasks={'x': dumps_task((inc, 1)), 'y': dumps_task((inc, 'x'))}, dependencies={'x': [], 'y': ['x']}, keys=['y'], client='ident') assert s.tasks assert s.dependencies s.remove_client(client='ident') assert not s.tasks assert not s.dependencies @gen_cluster() def test_server_listens_to_other_ops(s, a, b): with rpc(ip='127.0.0.1', port=s.port) as r: ident = yield r.identity() assert ident['type'] == 'Scheduler' @gen_cluster() def test_remove_worker_from_scheduler(s, a, b): dsk = {('x', i): (inc, i) for i in range(20)} s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), dependencies={k: set() for k in dsk}) assert s.ready assert not any(stack for stack in s.stacks.values()) assert a.address in s.worker_streams s.remove_worker(address=a.address) assert a.address not in s.ncores assert len(s.ready) + len(s.processing[b.address]) == \ len(dsk) # b owns everything s.validate_state() @gen_cluster() def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') w.data['x-5'] = 6 w.data['y'] = 1 yield w._start(0) dsk = {('x-%d' % i).encode(): (inc, i) for i in range(10)} s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client', dependencies={k: set() for k in dsk}) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, services=s.services, coerce_address=False) s.validate_state() assert w.ip in s.host_info assert s.host_info[w.ip]['ports'] == set(map(str, [a.port, b.port, w.port])) yield w._close() @gen_cluster() def test_feed(s, a, b): def func(scheduler): return dumps((scheduler.processing, scheduler.stacks)) stream = yield connect(s.ip, s.port) yield write(stream, {'op': 'feed', 'function': dumps(func), 'interval': 0.01}) for i in range(5): response = yield read(stream) expected = s.processing, s.stacks assert cloudpickle.loads(response) == expected stream.close() @gen_cluster() def test_feed_setup_teardown(s, a, b): def setup(scheduler): return 1 def func(scheduler, state): assert state == 1 return 'OK' def teardown(scheduler, state): scheduler.flag = 'done' stream = yield connect(s.ip, s.port) yield write(stream, {'op': 'feed', 'function': dumps(func), 'setup': dumps(setup), 'teardown': dumps(teardown), 'interval': 0.01}) for i in range(5): response = yield read(stream) assert response == 'OK' stream.close() start = time() while not hasattr(s, 'flag'): yield gen.sleep(0.01) assert time() - start < 5 @gen_cluster() def test_feed_large_bytestring(s, a, b): np = pytest.importorskip('numpy') x = np.ones(10000000) def func(scheduler): y = x return True stream = yield connect(s.ip, s.port) yield write(stream, {'op': 'feed', 'function': dumps(func), 'interval': 0.01}) for i in range(5): response = yield read(stream) assert response == True stream.close() @gen_test(timeout=None) def test_scheduler_as_center(): s = Scheduler(validate=True) done = s.start(0) a = Worker('127.0.0.1', s.port, ip='127.0.0.1', ncores=1) a.data.update({'x': 1, 'y': 2}) b = Worker('127.0.0.1', s.port, ip='127.0.0.1', ncores=2) b.data.update({'y': 2, 'z': 3}) c = Worker('127.0.0.1', s.port, ip='127.0.0.1', ncores=3) yield [w._start(0) for w in [a, b, c]] assert s.ncores == {w.address: w.ncores for w in [a, b, c]} assert not s.who_has s.update_graph(tasks={'a': dumps_task((inc, 1))}, keys=['a'], dependencies={'a': []}) start = time() while not 'a' in s.who_has: assert time() - start < 5 yield gen.sleep(0.01) assert 'a' in a.data or 'a' in b.data or 'a' in c.data with ignoring(StreamClosedError): yield [w._close() for w in [a, b, c]] assert s.ncores == {} assert s.who_has == {} yield s.close() @gen_cluster(client=True) def test_delete_data(c, s, a, b): d = yield c._scatter({'x': 1, 'y': 2, 'z': 3}) assert set(s.who_has) == {'x', 'y', 'z'} assert set(a.data) | set(b.data) == {'x', 'y', 'z'} assert merge(a.data, b.data) == {'x': 1, 'y': 2, 'z': 3} del d['x'] del d['y'] start = time() while set(a.data) | set(b.data) != {'z'}: yield gen.sleep(0.01) assert time() < start + 5 @gen_cluster(client=True) def test_delete_callback(c, s, a, b): d = yield c._scatter({'x': 1}, workers=a.address) assert s.who_has['x'] == {a.address} assert s.has_what[a.address] == {'x'} s.client_releases_keys(client=c.id, keys=['x']) assert 'x' not in s.who_has assert 'x' not in s.has_what[a.address] assert a.data['x'] == 1 # still in memory assert s.deleted_keys == {a.address: {'x'}} yield s.clear_data_from_workers() assert not s.deleted_keys assert not a.data assert s._delete_periodic_callback.is_running() @gen_cluster() def test_self_aliases(s, a, b): a.data['a'] = 1 s.update_data(who_has={'a': [a.address]}, nbytes={'a': 10}, client='client') s.update_graph(tasks=valmap(dumps_task, {'a': 'a', 'b': (inc, 'a')}), keys=['b'], client='client', dependencies={'b': ['a']}) sched, report = Queue(), Queue() s.handle_queues(sched, report) msg = yield report.get() assert msg['op'] == 'stream-start' while True: msg = yield report.get() if msg['op'] == 'key-in-memory' and msg['key'] == 'b': break @gen_cluster() def test_filtered_communication(s, a, b): c = yield connect(ip=s.ip, port=s.port) f = yield connect(ip=s.ip, port=s.port) yield write(c, {'op': 'register-client', 'client': 'c'}) yield write(f, {'op': 'register-client', 'client': 'f'}) yield read(c) yield read(f) c = BatchedStream(c, 0) f = BatchedStream(f, 0) assert set(s.streams) == {'c', 'f'} yield write(c, {'op': 'update-graph', 'tasks': {'x': dumps_task((inc, 1)), 'y': dumps_task((inc, 'x'))}, 'dependencies': {'x': [], 'y': ['x']}, 'client': 'c', 'keys': ['y']}) yield write(f, {'op': 'update-graph', 'tasks': {'x': dumps_task((inc, 1)), 'z': dumps_task((add, 'x', 10))}, 'dependencies': {'x': [], 'z': ['x']}, 'client': 'f', 'keys': ['z']}) msg = yield read(c) assert msg['op'] == 'key-in-memory' assert msg['key'] == 'y' msg = yield read(f) assert msg['op'] == 'key-in-memory' assert msg['key'] == 'z' def test_dumps_function(): a = dumps_function(inc) assert cloudpickle.loads(a)(10) == 11 b = dumps_function(inc) assert a is b c = dumps_function(dec) assert a != c def test_dumps_task(): d = dumps_task((inc, 1)) assert set(d) == {'function', 'args'} f = lambda x, y=2: x + y d = dumps_task((apply, f, (1,), {'y': 10})) assert cloudpickle.loads(d['function'])(1, 2) == 3 assert cloudpickle.loads(d['args']) == (1,) assert cloudpickle.loads(d['kwargs']) == {'y': 10} d = dumps_task((apply, f, (1,))) assert cloudpickle.loads(d['function'])(1, 2) == 3 assert cloudpickle.loads(d['args']) == (1,) assert set(d) == {'function', 'args'} @gen_cluster() def test_ready_remove_worker(s, a, b): s.add_client(client='client') s.update_graph(tasks={'x-%d' % i: dumps_task((inc, i)) for i in range(20)}, keys=['x-%d' % i for i in range(20)], client='client', dependencies={'x-%d' % i: [] for i in range(20)}) assert all(len(s.processing[w]) >= s.ncores[w] for w in s.ncores) assert not any(stack for stack in s.stacks.values()) assert len(s.ready) + sum(map(len, s.processing.values())) == 20 s.remove_worker(address=a.address) for collection in [s.ncores, s.stacks, s.processing]: assert set(collection) == {b.address} assert all(len(s.processing[w]) >= s.ncores[w] for w in s.ncores) assert set(s.processing) == {b.address} assert not any(stack for stack in s.stacks.values()) assert len(s.ready) + sum(map(len, s.processing.values())) == 20 @gen_cluster(Worker=Nanny) def test_restart(s, a, b): s.add_client(client='client') s.update_graph(tasks={'x-%d' % i: dumps_task((inc, i)) for i in range(20)}, keys=['x-%d' % i for i in range(20)], client='client', dependencies={'x-%d' % i: [] for i in range(20)}) assert len(s.ready) + sum(map(len, s.processing.values())) == 20 assert s.ready yield s.restart() for c in [s.stacks, s.processing, s.ncores]: assert len(c) == 2 for c in [s.stacks, s.processing]: assert not any(v for v in c.values()) assert not s.ready assert not s.tasks assert not s.dependencies @gen_cluster() def test_ready_add_worker(s, a, b): s.add_client(client='client') s.update_graph(tasks={'x-%d' % i: dumps_task((inc, i)) for i in range(20)}, keys=['x-%d' % i for i in range(20)], client='client', dependencies={'x-%d' % i: [] for i in range(20)}) assert all(len(s.processing[w]) == s.ncores[w] for w in s.ncores) assert len(s.ready) + sum(map(len, s.processing.values())) == 20 w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') w.listen(0) s.add_worker(address=w.address, ncores=w.ncores, coerce_address=False) assert w.address in s.ncores assert all(len(s.processing[w]) == s.ncores[w] for w in s.ncores) assert len(s.ready) + sum(map(len, s.processing.values())) == 20 w.scheduler.close_rpc() @gen_cluster() def test_broadcast(s, a, b): result = yield s.broadcast(msg={'op': 'ping'}) assert result == {a.address: b'pong', b.address: b'pong'} result = yield s.broadcast(msg={'op': 'ping'}, workers=[a.address]) assert result == {a.address: b'pong'} result = yield s.broadcast(msg={'op': 'ping'}, hosts=[a.ip]) assert result == {a.address: b'pong', b.address: b'pong'} @gen_cluster(Worker=Nanny) def test_broadcast_nanny(s, a, b): result1 = yield s.broadcast(msg={'op': 'identity'}, nanny=True) assert all(d['type'] == 'Nanny' for d in result1.values()) result2 = yield s.broadcast(msg={'op': 'identity'}, workers=[a.worker_address], nanny=True) assert len(result2) == 1 assert first(result2.values())['id'] == a.id result3 = yield s.broadcast(msg={'op': 'identity'}, hosts=[a.ip], nanny=True) assert result1 == result3 @gen_test() def test_worker_name(): s = Scheduler(validate=True) s.start(0) w = Worker(s.ip, s.port, name='alice') yield w._start() assert s.worker_info[w.address]['name'] == 'alice' assert s.aliases['alice'] == w.address with pytest.raises(ValueError): w = Worker(s.ip, s.port, name='alice') yield w._start() yield s.close() yield w._close() @gen_test() def test_coerce_address(): s = Scheduler(validate=True) s.start(0) a = Worker(s.ip, s.port, name='alice') b = Worker(s.ip, s.port, name=123) c = Worker(s.ip, s.port, name='charlie', ip='127.0.0.2') yield [a._start(), b._start(), c._start()] assert s.coerce_address(b'127.0.0.1') == '127.0.0.1' assert s.coerce_address(('127.0.0.1', 8000)) == '127.0.0.1:8000' assert s.coerce_address(['127.0.0.1', 8000]) == '127.0.0.1:8000' assert s.coerce_address([b'127.0.0.1', 8000]) == '127.0.0.1:8000' assert s.coerce_address(('127.0.0.1', '8000')) == '127.0.0.1:8000' assert s.coerce_address(b'localhost') == '127.0.0.1' assert s.coerce_address('localhost') == '127.0.0.1' assert s.coerce_address(u'localhost') == '127.0.0.1' assert s.coerce_address('localhost:8000') == '127.0.0.1:8000' assert s.coerce_address(a.address) == a.address assert s.coerce_address(a.address_tuple) == a.address assert s.coerce_address(123) == b.address assert s.coerce_address('charlie') == c.address yield s.close() yield [w._close() for w in [a, b, c]] @pytest.mark.skipif(sys.platform.startswith('win'), reason="file descriptors not really a thing") @gen_cluster(ncores=[]) def test_file_descriptors_dont_leak(s): psutil = pytest.importorskip('psutil') proc = psutil.Process() before = proc.num_fds() w = Worker(s.ip, s.port) yield w._start(0) yield w._close() during = proc.num_fds() start = time() while proc.num_fds() > before: yield gen.sleep(0.01) assert time() < start + 5 @gen_cluster() def test_update_graph_culls(s, a, b): s.add_client(client='client') s.update_graph(tasks={'x': dumps_task((inc, 1)), 'y': dumps_task((inc, 'x')), 'z': dumps_task((inc, 2))}, keys=['y'], dependencies={'y': 'x', 'x': [], 'z': []}, client='client') assert 'z' not in s.tasks assert 'z' not in s.dependencies @pytest.mark.skipif(not sys.platform.startswith('linux'), reason="Need 127.0.0.2 to mean localhost") @gen_cluster(ncores=[('127.0.0.1', 2), ('127.0.0.2', 2), ('127.0.0.1', 1)]) def test_host_health(s, a, b, c): start = time() for w in [a, b, c]: assert w.ip in s.host_info assert 0 <= s.host_info[w.ip]['cpu'] <= 100 assert 0 < s.host_info[w.ip]['memory'] assert 0 < s.host_info[w.ip]['memory_percent'] < 100 assert isinstance(s.host_info[w.ip]['last-seen'], (int, float)) assert -1 < s.worker_info[w.address]['time-delay'] < 1 assert set(s.host_info) == {'127.0.0.1', '127.0.0.2'} assert s.host_info['127.0.0.1']['cores'] == 3 assert s.host_info['127.0.0.1']['ports'] == {str(a.port), str(c.port)} assert s.host_info['127.0.0.2']['cores'] == 2 assert s.host_info['127.0.0.2']['ports'] == {str(b.port)} s.remove_worker(address=a.address) assert set(s.host_info) == {'127.0.0.1', '127.0.0.2'} assert s.host_info['127.0.0.1']['cores'] == 1 assert s.host_info['127.0.0.1']['ports'] == {str(c.port)} assert s.host_info['127.0.0.2']['cores'] == 2 assert s.host_info['127.0.0.2']['ports'] == {str(b.port)} s.remove_worker(address=b.address) assert set(s.host_info) == {'127.0.0.1'} assert s.host_info['127.0.0.1']['cores'] == 1 assert s.host_info['127.0.0.1']['ports'] == {str(c.port)} s.remove_worker(address=c.address) assert not s.host_info @gen_cluster(ncores=[]) def test_add_worker_is_idempotent(s): s.add_worker(address=alice, ncores=1, coerce_address=False) ncores = s.ncores.copy() s.add_worker(address=alice, coerce_address=False) assert s.ncores == s.ncores def test_io_loop(loop): s = Scheduler(loop=loop, validate=True) assert s.io_loop is loop @gen_cluster(client=True) def test_transition_story(c, s, a, b): x = delayed(inc)(1) y = delayed(inc)(x) f = c.persist(y) yield _wait([f]) assert s.transition_log story = s.transition_story(x.key) assert all(line in s.transition_log for line in story) assert len(story) < len(s.transition_log) assert all(x.key == line[0] or x.key in line[-1] for line in story) assert len(s.transition_story(x.key, y.key)) > len(story) @gen_test() def test_launch_without_blocked_services(): from distributed.http import HTTPScheduler s = Scheduler(services={('http', 3849): HTTPScheduler}) s.start(0) s2 = Scheduler(services={('http', 3849): HTTPScheduler}) s2.start(0) assert not s2.services yield [s.close(), s2.close()] @gen_cluster(ncores=[], client=True) def test_scatter_no_workers(c, s): with pytest.raises(gen.TimeoutError): yield gen.with_timeout(timedelta(seconds=0.1), s.scatter(data={'x': dumps(1)}, client='alice')) w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') yield [c._scatter(data={'x': 1}), w._start()] assert w.data['x'] == 1 yield w._close() @gen_cluster(ncores=[]) def test_scheduler_sees_memory_limits(s): w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1', memory_limit=12345) yield w._start(0) assert s.worker_info[w.address]['memory_limit'] == 12345 yield w._close() @gen_cluster(client=True, timeout=1000) def test_retire_workers(c, s, a, b): [x] = yield c._scatter([1], workers=a.address) [y] = yield c._scatter([list(range(1000))], workers=b.address) assert s.workers_to_close() == [a.address] workers = yield s.retire_workers() assert workers == [a.address] assert list(s.ncores) == [b.address] assert s.workers_to_close() == [] assert s.has_what[b.address] == {x.key, y.key} workers = yield s.retire_workers() assert not workers distributed-1.14.3/distributed/tests/test_sizeof.py000066400000000000000000000016461301211143400225130ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import sys import pytest from distributed.sizeof import sizeof, getsizeof def test_base(): assert sizeof(1) == getsizeof(1) def test_containers(): assert sizeof([1, 2, [3]]) > (getsizeof(3) * 3 + getsizeof([])) def test_numpy(): np = pytest.importorskip('numpy') assert sizeof(np.empty(1000, dtype='f8')) >= 8000 def test_pandas(): pd = pytest.importorskip('pandas') df = pd.DataFrame({'x': [1, 2, 3], 'y': ['a'*1000, 'b'*1000, 'c'*1000]}, index=[10, 20, 30]) assert sizeof(df) >= sizeof(df.x) + sizeof(df.y) - sizeof(df.index) assert sizeof(df.x) >= sizeof(df.index) if pd.__version__ >= '0.17.1': assert sizeof(df.y) >= 1000 * 3 assert sizeof(df.index) >= 20 assert isinstance(sizeof(df), int) assert isinstance(sizeof(df.x), int) assert isinstance(sizeof(df.index), int) distributed-1.14.3/distributed/tests/test_steal.py000066400000000000000000000076771301211143400223360ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import sys import pytest from toolz import sliding_window from tornado import gen import dask from dask import delayed from distributed.client import Client, _wait, wait from distributed.utils_test import (cluster, slowinc, slowadd, randominc, loop, inc, dec, div, throws, gen_cluster, gen_test, double, deep) import pytest @pytest.mark.skipif(not sys.platform.startswith('linux'), reason="Need 127.0.0.2 to mean localhost") @gen_cluster(client=True, ncores=[('127.0.0.1', 2), ('127.0.0.2', 2)], timeout=20) def test_work_stealing(c, s, a, b): [x] = yield c._scatter([1]) futures = c.map(slowadd, range(50), [x] * 50) yield _wait(futures) assert len(a.data) > 10 assert len(b.data) > 10 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2) def test_dont_steal_expensive_data_fast_computation(c, s, a, b): np = pytest.importorskip('numpy') x = c.submit(np.arange, 1000000, workers=a.address) yield _wait([x]) future = c.submit(np.sum, [1], workers=a.address) # learn that sum is fast yield _wait([future]) cheap = [c.submit(np.sum, x, pure=False, workers=a.address, allow_other_workers=True) for i in range(10)] yield _wait(cheap) assert len(b.data) == 0 assert len(a.data) == 12 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2) def test_steal_cheap_data_slow_computation(c, s, a, b): x = c.submit(slowinc, 100, delay=0.1) # learn that slowinc is slow yield _wait([x]) futures = c.map(slowinc, range(10), delay=0.01, workers=a.address, allow_other_workers=True) yield _wait(futures) assert abs(len(a.data) - len(b.data)) <= 3 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2) def test_steal_expensive_data_slow_computation(c, s, a, b): np = pytest.importorskip('numpy') x = c.submit(slowinc, 100, delay=0.1, workers=a.address) yield _wait([x]) # learn that slowinc is slow x = c.submit(np.arange, 1000000, workers=a.address) # put expensive data yield _wait([x]) slow = [c.submit(slowinc, x, delay=0.1, pure=False) for i in range(4)] yield _wait([slow]) assert b.data # not empty @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10) def test_worksteal_many_thieves(c, s, *workers): x = c.submit(slowinc, -1, delay=0.1) yield x._result() xs = c.map(slowinc, [x] * 100, pure=False, delay=0.01) yield _wait(xs) for w, keys in s.has_what.items(): assert 2 < len(keys) < 50 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2) def test_dont_steal_unknown_functions(c, s, a, b): futures = c.map(inc, [1, 2], workers=a.address, allow_other_workers=True) yield _wait(futures) assert len(a.data) == 2 assert len(b.data) == 0 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2) def test_eventually_steal_unknown_functions(c, s, a, b): futures = c.map(slowinc, range(10), delay=0.1, workers=a.address, allow_other_workers=True) yield _wait(futures) assert len(a.data) >= 3 assert len(b.data) >= 3 @pytest.mark.xfail(reason='') @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3) def test_steal_related_tasks(e, s, a, b, c): futures = e.map(slowinc, range(20), delay=0.05, workers=a.address, allow_other_workers=True) yield _wait(futures) nearby = 0 for f1, f2 in sliding_window(2, futures): if s.who_has[f1.key] == s.who_has[f2.key]: nearby += 1 assert nearby > 10 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10, timeout=1000) def test_dont_steal_fast_tasks(c, s, *workers): np = pytest.importorskip('numpy') x = c.submit(np.random.random, 10000000, workers=workers[0].address) def do_nothing(x, y=None): pass futures = c.map(do_nothing, range(1000), y=x) yield _wait(futures) assert len(s.has_what[workers[0].address]) == 1001 distributed-1.14.3/distributed/tests/test_stress.py000066400000000000000000000111071301211143400225300ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from concurrent.futures import CancelledError from datetime import timedelta from operator import add from time import sleep, time from dask import delayed import pytest from toolz import concat, sliding_window from distributed import Client, wait, Nanny from distributed.utils import All from distributed.utils_test import (gen_cluster, cluster, inc, slowinc, loop, slowadd, slow) from distributed.client import _wait from tornado import gen @gen_cluster(client=True) def test_stress_1(c, s, a, b): n = 2**6 seq = c.map(inc, range(n)) while len(seq) > 1: yield gen.sleep(0.1) seq = [c.submit(add, seq[i], seq[i + 1]) for i in range(0, len(seq), 2)] result = yield seq[0]._result() assert result == sum(map(inc, range(n))) @pytest.mark.parametrize(('func', 'n'), [(slowinc, 100), (inc, 1000)]) def test_stress_gc(loop, func, n): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = c.submit(func, 1) for i in range(n): x = c.submit(func, x) assert x.result() == n + 2 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4, timeout=None) def test_cancel_stress(c, s, *workers): da = pytest.importorskip('dask.array') x = da.random.random((40, 40), chunks=(1, 1)) x = c.persist(x) yield _wait([x]) y = (x.sum(axis=0) + x.sum(axis=1) + 1).std() for i in range(5): f = c.compute(y) while len(s.waiting) > (len(y.dask) - len(x.dask)) / 2: yield gen.sleep(0.01) yield c._cancel(f) def test_cancel_stress_sync(loop): da = pytest.importorskip('dask.array') x = da.random.random((40, 40), chunks=(1, 1)) with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = c.persist(x) y = (x.sum(axis=0) + x.sum(axis=1) + 1).std() wait(x) for i in range(5): f = c.compute(y) sleep(1) c.cancel(f) @gen_cluster(ncores=[], client=True, timeout=None) def test_stress_creation_and_deletion(c, s): # Assertions are handled by the validate mechanism in the scheduler s.allowed_failures = 100000 da = pytest.importorskip('dask.array') x = da.random.random(size=(2000, 2000), chunks=(100, 100)) y = (x + 1).T + (x * 2) - x.mean(axis=1) z = c.persist(y) @gen.coroutine def create_and_destroy_worker(delay): start = time() while time() < start + 10: n = Nanny(s.ip, s.port, ncores=2, loop=s.loop) n.start(0) yield gen.sleep(delay) yield n._close() print("Killed nanny") yield gen.with_timeout(timedelta(minutes=1), All([create_and_destroy_worker(0.1 * i) for i in range(10)])) @gen_cluster(ncores=[('127.0.0.1', 1)] * 10, client=True, timeout=60) def test_stress_scatter_death(c, s, *workers): import random np = pytest.importorskip('numpy') L = yield c._scatter([np.random.random(10000) for i in range(len(workers))]) yield c._replicate(L, n=2) adds = [delayed(slowadd, pure=True)(random.choice(L), random.choice(L), delay=0.05) for i in range(50)] adds = [delayed(slowadd, pure=True)(a, b, delay=0.02) for a, b in sliding_window(2, adds)] futures = c.compute(adds) alive = list(workers) from distributed.scheduler import logger for i in range(7): yield gen.sleep(0.1) try: s.validate_state() except Exception as c: logger.exception(c) import pdb; pdb.set_trace() w = random.choice(alive) yield w._close() alive.remove(w) try: yield gen.with_timeout(timedelta(seconds=10), c._gather(futures)) except gen.TimeoutError: import pdb; pdb.set_trace() except CancelledError: pass def vsum(*args): return sum(args) @slow @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 80, timeout=1000) def test_stress_communication(c, s, *workers): s.validate = False # very slow otherwise da = pytest.importorskip('dask.array') n = 40 xs = [da.random.random((100, 100), chunks=(5, 5)) for i in range(n)] ys = [x + x.T for x in xs] z = da.atop(vsum, 'ij', *concat(zip(ys, ['ij'] * n))) future = c.compute(z.sum()) result = yield future._result() assert isinstance(result, float) distributed-1.14.3/distributed/tests/test_submit_cli.py000066400000000000000000000035241301211143400233430ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from mock import Mock from tornado import gen from tornado.ioloop import IOLoop from distributed.submit import RemoteClient, _submit, _remote from distributed.utils_test import valid_python_script, invalid_python_script,\ current_loop def test_dask_submit_cli_writes_result_to_stdout(current_loop, tmpdir, valid_python_script): @gen.coroutine def test(): remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir)) yield remote_client._start() out, err = yield _submit('127.0.0.1:{0}'.format(remote_client.port), str(valid_python_script)) assert b'hello world!' in out yield remote_client._close() current_loop.run_sync(test, timeout=5) def test_dask_submit_cli_writes_traceback_to_stdout(current_loop, tmpdir, invalid_python_script): @gen.coroutine def test(): remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir)) yield remote_client._start() out, err = yield _submit('127.0.0.1:{0}'.format(remote_client.port), str(invalid_python_script)) assert b'Traceback' in err yield remote_client._close() current_loop.run_sync(test, timeout=5) def test_cli_runs_remote_client(): mock_remote_client = Mock(spec=RemoteClient) mock_ioloop = Mock(spec=IOLoop.current()) _remote('127.0.0.1:8799', 8788, loop=mock_ioloop, client=mock_remote_client) mock_remote_client.assert_called_once_with(ip='127.0.0.1', loop=mock_ioloop) mock_remote_client().start.assert_called_once_with(port=8799) assert mock_ioloop.start.called assert mock_ioloop.close.called assert mock_remote_client().stop.called distributed-1.14.3/distributed/tests/test_submit_remote_client.py000066400000000000000000000037431301211143400254300ustar00rootroot00000000000000import os from tornado import gen from distributed import rpc from distributed.submit import RemoteClient from distributed.utils_test import current_loop, valid_python_script,\ invalid_python_script def test_remote_client_uploads_a_file(current_loop, tmpdir, ): @gen.coroutine def test(): remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir)) yield remote_client._start(0) remote_process = rpc(ip='127.0.0.1', port=remote_client.port) upload = yield remote_process.upload_file(filename='script.py', file_payload='x=1') assert upload == {'status': 'OK', 'nbytes': 3} assert tmpdir.join('script.py').read() == "x=1" yield remote_client._close() current_loop.run_sync(test, timeout=5) def test_remote_client_execution_outputs_to_stdout(current_loop, tmpdir): @gen.coroutine def test(): remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir)) yield remote_client._start(0) rr = rpc(ip='127.0.0.1', port=remote_client.port) yield rr.upload_file(filename='script.py', file_payload='print("hello world!")') message = yield rr.execute(filename='script.py') assert message['stdout'] == b'hello world!' + os.linesep.encode() assert message['returncode'] == 0 yield remote_client._close() current_loop.run_sync(test, timeout=5) def test_remote_client_execution_outputs_stderr(current_loop, tmpdir, invalid_python_script): @gen.coroutine def test(): remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir)) yield remote_client._start(0) rr = rpc(ip='127.0.0.1', port=remote_client.port) yield rr.upload_file(filename='script.py', file_payload='a+1') message = yield rr.execute(filename='script.py') assert b'\'a\' is not defined' in message['stderr'] assert message['returncode'] == 1 yield remote_client._close() current_loop.run_sync(test, timeout=5) distributed-1.14.3/distributed/tests/test_sync.py000066400000000000000000000006501301211143400221620ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from distributed.sync import connect_sync, read_sync, write_sync from distributed.utils_test import cluster, loop def test_core_sync(loop): with cluster(nworkers=0) as (s, []): sock = connect_sync('127.0.0.1', s['port']) write_sync(sock, {'op': 'identity'}) result = read_sync(sock) assert result['type'] == 'Scheduler' distributed-1.14.3/distributed/tests/test_threadpoolexecutor.py000066400000000000000000000010611301211143400251230ustar00rootroot00000000000000from distributed.threadpoolexecutor import ThreadPoolExecutor, secede from time import time, sleep def test_tpe(): e = ThreadPoolExecutor(2) list(e.map(sleep, [0.01] * 4)) threads = e._threads.copy() assert len(threads) == 2 def f(): secede() return 1 assert e.submit(f).result() == 1 assert len(e._threads) == 1 list(e.map(sleep, [0.01] * 4)) assert len(threads | e._threads) == 3 start = time() while all(t.is_alive() for t in threads): sleep(0.01) assert time() < start + 1 distributed-1.14.3/distributed/tests/test_utils.py000066400000000000000000000157771301211143400223660ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from collections import Iterator from functools import partial import io from time import time, sleep from threading import Thread import threading import traceback import pytest from tornado import gen from tornado.ioloop import IOLoop from tornado.locks import Event import dask from distributed.utils import (All, sync, is_kernel, ensure_ip, str_graph, truncate_exception, get_traceback, queue_to_iterator, iterator_to_queue, _maybe_complex, read_block, seek_delimiter, funcname, ensure_bytes) from distributed.utils_test import loop, inc, throws, div from distributed.compatibility import Queue, isqueue, PY2 def test_All(loop): @gen.coroutine def throws(): 1 / 0 @gen.coroutine def slow(): yield gen.sleep(10) @gen.coroutine def inc(x): raise gen.Return(x + 1) @gen.coroutine def f(): results = yield All(*[inc(i) for i in range(10)]) assert results == list(range(1, 11)) start = time() for tasks in [[throws(), slow()], [slow(), throws()]]: try: yield All(tasks) assert False except ZeroDivisionError: pass end = time() assert end - start < 10 loop.run_sync(f) def test_sync(loop): e = Event() e2 = threading.Event() @gen.coroutine def wait_until_event(): e2.set() yield e.wait() thread = Thread(target=loop.run_sync, args=(wait_until_event,)) thread.daemon = True thread.start() e2.wait() result = sync(loop, inc, 1) assert result == 2 loop.add_callback(e.set) thread.join() def test_sync_error(loop): e = Event() @gen.coroutine def wait_until_event(): yield e.wait() thread = Thread(target=loop.run_sync, args=(wait_until_event,)) thread.daemon = True thread.start() while not loop._running: sleep(0.01) try: result = sync(loop, throws, 1) except Exception as exc: f = exc assert 'hello' in str(exc) tb = get_traceback() L = traceback.format_tb(tb) assert any('throws' in line for line in L) def function1(x): return function2(x) def function2(x): return throws(x) try: result = sync(loop, function1, 1) except Exception as exc: assert 'hello' in str(exc) tb = get_traceback() L = traceback.format_tb(tb) assert any('function1' in line for line in L) assert any('function2' in line for line in L) loop.add_callback(e.set) thread.join() def test_sync_inactive_loop(loop): @gen.coroutine def f(x): raise gen.Return(x + 1) y = sync(loop, f, 1) assert y == 2 def test_is_kernel(): pytest.importorskip('IPython') assert is_kernel() is False def test_ensure_ip(): assert ensure_ip('localhost') == '127.0.0.1' assert ensure_ip('localhost:8787') == '127.0.0.1:8787' assert ensure_ip(b'localhost:8787') == '127.0.0.1:8787' assert ensure_ip('123.123.123.123') == '123.123.123.123' assert ensure_ip('123.123.123.123:8787') == '123.123.123.123:8787' def test_truncate_exception(): e = ValueError('a'*1000) assert len(str(e)) >= 1000 f = truncate_exception(e, 100) assert type(f) == type(e) assert len(str(f)) < 200 assert 'aaaa' in str(f) e = ValueError('a') assert truncate_exception(e) is e def test_get_traceback(): def a(x): return div(x, 0) def b(x): return a(x) def c(x): return b(x) try: c(x) except Exception as e: tb = get_traceback() assert type(tb).__name__ == 'traceback' def test_queue_to_iterator(): q = Queue() q.put(1) q.put(2) seq = queue_to_iterator(q) assert isinstance(seq, Iterator) assert next(seq) == 1 assert next(seq) == 2 def test_iterator_to_queue(): seq = iter([1, 2, 3]) q = iterator_to_queue(seq) assert isqueue(q) assert q.get() == 1 def test_str_graph(): dsk = {b'x': 1} assert str_graph(dsk) == dsk dsk = {('x', 1): (inc, 1)} assert str_graph(dsk) == {str(('x', 1)): (inc, 1)} dsk = {('x', 1): (inc, 1), ('x', 2): (inc, ('x', 1))} assert str_graph(dsk) == {str(('x', 1)): (inc, 1), str(('x', 2)): (inc, str(('x', 1)))} dsks = [{'x': 1}, {('x', 1): (inc, 1), ('x', 2): (inc, ('x', 1))}, {('x', 1): (sum, [1, 2, 3]), ('x', 2): (sum, [('x', 1), ('x', 1)])}] for dsk in dsks: sdsk = str_graph(dsk) keys = list(dsk) skeys = [str(k) for k in keys] assert all(isinstance(k, (str, bytes)) for k in sdsk) assert dask.get(dsk, keys) == dask.get(sdsk, skeys) def test_maybe_complex(): assert not _maybe_complex(1) assert not _maybe_complex('x') assert _maybe_complex((inc, 1)) assert _maybe_complex([(inc, 1)]) assert _maybe_complex([(inc, 1)]) assert _maybe_complex({'x': (inc, 1)}) def test_read_block(): delimiter = b'\n' data = delimiter.join([b'123', b'456', b'789']) f = io.BytesIO(data) assert read_block(f, 1, 2) == b'23' assert read_block(f, 0, 1, delimiter=b'\n') == b'123\n' assert read_block(f, 0, 2, delimiter=b'\n') == b'123\n' assert read_block(f, 0, 3, delimiter=b'\n') == b'123\n' assert read_block(f, 0, 5, delimiter=b'\n') == b'123\n456\n' assert read_block(f, 0, 8, delimiter=b'\n') == b'123\n456\n789' assert read_block(f, 0, 100, delimiter=b'\n') == b'123\n456\n789' assert read_block(f, 1, 1, delimiter=b'\n') == b'' assert read_block(f, 1, 5, delimiter=b'\n') == b'456\n' assert read_block(f, 1, 8, delimiter=b'\n') == b'456\n789' for ols in [[(0, 3), (3, 3), (6, 3), (9, 2)], [(0, 4), (4, 4), (8, 4)]]: out = [read_block(f, o, l, b'\n') for o, l in ols] assert b"".join(filter(None, out)) == data def test_seek_delimiter_endline(): f = io.BytesIO(b'123\n456\n789') # if at zero, stay at zero seek_delimiter(f, b'\n', 5) assert f.tell() == 0 # choose the first block for bs in [1, 5, 100]: f.seek(1) seek_delimiter(f, b'\n', blocksize=bs) assert f.tell() == 4 # handle long delimiters well, even with short blocksizes f = io.BytesIO(b'123abc456abc789') for bs in [1, 2, 3, 4, 5, 6, 10]: f.seek(1) seek_delimiter(f, b'abc', blocksize=bs) assert f.tell() == 6 # End at the end f = io.BytesIO(b'123\n456') f.seek(5) seek_delimiter(f, b'\n', 5) assert f.tell() == 7 def test_funcname(): def f(): pass assert funcname(f) == 'f' assert funcname(partial(f)) == 'f' assert funcname(partial(partial(f))) == 'f' def test_ensure_bytes(): data = [b'1', '1', memoryview(b'1'), bytearray(b'1')] if PY2: data.append(buffer(b'1')) for d in data: result = ensure_bytes(d) assert isinstance(result, bytes) assert result == b'1' distributed-1.14.3/distributed/tests/test_utils_comm.py000066400000000000000000000016701301211143400233640ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import pytest from distributed.utils_test import gen_cluster from distributed.utils_comm import ( scatter_to_workers, pack_data, gather_from_workers) def test_pack_data(): data = {'x': 1} assert pack_data(('x', 'y'), data) == (1, 'y') data = {b'x': 1} assert pack_data((b'x', 'y'), data) == (1, 'y') assert pack_data({'a': b'x', 'b': 'y'}, data) == {'a': 1, 'b': 'y'} assert pack_data({'a': [b'x'], 'b': 'y'}, data) == {'a': [1], 'b': 'y'} @gen_cluster() def test_gather_from_workers_permissive(s, a, b): yield a.update_data(data={'x': 1}) with pytest.raises(KeyError): yield gather_from_workers({'x': [a.address], 'y': [b.address]}) data, bad = yield gather_from_workers({'x': [a.address], 'y': [b.address]}, permissive=True) assert data == {'x': 1} assert list(bad) == ['y'] distributed-1.14.3/distributed/tests/test_utils_test.py000066400000000000000000000020311301211143400234000ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from distributed.utils_test import (cluster, loop, gen_cluster, gen_test) from distributed.core import rpc from distributed import Scheduler, Worker, Client from tornado import gen def test_cluster(loop): with cluster() as (s, [a, b]): with rpc(ip='127.0.0.1', port=s['port']) as s: ident = loop.run_sync(s.identity) assert ident['type'] == 'Scheduler' assert len(ident['workers']) == 2 @gen_cluster(client=True) def test_gen_cluster(e, s, a, b): assert isinstance(e, Client) assert isinstance(s, Scheduler) for w in [a, b]: assert isinstance(w, Worker) assert s.ncores == {w.address: w.ncores for w in [a, b]} @gen_cluster(client=False) def test_gen_cluster_without_client(s, a, b): assert isinstance(s, Scheduler) for w in [a, b]: assert isinstance(w, Worker) assert s.ncores == {w.address: w.ncores for w in [a, b]} @gen_test() def test_gen_test(): yield gen.sleep(0.01) distributed-1.14.3/distributed/tests/test_worker.py000066400000000000000000000420601301211143400225200ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from concurrent.futures import ThreadPoolExecutor import logging from numbers import Integral from operator import add import os import re import shutil import sys import traceback import pytest from toolz import pluck from tornado import gen from tornado.ioloop import TimeoutError from distributed.batched import BatchedStream from distributed.core import rpc, connect, read, write from distributed.client import _wait from distributed.scheduler import Scheduler from distributed.protocol import to_serialize from distributed.protocol.pickle import dumps, loads from distributed.sizeof import sizeof from distributed.worker import Worker, error_message, logger, TOTAL_MEMORY from distributed.utils import ignoring from distributed.utils_test import (loop, inc, gen_cluster, slow, slowinc, throws, current_loop, gen_test) def test_worker_ncores(): from distributed.worker import _ncores w = Worker('127.0.0.1', 8019) try: assert w.executor._max_workers == _ncores finally: shutil.rmtree(w.local_dir) @gen_cluster() def test_str(s, a, b): assert a.address in str(a) assert a.address in repr(a) assert str(a.ncores) in str(a) assert str(a.ncores) in repr(a) def test_identity(): w = Worker('127.0.0.1', 8019) ident = w.identity(None) assert ident['type'] == 'Worker' assert ident['scheduler'] == ('127.0.0.1', 8019) assert isinstance(ident['ncores'], int) assert isinstance(ident['memory_limit'], int) def test_health(): w = Worker('127.0.0.1', 8019) d = w.host_health() assert isinstance(d, dict) d = w.host_health() try: import psutil except ImportError: pass else: assert 'disk-read' in d assert 'disk-write' in d assert 'network-recv' in d assert 'network-send' in d @gen_cluster() def test_worker_bad_args(c, a, b): aa = rpc(ip=a.ip, port=a.port) bb = rpc(ip=b.ip, port=b.port) class NoReprObj(object): """ This object cannot be properly represented as a string. """ def __str__(self): raise ValueError("I have no str representation.") def __repr__(self): raise ValueError("I have no repr representation.") response = yield aa.compute(key='x', function=dumps(NoReprObj), args=dumps(()), who_has={}) assert not a.active assert response['status'] == 'OK' assert a.data['x'] assert isinstance(response['compute_start'], float) assert isinstance(response['compute_stop'], float) assert isinstance(response['thread'], Integral) def bad_func(*args, **kwargs): 1 / 0 class MockLoggingHandler(logging.Handler): """Mock logging handler to check for expected logs.""" def __init__(self, *args, **kwargs): self.reset() logging.Handler.__init__(self, *args, **kwargs) def emit(self, record): self.messages[record.levelname.lower()].append(record.getMessage()) def reset(self): self.messages = { 'debug': [], 'info': [], 'warning': [], 'error': [], 'critical': [], } hdlr = MockLoggingHandler() old_level = logger.level logger.setLevel(logging.DEBUG) logger.addHandler(hdlr) response = yield bb.compute(key='y', function=dumps(bad_func), args=dumps(['x']), kwargs=dumps({'k': 'x'}), who_has={'x': [a.address]}) assert not b.active assert response['status'] == 'error' # Make sure job died because of bad func and not because of bad # argument. assert isinstance(loads(response['exception']), ZeroDivisionError) if sys.version_info[0] >= 3: assert any('1 / 0' in line for line in pluck(3, traceback.extract_tb( loads(response['traceback']))) if line) assert hdlr.messages['warning'][0] == " Compute Failed\n" \ "Function: bad_func\n" \ "args: (< could not convert arg to str >)\n" \ "kwargs: {'k': < could not convert arg to str >}\n" assert re.match(r"^Send compute response to scheduler: y, " \ "\{.*'args': \(< could not convert arg to str >\), .*" \ "'kwargs': \{'k': < could not convert arg to str >\}.*\}", hdlr.messages['debug'][0]) or \ re.match("^Send compute response to scheduler: y, " \ "\{.*'kwargs': \{'k': < could not convert arg to str >\}, .*" \ "'args': \(< could not convert arg to str >\).*\}", hdlr.messages['debug'][0]) logger.setLevel(old_level) # Now we check that both workers are still alive. assert not a.active response = yield aa.compute(key='z', function=dumps(add), args=dumps([1, 2]), who_has={}, close=True) assert not a.active assert response['status'] == 'OK' assert a.data['z'] == 3 assert isinstance(response['compute_start'], float) assert isinstance(response['compute_stop'], float) assert isinstance(response['thread'], Integral) assert not b.active response = yield bb.compute(key='w', function=dumps(add), args=dumps([1, 2]), who_has={}, close=True) assert not b.active assert response['status'] == 'OK' assert b.data['w'] == 3 assert isinstance(response['compute_start'], float) assert isinstance(response['compute_stop'], float) assert isinstance(response['thread'], Integral) aa.close_rpc() bb.close_rpc() @gen_cluster() def test_worker(c, a, b): aa = rpc(ip=a.ip, port=a.port) bb = rpc(ip=b.ip, port=b.port) result = yield aa.identity() assert not a.active response = yield aa.compute(key='x', function=dumps(add), args=dumps([1, 2]), who_has={}, close=True) assert not a.active assert response['status'] == 'OK' assert a.data['x'] == 3 assert isinstance(response['compute_start'], float) assert isinstance(response['compute_stop'], float) assert isinstance(response['thread'], Integral) response = yield bb.compute(key='y', function=dumps(add), args=dumps(['x', 10]), who_has={'x': [a.address]}) assert response['status'] == 'OK' assert b.data['y'] == 13 assert response['nbytes'] == sizeof(b.data['y']) assert isinstance(response['transfer_start'], float) assert isinstance(response['transfer_stop'], float) def bad_func(): 1 / 0 response = yield bb.compute(key='z', function=dumps(bad_func), args=dumps(()), close=True) assert not b.active assert response['status'] == 'error' assert isinstance(loads(response['exception']), ZeroDivisionError) if sys.version_info[0] >= 3: assert any('1 / 0' in line for line in pluck(3, traceback.extract_tb( loads(response['traceback']))) if line) aa.close_rpc() yield a._close() assert a.address not in c.ncores and b.address in c.ncores assert list(c.ncores.keys()) == [b.address] assert isinstance(b.address, str) assert b.ip in b.address assert str(b.port) in b.address bb.close_rpc() def test_compute_who_has(current_loop): @gen.coroutine def f(): s = Scheduler() s.listen(0) x = Worker(s.ip, s.port, ip='127.0.0.1') y = Worker(s.ip, s.port, ip='127.0.0.1') z = Worker(s.ip, s.port, ip='127.0.0.1') x.data['a'] = 1 y.data['a'] = 2 yield [x._start(), y._start(), z._start()] zz = rpc(ip=z.ip, port=z.port) yield zz.compute(function=dumps(inc), args=dumps(('a',)), who_has={'a': [x.address]}, key='b') assert z.data['b'] == 2 if 'a' in z.data: del z.data['a'] yield zz.compute(function=dumps(inc), args=dumps(('a',)), who_has={'a': [y.address]}, key='c') assert z.data['c'] == 3 yield [x._close(), y._close(), z._close()] zz.close_rpc() current_loop.run_sync(f, timeout=5) @gen_cluster() def dont_test_workers_update_center(s, a, b): aa = rpc(ip=a.ip, port=a.port) response = yield aa.update_data(data={'x': dumps(1), 'y': dumps(2)}) assert response['status'] == 'OK' assert response['nbytes'] == {'x': sizeof(1), 'y': sizeof(2)} assert a.data == {'x': 1, 'y': 2} assert s.who_has == {'x': {a.address}, 'y': {a.address}} assert s.has_what[a.address] == {'x', 'y'} yield aa.delete_data(keys=['x'], close=True) assert not s.who_has['x'] assert all('x' not in s for s in c.has_what.values()) aa.close_rpc() @slow @gen_cluster() def dont_test_delete_data_with_missing_worker(c, a, b): bad = '127.0.0.1:9001' # this worker doesn't exist c.who_has['z'].add(bad) c.who_has['z'].add(a.address) c.has_what[bad].add('z') c.has_what[a.address].add('z') a.data['z'] = 5 cc = rpc(ip=c.ip, port=c.port) yield cc.delete_data(keys=['z']) # TODO: this hangs for a while assert 'z' not in a.data assert not c.who_has['z'] assert not c.has_what[bad] assert not c.has_what[a.address] cc.close_rpc() @gen_cluster() def test_upload_file(s, a, b): assert not os.path.exists(os.path.join(a.local_dir, 'foobar.py')) assert not os.path.exists(os.path.join(b.local_dir, 'foobar.py')) assert a.local_dir != b.local_dir aa = rpc(ip=a.ip, port=a.port) bb = rpc(ip=b.ip, port=b.port) yield [aa.upload_file(filename='foobar.py', data=b'x = 123'), bb.upload_file(filename='foobar.py', data='x = 123')] assert os.path.exists(os.path.join(a.local_dir, 'foobar.py')) assert os.path.exists(os.path.join(b.local_dir, 'foobar.py')) def g(): import foobar return foobar.x yield aa.compute(function=dumps(g), key='x') result = yield aa.get_data(keys=['x']) assert result == {'x': 123} yield a._close() yield b._close() aa.close_rpc() bb.close_rpc() assert not os.path.exists(os.path.join(a.local_dir, 'foobar.py')) @gen_cluster() def test_upload_egg(s, a, b): eggname = 'mytestegg-1.0.0-py3.4.egg' local_file = __file__.replace('test_worker.py', eggname) assert not os.path.exists(os.path.join(a.local_dir, eggname)) assert not os.path.exists(os.path.join(b.local_dir, eggname)) assert a.local_dir != b.local_dir aa = rpc(ip=a.ip, port=a.port) bb = rpc(ip=b.ip, port=b.port) with open(local_file, 'rb') as f: payload = f.read() yield [aa.upload_file(filename=eggname, data=payload), bb.upload_file(filename=eggname, data=payload)] assert os.path.exists(os.path.join(a.local_dir, eggname)) assert os.path.exists(os.path.join(b.local_dir, eggname)) def g(x): import testegg return testegg.inc(x) yield aa.compute(function=dumps(g), key='x', args=dumps((10,))) result = yield aa.get_data(keys=['x']) assert result == {'x': 10 + 1} yield a._close() yield b._close() aa.close_rpc() bb.close_rpc() assert not os.path.exists(os.path.join(a.local_dir, eggname)) @gen_cluster() def test_broadcast(s, a, b): with rpc(ip=s.ip, port=s.port) as cc: results = yield cc.broadcast(msg={'op': 'ping'}) assert results == {a.address: b'pong', b.address: b'pong'} @gen_test() def test_worker_with_port_zero(): s = Scheduler() s.listen(8007) w = Worker(s.ip, s.port, ip='127.0.0.1') yield w._start() assert isinstance(w.port, int) assert w.port > 1024 @slow def test_worker_waits_for_center_to_come_up(current_loop): @gen.coroutine def f(): w = Worker('127.0.0.1', 8007, ip='127.0.0.1') yield w._start() try: current_loop.run_sync(f, timeout=4) except TimeoutError: pass @gen_cluster() def test_worker_task(s, a, b): with rpc(ip=a.ip, port=a.port) as aa: yield aa.compute(task=to_serialize((inc, 1)), key='x', report=False) assert a.data['x'] == 2 @gen_cluster() def test_worker_task_data(s, a, b): with rpc(ip=a.ip, port=a.port) as aa: yield aa.compute(task=2, key='x', report=False) assert a.data['x'] == 2 @gen_cluster() def test_worker_task_bytes(s, a, b): with rpc(ip=a.ip, port=a.port) as aa: yield aa.compute(task=to_serialize((inc, 1)), key='x', report=False) assert a.data['x'] == 2 yield aa.compute(function=dumps(inc), args=dumps((10,)), key='y', report=False) assert a.data['y'] == 11 def test_error_message(): class MyException(Exception): def __init__(self, a, b): self.args = (a + b,) def __str__(self): return "MyException(%s)" % self.args msg = error_message(MyException('Hello', 'World!')) assert 'Hello' in str(msg['exception']) @gen_cluster() def test_gather(s, a, b): b.data['x'] = 1 b.data['y'] = 2 with rpc(ip=a.ip, port=a.port) as aa: resp = yield aa.gather(who_has={'x': [b.address], 'y': [b.address]}) assert resp['status'] == 'OK' assert a.data['x'] == b.data['x'] assert a.data['y'] == b.data['y'] @gen_cluster() def test_compute_stream(s, a, b): stream = yield connect(a.ip, a.port) yield write(stream, {'op': 'compute-stream'}) msgs = [{'op': 'compute-task', 'function': dumps(inc), 'args': dumps((i,)), 'key': 'x-%d' % i} for i in range(10)] bstream = BatchedStream(stream, 0) for msg in msgs[:5]: yield write(stream, msg) for i in range(5): msg = yield read(bstream) assert msg['status'] == 'OK' assert msg['key'][0] == 'x' for msg in msgs[5:]: yield write(stream, msg) for i in range(5): msg = yield read(bstream) assert msg['status'] == 'OK' assert msg['key'][0] == 'x' yield write(stream, {'op': 'close'}) @gen_cluster(client=True, ncores=[('127.0.0.1', 1)]) def test_active_holds_tasks(e, s, w): future = e.submit(slowinc, 1, delay=0.2) yield gen.sleep(0.1) assert future.key in w.active yield future._result() assert future.key not in w.active future = e.submit(throws, 1) with ignoring(Exception): yield _wait([future]) assert not w.active def test_io_loop(loop): s = Scheduler(loop=loop) s.listen(0) assert s.io_loop is loop w = Worker(s.ip, s.port, loop=loop) assert w.io_loop is loop @gen_cluster(client=True, ncores=[]) def test_spill_to_disk(e, s): np = pytest.importorskip('numpy') w = Worker(s.ip, s.port, loop=s.loop, memory_limit=1000) yield w._start() x = e.submit(np.random.randint, 0, 255, size=500, dtype='u1', key='x') yield _wait(x) y = e.submit(np.random.randint, 0, 255, size=500, dtype='u1', key='y') yield _wait(y) assert set(w.data) == {x.key, y.key} assert set(w.data.fast) == {x.key, y.key} z = e.submit(np.random.randint, 0, 255, size=500, dtype='u1', key='z') yield _wait(z) assert set(w.data) == {x.key, y.key, z.key} assert set(w.data.fast) == {y.key, z.key} assert set(w.data.slow) == {x.key} yield x._result() assert set(w.data.fast) == {x.key, z.key} assert set(w.data.slow) == {y.key} yield w._close() @gen_cluster(client=True) def test_access_key(c, s, a, b): def f(i): from distributed.worker import thread_state return thread_state.key futures = [c.submit(f, i, key='x-%d' % i) for i in range(20)] results = yield c._gather(futures) assert list(results) == ['x-%d' % i for i in range(20)] @gen_cluster(client=True) def test_run_dask_worker(c, s, a, b): def f(dask_worker=None): return dask_worker.id response = yield c._run(f) assert response == {a.address: a.id, b.address: b.id} @gen_cluster(client=True, ncores=[]) def test_Executor(c, s): with ThreadPoolExecutor(2) as e: w = Worker(s.ip, s.port, executor=e) assert w.executor is e yield w._start() future = c.submit(inc, 1) result = yield future._result() assert result == 2 assert e._threads # had to do some work yield w._close() @slow @gen_cluster(client=True, ncores=[('127.0.0.1', 1)]) def test_str(c, s, w): da = pytest.importorskip('dask.array') x = da.ones(int(TOTAL_MEMORY * 0.7), chunks=10000000, dtype='u1') y = c.persist(x) yield _wait(y) assert len(w.data.slow) # something is on disk distributed-1.14.3/distributed/tests/test_worker_client.py000066400000000000000000000045631301211143400240640ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from datetime import timedelta from time import time from tornado import gen from distributed import local_client from distributed.utils_test import gen_cluster, inc, double @gen_cluster(client=True) def test_submit_from_worker(c, s, a, b): def func(x): with local_client() as c: x = c.submit(inc, x) y = c.submit(double, x) result = x.result() + y.result() return result x, y = c.map(func, [10, 20]) xx, yy = yield c._gather([x, y]) assert xx == 10 + 1 + (10 + 1) * 2 assert yy == 20 + 1 + (20 + 1) * 2 assert len(s.transition_log) > 10 assert len(s.wants_what) == 1 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2) def test_scatter_from_worker(c, s, a, b): def func(): with local_client() as c: futures = c.scatter([1, 2, 3, 4, 5]) assert isinstance(futures, (list, tuple)) assert len(futures) == 5 x = dict(c.worker.data) y = {f.key: i for f, i in zip(futures, [1, 2, 3, 4, 5])} assert x == y total = c.submit(sum, futures) return total.result() future = c.submit(func) result = yield future._result() assert result == sum([1, 2, 3, 4, 5]) def func(): with local_client() as c: correct = True for data in [[1, 2], (1, 2), {1, 2}]: futures = c.scatter(data) correct &= type(futures) == type(data) o = object() futures = c.scatter({'x': o}) correct &= c.worker.data['x'] is o return correct future = c.submit(func) result = yield future._result() assert result is True start = time() while not all(v == 1 for v in s.ncores.values()): yield gen.sleep(0.1) print(s.ncores) assert time() < start + 5 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2) def test_gather_multi_machine(c, s, a, b): a_address = b.address b_address = b.address def func(): with local_client() as ee: x = ee.submit(inc, 1, workers=a_address) y = ee.submit(inc, 2, workers=b_address) xx, yy = ee.gather([x, y]) return xx, yy future = c.submit(func) result = yield future._result() assert result == (2, 3) distributed-1.14.3/distributed/tests/test_worker_failure.py000066400000000000000000000212001301211143400242200ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from concurrent.futures import CancelledError from operator import add import os from time import time, sleep from dask import delayed import pytest from toolz import partition_all from tornado import gen from distributed.compatibility import PY3 from distributed.client import _wait from distributed.nanny import isalive from distributed.utils import sync, ignoring from distributed.utils_test import (gen_cluster, cluster, inc, loop, slow, div, slowinc, slowadd) from distributed import Client, Nanny, wait def test_submit_after_failed_worker(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: L = c.map(inc, range(10)) wait(L) a['proc'].terminate() total = c.submit(sum, L) assert total.result() == sum(map(inc, range(10))) def test_gather_after_failed_worker(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: L = c.map(inc, range(10)) wait(L) a['proc'].terminate() result = c.gather(L) assert result == list(map(inc, range(10))) @slow def test_gather_then_submit_after_failed_workers(loop): with cluster(nworkers=4) as (s, [w, x, y, z]): with Client(('127.0.0.1', s['port']), loop=loop) as c: L = c.map(inc, range(20)) wait(L) w['proc'].terminate() total = c.submit(sum, L) wait([total]) (_, port) = first(c.scheduler.who_has[total.key]) for d in [x, y, z]: if d['port'] == port: d['proc'].terminate() result = c.gather([total]) assert result == [sum(map(inc, range(20)))] @gen_cluster(Worker=Nanny, timeout=60, client=True) def test_failed_worker_without_warning(c, s, a, b): L = c.map(inc, range(10)) yield _wait(L) original_process = a.process a.process.terminate() start = time() while a.process is original_process and not isalive(a.process): yield gen.sleep(0.01) assert time() - start < 10 yield gen.sleep(0.5) start = time() while len(s.ncores) < 2: yield gen.sleep(0.01) assert time() - start < 10 yield _wait(L) L2 = c.map(inc, range(10, 20)) yield _wait(L2) assert all(len(keys) > 0 for keys in s.has_what.values()) ncores2 = s.ncores.copy() yield c._restart() L = c.map(inc, range(10)) yield _wait(L) assert all(len(keys) > 0 for keys in s.has_what.values()) assert not (set(ncores2) & set(s.ncores)) # no overlap @gen_cluster(Worker=Nanny, client=True) def test_restart(c, s, a, b): assert s.ncores == {a.worker_address: 1, b.worker_address: 2} x = c.submit(inc, 1) y = c.submit(inc, x) z = c.submit(div, 1, 0) yield y._result() assert set(s.who_has) == {x.key, y.key} f = yield c._restart() assert f is c assert len(s.stacks) == 2 assert len(s.processing) == 2 assert not s.who_has assert x.cancelled() assert y.cancelled() assert z.cancelled() assert z.key not in s.exceptions assert not s.who_wants assert not s.wants_what @gen_cluster(Worker=Nanny, client=True) def test_restart_cleared(c, s, a, b): x = 2 * delayed(1) + 1 f = c.compute(x) yield _wait([f]) assert s.released yield c._restart() for coll in [s.tasks, s.dependencies, s.dependents, s.waiting, s.waiting_data, s.who_has, s.restrictions, s.loose_restrictions, s.released, s.priority, s.exceptions, s.who_wants, s.exceptions_blame]: assert not coll def test_restart_sync_no_center(loop): with cluster(nanny=True) as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = c.submit(inc, 1) c.restart() assert x.cancelled() y = c.submit(inc, 2) assert y.result() == 3 assert len(c.ncores()) == 2 def test_restart_sync(loop): with cluster(nanny=True) as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: x = c.submit(div, 1, 2) x.result() assert sync(loop, c.scheduler.who_has) c.restart() assert not sync(loop, c.scheduler.who_has) assert x.cancelled() assert len(c.ncores()) == 2 with pytest.raises(CancelledError): x.result() y = c.submit(div, 1, 3) assert y.result() == 1 / 3 @gen_cluster(Worker=Nanny, client=True, timeout=20) def test_restart_fast(c, s, a, b): L = c.map(sleep, range(10)) start = time() yield c._restart() assert time() - start < 10 assert len(s.ncores) == 2 assert all(x.status == 'cancelled' for x in L) x = c.submit(inc, 1) result = yield x._result() assert result == 2 def test_restart_fast_sync(loop): with cluster(nanny=True) as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: L = c.map(sleep, range(10)) start = time() c.restart() assert time() - start < 10 assert len(c.ncores()) == 2 assert all(x.status == 'cancelled' for x in L) x = c.submit(inc, 1) assert x.result() == 2 @gen_cluster(Worker=Nanny, client=True, timeout=20) def test_fast_kill(c, s, a, b): L = c.map(sleep, range(10)) start = time() yield c._restart() assert time() - start < 10 assert all(x.status == 'cancelled' for x in L) x = c.submit(inc, 1) result = yield x._result() assert result == 2 @gen_cluster(Worker=Nanny) def test_multiple_clients_restart(s, a, b): e1 = Client((s.ip, s.port), start=False) yield e1._start() e2 = Client((s.ip, s.port), start=False) yield e2._start() x = e1.submit(inc, 1) y = e2.submit(inc, 2) xx = yield x._result() yy = yield y._result() assert xx == 2 assert yy == 3 yield e1._restart() assert x.cancelled() assert y.cancelled() yield e1._shutdown(fast=True) yield e2._shutdown(fast=True) @gen_cluster(Worker=Nanny) def test_restart_scheduler(s, a, b): import gc; gc.collect() yield s.restart() assert len(s.ncores) == 2 @gen_cluster(Worker=Nanny, client=True) def test_forgotten_futures_dont_clean_up_new_futures(c, s, a, b): x = c.submit(inc, 1) yield c._restart() y = c.submit(inc, 1) del x import gc; gc.collect() yield gen.sleep(0.1) yield y._result() @gen_cluster(client=True, timeout=60) def test_broken_worker_during_computation(c, s, a, b): n = Nanny(s.ip, s.port, ncores=2, loop=s.loop) n.start(0) start = time() while len(s.ncores) < 3: yield gen.sleep(0.01) assert time() < start + 5 L = c.map(inc, range(256)) for i in range(8): L = c.map(add, *zip(*partition_all(2, L))) from random import random yield gen.sleep(random() / 2) with ignoring(OSError): n.process.terminate() yield gen.sleep(random() / 2) with ignoring(OSError): n.process.terminate() result = yield c._gather(L) assert isinstance(result[0], int) yield n._close() @gen_cluster(client=True, Worker=Nanny) def test_restart_during_computation(c, s, a, b): xs = [delayed(slowinc)(i, delay=0.01) for i in range(50)] ys = [delayed(slowinc)(i, delay=0.01) for i in xs] zs = [delayed(slowadd)(x, y, delay=0.01) for x, y in zip(xs, ys)] total = delayed(sum)(zs) result = c.compute(total) yield gen.sleep(0.5) assert s.rprocessing yield c._restart() assert not s.rprocessing assert len(s.ncores) == 2 assert not s.task_state @pytest.mark.skipif(not os.path.exists('myenv.zip') or not PY3, reason='Depends on large local file') @gen_cluster(client=True, Worker=Nanny, timeout=120) def test_upload_environment(c, s, a, b): responses = yield c._upload_environment('myenv.zip') assert os.path.exists(os.path.join(a.local_dir, 'myenv')) assert os.path.exists(os.path.join(b.local_dir, 'myenv')) @pytest.mark.skipif(not os.path.exists('myenv.zip') or not PY3, reason='Depends on large local file') @gen_cluster(client=True, Worker=Nanny, timeout=120) def test_restart_environment(c, s, a, b): yield c._restart(environment='myenv.zip') def get_executable(): import sys return sys.executable results = yield c._run(get_executable) assert results == {n.worker_address: os.path.join(n.local_dir, 'myenv', 'bin', 'python') for n in [a, b]} distributed-1.14.3/distributed/threadpoolexecutor.py000066400000000000000000000114261301211143400227300ustar00rootroot00000000000000""" Modified ThreadPoolExecutor to support threads leaving the thread pool This includes a global `secede` method that a submitted function can call to have its thread leave the ThreadPoolExecutor's thread pool. This allows the thread pool to allocate another thread if necessary and so is useful when a function realises that it is going to be a long-running job that doesn't want to take up space. When the function finishes its thread will terminate gracefully. This code copies and modifies two functions from the `concurrent.futures.thread` module, notably `_worker` and ThreadPoolExecutor._adjust_thread_count` to allow for checking against a global `threading.local` state. These functions are subject to the following license, which is included as a comment at the end of this file: https://docs.python.org/3/license.html ... and are under copyright by the Python Software Foundation Copyright 2001-2016 Python Software Foundation; All Rights Reserved """ from __future__ import print_function, division, absolute_import from concurrent.futures import thread from threading import local, Thread from .compatibility import get_thread_identity thread_state = local() def _worker(executor, work_queue): thread_state.proceed = True thread_state.executor = executor try: while thread_state.proceed: task = work_queue.get() if task is not None: # sentinel task.run() del task elif thread._shutdown or executor is None or executor._shutdown: work_queue.put(None) return del executor except BaseException: _base.LOGGER.critical('Exception in worker', exc_info=True) finally: del thread_state.proceed del thread_state.executor class ThreadPoolExecutor(thread.ThreadPoolExecutor): def _adjust_thread_count(self): if len(self._threads) < self._max_workers: t = Thread(target=_worker, args=(self, self._work_queue)) t.daemon = True self._threads.add(t) t.start() def secede(): """ Have this thread secede from the ThreadPoolExecutor """ thread_state.proceed = False ident = get_thread_identity() for t in thread_state.executor._threads: if t.ident == ident: thread_state.executor._threads.remove(t) break """ PSF LICENSE AGREEMENT FOR PYTHON 3.5.2 ====================================== 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using Python 3.5.2 software in source or binary form and its associated documentation. 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python 3.5.2 alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright c 2001-2016 Python Software Foundation; All Rights Reserved" are retained in Python 3.5.2 alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on or incorporates Python 3.5.2 or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python 3.5.2. 4. PSF is making Python 3.5.2 available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 3.5.2 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 3.5.2 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 3.5.2, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions. 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. 8. By copying, installing or otherwise using Python 3.5.2, Licensee agrees to be bound by the terms and conditions of this License Agreement. """ distributed-1.14.3/distributed/utils.py000066400000000000000000000323541301211143400201530ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from collections import Iterable from contextlib import contextmanager import logging import os import re import socket import six import sys import tblib.pickling_support import tempfile from threading import Thread try: import resource except ImportError: resource = None from dask import istask from toolz import memoize, valmap from tornado import gen from tornado.iostream import StreamClosedError from .compatibility import Queue, PY3, PY2 from .config import config logger = logging.getLogger(__name__) def funcname(func): """Get the name of a function.""" while hasattr(func, 'func'): func = func.func try: return func.__name__ except: return str(func) def get_fileno_limit(): """ Get the maximum number of open files per process. """ if resource is not None: return resource.getrlimit(resource.RLIMIT_NOFILE)[0] else: # Default ceiling for Windows when using the CRT, though it # is settable using _setmaxstdio(). return 512 def get_ip(host='8.8.8.8', port=80): try: return [(s.connect((host, port)), s.getsockname()[0], s.close()) for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]][0][1] except OSError: return '127.0.0.1' @contextmanager def ignoring(*exceptions): try: yield except exceptions: pass @gen.coroutine def ignore_exceptions(coroutines, *exceptions): """ Process list of coroutines, ignoring certain exceptions >>> coroutines = [cor(...) for ...] # doctest: +SKIP >>> x = yield ignore_exceptions(coroutines, TypeError) # doctest: +SKIP """ wait_iterator = gen.WaitIterator(*coroutines) results = [] while not wait_iterator.done(): with ignoring(*exceptions): result = yield wait_iterator.next() results.append(result) raise gen.Return(results) @gen.coroutine def All(*args): """ Wait on many tasks at the same time Err once any of the tasks err. See https://github.com/tornadoweb/tornado/issues/1546 """ if len(args) == 1 and isinstance(args[0], Iterable): args = args[0] tasks = gen.WaitIterator(*args) results = [None for _ in args] while not tasks.done(): result = yield tasks.next() results[tasks.current_index] = result raise gen.Return(results) def sync(loop, func, *args, **kwargs): """ Run coroutine in loop running in separate thread """ if not loop._running: try: return loop.run_sync(lambda: func(*args, **kwargs)) except RuntimeError: # loop already running pass from threading import Event e = Event() result = [None] error = [False] traceback = [False] @gen.coroutine def f(): try: result[0] = yield gen.maybe_future(func(*args, **kwargs)) except Exception as exc: logger.exception(exc) result[0] = exc error[0] = exc exc_type, exc_value, exc_traceback = sys.exc_info() traceback[0] = exc_traceback finally: e.set() a = loop.add_callback(f) while not e.is_set(): e.wait(1000000) if error[0]: six.reraise(type(error[0]), error[0], traceback[0]) else: return result[0] @contextmanager def tmp_text(filename, text): fn = os.path.join(tempfile.gettempdir(), filename) with open(fn, 'w') as f: f.write(text) try: yield fn finally: if os.path.exists(fn): os.remove(fn) def clear_queue(q): while not q.empty(): q.get_nowait() def is_kernel(): """ Determine if we're running within an IPython kernel >>> is_kernel() False """ # http://stackoverflow.com/questions/34091701/determine-if-were-in-an-ipython-notebook-session if 'IPython' not in sys.modules: # IPython hasn't been imported return False from IPython import get_ipython # check for `kernel` attribute on the IPython instance return getattr(get_ipython(), 'kernel', None) is not None hex_pattern = re.compile('[a-f]+') def key_split(s): """ >>> key_split('x') 'x' >>> key_split('x-1') 'x' >>> key_split('x-1-2-3') 'x' >>> key_split(('x-2', 1)) 'x' >>> key_split("('x-2', 1)") 'x' >>> key_split('hello-world-1') 'hello-world' >>> key_split(b'hello-world-1') 'hello-world' >>> key_split('ae05086432ca935f6eba409a8ecd4896') 'data' >>> key_split('>> key_split(None) 'Other' >>> key_split('x-abcdefab') # ignores hex 'x' """ if type(s) is bytes: s = s.decode() if type(s) is tuple: s = s[0] try: words = s.split('-') if not words[0][0].isalpha(): result = words[0].lstrip("'(\"") else: result = words[0] for word in words[1:]: if word.isalpha() and not (len(word) == 8 and hex_pattern.match(word) is not None): result += '-' + word else: break if len(result) == 32 and re.match(r'[a-f0-9]{32}', result): return 'data' else: if result[0] == '<': result = result.strip('<>').split()[0].split('.')[-1] return result except: return 'Other' @contextmanager def log_errors(pdb=False): try: yield except (StreamClosedError, gen.Return): raise except Exception as e: logger.exception(e) if pdb: import pdb; pdb.set_trace() raise @memoize def ensure_ip(hostname): """ Ensure that address is an IP address Examples -------- >>> ensure_ip('localhost') '127.0.0.1' >>> ensure_ip('123.123.123.123') # pass through IP addresses '123.123.123.123' >>> ensure_ip('localhost:5000') '127.0.0.1:5000' """ if PY3 and isinstance(hostname, bytes): hostname = hostname.decode() if ':' in hostname: host, port = hostname.rsplit(':', 1) return ':'.join([ensure_ip(host), port]) if re.match('\d+\.\d+\.\d+\.\d+', hostname): # is IP return hostname else: try: return socket.gethostbyname(hostname) except Exception as e: logger.warn("Could not resolve hostname: %s", hostname, exc_info=True) raise tblib.pickling_support.install() def get_traceback(): exc_type, exc_value, exc_traceback = sys.exc_info() bad = [os.path.join('distributed', 'worker'), os.path.join('distributed', 'scheduler'), os.path.join('tornado', 'gen.py'), os.path.join('concurrent', 'futures')] while exc_traceback and any(b in exc_traceback.tb_frame.f_code.co_filename for b in bad): exc_traceback = exc_traceback.tb_next return exc_traceback def truncate_exception(e, n=10000): """ Truncate exception to be about a certain length """ if len(str(e)) > n: try: return type(e)("Long error message", str(e)[:n]) except: return Exception("Long error message", type(e), str(e)[:n]) else: return e def queue_to_iterator(q): while True: result = q.get() if isinstance(result, StopIteration): raise result yield result def _dump_to_queue(seq, q): for item in seq: q.put(item) def iterator_to_queue(seq, maxsize=0): q = Queue(maxsize=maxsize) t = Thread(target=_dump_to_queue, args=(seq, q)) t.daemon = True t.start() return q def tokey(o): """ Convert an object to a bytestring, using str Examples -------- >>> tokey(b'x') b'x' >>> tokey('x') 'x' >>> tokey(1) '1' """ t = type(o) if t is str or t is bytes: return o else: return str(o) def _maybe_complex(task): """ Possibly contains a nested task """ return (istask(task) or type(task) is list and any(map(_maybe_complex, task)) or type(task) is dict and any(map(_maybe_complex, task.values()))) def str_graph(dsk, extra_values=()): def convert(task): if type(task) is list: return [convert(v) for v in task] if type(task) is dict: return valmap(convert, task) if istask(task): return (task[0],) + tuple(map(convert, task[1:])) try: if task in dsk or task in extra_values: return tokey(task) except TypeError: pass return task return {tokey(k): convert(v) for k, v in dsk.items()} import logging from .compatibility import logging_names logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s', level=logging.INFO) for name, level in config.get('logging', {}).items(): LEVEL = logging_names[level.upper()] logging.getLogger(name).setLevel(LEVEL) # http://stackoverflow.com/questions/21234772/python-tornado-disable-logging-to-stderr stream = logging.StreamHandler(sys.stderr) stream.setLevel(logging.CRITICAL) logging.getLogger('tornado').addHandler(stream) logging.getLogger('tornado').propagate = False from contextlib import contextmanager import shutil def seek_delimiter(file, delimiter, blocksize): """ Seek current file to next byte after a delimiter bytestring This seeks the file to the next byte following the delimiter. It does not return anything. Use ``file.tell()`` to see location afterwards. Parameters ---------- file: a file delimiter: bytes a delimiter like ``b'\n'`` or message sentinel blocksize: int Number of bytes to read from the file at once. """ if file.tell() == 0: return last = b'' while True: current = file.read(blocksize) if not current: return full = last + current try: i = full.index(delimiter) file.seek(file.tell() - (len(full) - i) + len(delimiter)) return except ValueError: pass last = full[-len(delimiter):] def read_block(f, offset, length, delimiter=None): """ Read a block of bytes from a file Parameters ---------- fn: string Path to filename on S3 offset: int Byte offset to start read length: int Number of bytes to read delimiter: bytes (optional) Ensure reading starts and stops at delimiter bytestring If using the ``delimiter=`` keyword argument we ensure that the read starts and stops at delimiter boundaries that follow the locations ``offset`` and ``offset + length``. If ``offset`` is zero then we start at zero. The bytestring returned WILL include the terminating delimiter string. Examples -------- >>> from io import BytesIO # doctest: +SKIP >>> f = BytesIO(b'Alice, 100\\nBob, 200\\nCharlie, 300') # doctest: +SKIP >>> read_block(f, 0, 13) # doctest: +SKIP b'Alice, 100\\nBo' >>> read_block(f, 0, 13, delimiter=b'\\n') # doctest: +SKIP b'Alice, 100\\nBob, 200\\n' >>> read_block(f, 10, 10, delimiter=b'\\n') # doctest: +SKIP b'Bob, 200\\nCharlie, 300' """ if delimiter: f.seek(offset) seek_delimiter(f, delimiter, 2**16) start = f.tell() length -= start - offset f.seek(start + length) seek_delimiter(f, delimiter, 2**16) end = f.tell() eof = not f.read(1) offset = start length = end - start f.seek(offset) bytes = f.read(length) return bytes @contextmanager def tmpfile(extension=''): extension = '.' + extension.lstrip('.') handle, filename = tempfile.mkstemp(extension) os.close(handle) os.remove(filename) yield filename if os.path.exists(filename): if os.path.isdir(filename): shutil.rmtree(filename) else: try: os.remove(filename) except OSError: # sometimes we can't remove a generated temp file pass def ensure_bytes(s): """ Turn string or bytes to bytes >>> ensure_bytes('123') b'123' >>> ensure_bytes(b'123') b'123' """ if isinstance(s, bytes): return s if isinstance(s, memoryview): return s.tobytes() if isinstance(s, bytearray) or PY2 and isinstance(s, buffer): return bytes(s) if hasattr(s, 'encode'): return s.encode() raise TypeError( "Object %s is neither a bytes object nor has an encode method" % s) def divide_n_among_bins(n, bins): """ >>> divide_n_among_bins(12, [1, 1]) [6, 6] >>> divide_n_among_bins(12, [1, 2]) [4, 8] >>> divide_n_among_bins(12, [1, 2, 1]) [3, 6, 3] >>> divide_n_among_bins(11, [1, 2, 1]) [2, 6, 3] >>> divide_n_among_bins(11, [.1, .2, .1]) [2, 6, 3] """ total = sum(bins) acc = 0.0 out = [] for b in bins: now = n / total * b + acc now, acc = divmod(now, 1) out.append(int(now)) return out def mean(seq): seq = list(seq) return sum(seq) / len(seq) distributed-1.14.3/distributed/utils_comm.py000066400000000000000000000154471301211143400211720ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from collections import Iterable, defaultdict from itertools import cycle import random import socket import uuid from tornado import gen from tornado.gen import Return from tornado.ioloop import IOLoop from tornado.iostream import StreamClosedError from dask.base import tokenize from toolz import merge, concat, groupby, drop, valmap from .core import rpc, coerce_to_rpc, coerce_to_address from .utils import ignore_exceptions, All, log_errors, tokey, sync from .protocol.pickle import dumps, loads no_default = '__no_default__' @gen.coroutine def gather_from_workers(who_has, rpc=rpc, close=True, permissive=False): """ Gather data directly from peers Parameters ---------- who_has: dict Dict mapping keys to sets of workers that may have that key Returns dict mapping key to value See Also -------- gather _gather """ bad_addresses = set() who_has = {k: set(v) for k, v in who_has.items()} results = dict() all_bad_keys = set() while len(results) + len(all_bad_keys) < len(who_has): d = defaultdict(list) rev = dict() bad_keys = set() for key, addresses in who_has.items(): if key in results: continue try: addr = random.choice(list(addresses - bad_addresses)) d[addr].append(key) rev[key] = addr except IndexError: bad_keys.add(key) if bad_keys: if permissive: all_bad_keys |= bad_keys else: raise KeyError(*bad_keys) rpcs = {addr: rpc(addr) for addr in d} try: coroutines = [rpcs[address].get_data(keys=keys, close=close) for address, keys in d.items()] response = yield ignore_exceptions(coroutines, socket.error, StreamClosedError) finally: for r in rpcs.values(): r.close_rpc() response = merge(response) bad_addresses |= {v for k, v in rev.items() if k not in response} results.update(merge(response)) if permissive: raise Return((results, all_bad_keys)) else: raise Return(results) class WrappedKey(object): """ Interface for a key in a dask graph. Subclasses must have .key attribute that refers to a key in a dask graph. Sometimes we want to associate metadata to keys in a dask graph. For example we might know that that key lives on a particular machine or can only be accessed in a certain way. Schedulers may have particular needs that can only be addressed by additional metadata. """ def __init__(self, key): self.key = key _round_robin_counter = [0] @gen.coroutine def scatter_to_workers(ncores, data, report=True, serialize=True): """ Scatter data directly to workers This distributes data in a round-robin fashion to a set of workers based on how many cores they have. ncores should be a dictionary mapping worker identities to numbers of cores. See scatter for parameter docstring """ if isinstance(ncores, Iterable) and not isinstance(ncores, dict): k = len(data) // len(ncores) ncores = {coerce_to_address(worker): k for worker in ncores} workers = list(concat([w] * nc for w, nc in ncores.items())) if isinstance(data, dict): names, data = list(zip(*data.items())) else: names = [] for x in data: try: names.append(tokenize(x)) except: names.append(str(uuid.uuid1())) worker_iter = drop(_round_robin_counter[0] % len(workers), cycle(workers)) _round_robin_counter[0] += len(data) L = list(zip(worker_iter, names, data)) d = groupby(0, L) d = {worker: {key: dumps(value) if serialize else value for _, key, value in v} for worker, v in d.items()} rpcs = {addr: rpc(addr) for addr in d} try: out = yield All([rpcs[address].update_data(data=v, close=True, report=report) for address, v in d.items()]) finally: for r in rpcs.values(): r.close_rpc() nbytes = merge(o['nbytes'] for o in out) who_has = {k: [w for w, _, _ in v] for k, v in groupby(1, L).items()} raise Return((names, who_has, nbytes)) collection_types = (tuple, list, set, frozenset) def unpack_remotedata(o, byte_keys=False, myset=None): """ Unpack WrappedKey objects from collection Returns original collection and set of all found keys Examples -------- >>> rd = WrappedKey('mykey') >>> unpack_remotedata(1) (1, set()) >>> unpack_remotedata(()) ((), set()) >>> unpack_remotedata(rd) ('mykey', {'mykey'}) >>> unpack_remotedata([1, rd]) ([1, 'mykey'], {'mykey'}) >>> unpack_remotedata({1: rd}) ({1: 'mykey'}, {'mykey'}) >>> unpack_remotedata({1: [rd]}) ({1: ['mykey']}, {'mykey'}) Use the ``byte_keys=True`` keyword to force string keys >>> rd = WrappedKey(('x', 1)) >>> unpack_remotedata(rd, byte_keys=True) ("('x', 1)", {"('x', 1)"}) """ if myset is None: myset = set() out = unpack_remotedata(o, byte_keys, myset) return out, myset typ = type(o) if typ in collection_types: if not o: return o outs = [unpack_remotedata(item, byte_keys, myset) for item in o] return type(o)(outs) elif typ is dict: if o: values = [unpack_remotedata(v, byte_keys, myset) for v in o.values()] return dict(zip(o.keys(), values)) else: return o elif issubclass(typ, WrappedKey): # TODO use type is Future k = o.key if byte_keys: k = tokey(k) myset.add(k) return k else: return o def pack_data(o, d): """ Merge known data into tuple or dict Parameters ---------- o: core data structures containing literals and keys d: dict mapping of keys to data Examples -------- >>> data = {'x': 1} >>> pack_data(('x', 'y'), data) (1, 'y') >>> pack_data({'a': 'x', 'b': 'y'}, data) # doctest: +SKIP {'a': 1, 'b': 'y'} >>> pack_data({'a': ['x'], 'b': 'y'}, data) # doctest: +SKIP {'a': [1], 'b': 'y'} """ typ = type(o) try: if o in d: return d[o] except TypeError: pass if typ in collection_types: return typ([pack_data(x, d) for x in o]) elif typ is dict: return {k: pack_data(v, d) for k, v in o.items()} else: try: if o in d: return d[o] except TypeError: pass return o distributed-1.14.3/distributed/utils_test.py000066400000000000000000000276741301211143400212230ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from contextlib import contextmanager from glob import glob import logging from multiprocessing import Process, Queue import os import shutil import signal import socket import subprocess import sys from time import time, sleep import uuid from toolz import merge from tornado import gen from tornado.ioloop import IOLoop, TimeoutError from tornado.iostream import StreamClosedError from .core import connect, read, write, rpc from .utils import ignoring, log_errors, sync import pytest logger = logging.getLogger(__name__) @pytest.fixture(scope='session') def valid_python_script(tmpdir_factory): local_file = tmpdir_factory.mktemp('data').join('file.py') local_file.write("print('hello world!')") return local_file @pytest.fixture(scope='session') def client_contract_script(tmpdir_factory): local_file = tmpdir_factory.mktemp('data').join('distributed_script.py') lines = ("from distributed import Client", "e = Client('127.0.0.1:8989')", 'print(e)') local_file.write('\n'.join(lines)) return local_file @pytest.fixture(scope='session') def invalid_python_script(tmpdir_factory): local_file = tmpdir_factory.mktemp('data').join('file.py') local_file.write("a+1") return local_file @pytest.yield_fixture def current_loop(): IOLoop.clear_instance() loop = IOLoop() loop.make_current() yield loop if loop._running: sync(loop, loop.stop) for i in range(5): try: loop.close(all_fds=True) return except Exception as e: f = e print(f) IOLoop.clear_instance() @pytest.yield_fixture def loop(): loop = IOLoop() yield loop if loop._running: sync(loop, loop.stop) for i in range(5): try: loop.close(all_fds=True) return except Exception as e: f = e print(f) @pytest.yield_fixture def zmq_ctx(): import zmq ctx = zmq.Context.instance() yield ctx ctx.destroy(linger=0) @contextmanager def mock_ipython(): import mock ip = mock.Mock() ip.user_ns = {} ip.kernel = None get_ip = lambda : ip with mock.patch('IPython.get_ipython', get_ip), \ mock.patch('distributed._ipython_utils.get_ipython', get_ip): yield ip def inc(x): return x + 1 def dec(x): return x - 1 def div(x, y): return x / y def deep(n): if n > 0: return deep(n - 1) else: return True def throws(x): raise Exception('hello!') def double(x): return x * 2 def slowinc(x, delay=0.02): from time import sleep sleep(delay) return x + 1 def randominc(x, scale=1): from time import sleep from random import random sleep(random() * scale) return x + 1 def slowadd(x, y, delay=0.02): from time import sleep sleep(delay) return x + y def run_scheduler(q, scheduler_port=0, **kwargs): from distributed import Scheduler from tornado.ioloop import IOLoop, PeriodicCallback import logging IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) scheduler = Scheduler(loop=loop, validate=True, **kwargs) done = scheduler.start(scheduler_port) q.put(scheduler.port) try: loop.start() finally: loop.close(all_fds=True) def run_worker(q, scheduler_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop, PeriodicCallback import logging with log_errors(): IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) worker = Worker('127.0.0.1', scheduler_port, ip='127.0.0.1', loop=loop, **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.port) try: loop.start() finally: loop.close(all_fds=True) def run_nanny(q, scheduler_port, **kwargs): from distributed import Nanny from tornado.ioloop import IOLoop, PeriodicCallback import logging with log_errors(): IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) worker = Nanny('127.0.0.1', scheduler_port, ip='127.0.0.1', loop=loop, **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.port) try: loop.start() finally: loop.run_sync(worker._close) loop.close(all_fds=True) @contextmanager def cluster(nworkers=2, nanny=False, worker_kwargs={}): rpc_active = rpc.active if nanny: _run_worker = run_nanny else: _run_worker = run_worker scheduler_q = Queue() scheduler = Process(target=run_scheduler, args=(scheduler_q,)) scheduler.daemon = True scheduler.start() sport = scheduler_q.get() workers = [] for i in range(nworkers): q = Queue() fn = '_test_worker-%s' % uuid.uuid1() proc = Process(target=_run_worker, args=(q, sport), kwargs=merge({'ncores': 1, 'local_dir': fn}, worker_kwargs)) workers.append({'proc': proc, 'queue': q, 'dir': fn}) for worker in workers: worker['proc'].start() for worker in workers: worker['port'] = worker['queue'].get() loop = IOLoop() start = time() try: with rpc(ip='127.0.0.1', port=sport) as s: while True: ncores = loop.run_sync(s.ncores) if len(ncores) == nworkers: break if time() - start > 5: raise Exception("Timeout on cluster creation") yield {'proc': scheduler, 'port': sport}, workers finally: logger.debug("Closing out test cluster") with ignoring(socket.error, TimeoutError, StreamClosedError): loop.run_sync(lambda: disconnect('127.0.0.1', sport), timeout=0.5) scheduler.terminate() scheduler.join(timeout=2) for port in [w['port'] for w in workers]: with ignoring(socket.error, TimeoutError, StreamClosedError): loop.run_sync(lambda: disconnect('127.0.0.1', port), timeout=0.5) for proc in [w['proc'] for w in workers]: with ignoring(Exception): proc.terminate() proc.join(timeout=2) for q in [w['queue'] for w in workers]: q.close() for fn in glob('_test_worker-*'): shutil.rmtree(fn) loop.close(all_fds=True) assert rpc.active == rpc_active # no new rpcs made @gen.coroutine def disconnect(ip, port): stream = yield connect(ip, port) try: yield write(stream, {'op': 'terminate', 'close': True}) response = yield read(stream) finally: stream.close() import pytest try: slow = pytest.mark.skipif( not pytest.config.getoption("--runslow"), reason="need --runslow option to run") except (AttributeError, ValueError): def slow(*args): pass from tornado import gen from tornado.ioloop import IOLoop def gen_test(timeout=10): """ Coroutine test @gen_test(timeout=5) def test_foo(): yield ... # use tornado coroutines """ def _(func): def test_func(): IOLoop.clear_instance() loop = IOLoop() loop.make_current() cor = gen.coroutine(func) try: loop.run_sync(cor, timeout=timeout) finally: loop.stop() loop.close(all_fds=True) return test_func return _ from .scheduler import Scheduler from .worker import Worker from .client import Client @gen.coroutine def start_cluster(ncores, loop, Worker=Worker, scheduler_kwargs={}): s = Scheduler(ip='127.0.0.1', loop=loop, validate=True, **scheduler_kwargs) done = s.start(0) workers = [Worker(s.ip, s.port, ncores=v, ip=k, name=i, loop=loop) for i, (k, v) in enumerate(ncores)] for w in workers: w.rpc = workers[0].rpc yield [w._start() for w in workers] start = time() while len(s.ncores) < len(ncores): yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") raise gen.Return((s, workers)) @gen.coroutine def end_cluster(s, workers): logger.debug("Closing out test cluster") for w in workers: with ignoring(TimeoutError, StreamClosedError, OSError): yield w._close(report=False) if w.local_dir and os.path.exists(w.local_dir): shutil.rmtree(w.local_dir) yield s.close() s.stop() def gen_cluster(ncores=[('127.0.0.1', 1), ('127.0.0.1', 2)], timeout=10, Worker=Worker, client=False, scheduler_kwargs={}): from distributed import Client """ Coroutine test with small cluster @gen_cluster() def test_foo(scheduler, worker1, worker2): yield ... # use tornado coroutines See also: start end """ def _(func): cor = gen.coroutine(func) def test_func(): rpc_active = rpc.active IOLoop.clear_instance() loop = IOLoop() loop.make_current() s, workers = loop.run_sync(lambda: start_cluster(ncores, loop, Worker=Worker, scheduler_kwargs=scheduler_kwargs)) args = [s] + workers if client: e = Client((s.ip, s.port), loop=loop, start=False) loop.run_sync(e._start) args = [e] + args try: loop.run_sync(lambda: cor(*args), timeout=timeout) finally: if client: loop.run_sync(e._shutdown) loop.run_sync(lambda: end_cluster(s, workers)) loop.stop() loop.close(all_fds=True) assert rpc.active == rpc_active return test_func return _ @contextmanager def make_hdfs(): from hdfs3 import HDFileSystem hdfs = HDFileSystem(host='localhost', port=8020) if hdfs.exists('/tmp/test'): hdfs.rm('/tmp/test') hdfs.mkdir('/tmp/test') try: yield hdfs finally: if hdfs.exists('/tmp/test'): hdfs.rm('/tmp/test') def raises(func, exc=Exception): try: func() return False except exc: return True @contextmanager def popen(*args, **kwargs): kwargs['stdout'] = subprocess.PIPE kwargs['stderr'] = subprocess.PIPE if sys.platform.startswith('win'): # Allow using CTRL_C_EVENT / CTRL_BREAK_EVENT kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP dump_stdout = False proc = subprocess.Popen(*args, **kwargs) try: yield proc except Exception: dump_stdout = True raise finally: if sys.platform.startswith('win'): proc.send_signal(signal.CTRL_BREAK_EVENT) else: proc.send_signal(signal.SIGINT) try: if sys.version_info[0] == 3: proc.wait(10) else: proc.wait() finally: # Make sure we don't leave the process lingering around with ignoring(OSError): proc.kill() # XXX Also dump stdout if return code != 0 ? if dump_stdout: line = '\n\nPrint from stderr\n=================\n' while line: print(line) line = proc.stderr.readline() line = '\n\nPrint from stdout\n=================\n' while line: print(line) line = proc.stdout.readline() distributed-1.14.3/distributed/versions.py000066400000000000000000000037471301211143400206670ustar00rootroot00000000000000""" utilities for package version introspection """ from __future__ import print_function, division, absolute_import import platform import struct import os import sys import locale import importlib required_packages = [('dask', lambda p: p.__version__), ('distributed', lambda p: p.__version__), ('msgpack', lambda p: '.'.join([str(v) for v in p.version])), ('cloudpickle', lambda p: p.__version__), ('toolz', lambda p: p.__version__)] optional_packages = [('numpy', lambda p: p.__version__), ('pandas', lambda p: p.__version__)] def get_versions(): """ Return basic information on our software installation, and out installed versions of packages. """ d = {'host': get_system_info(), 'packages': {'required': get_package_info(required_packages), 'optional': get_package_info(optional_packages)} } return d def get_system_info(): (sysname, nodename, release, version, machine, processor) = platform.uname() host = [("python", "%d.%d.%d.%s.%s" % sys.version_info[:]), ("python-bits", struct.calcsize("P") * 8), ("OS", "%s" % (sysname)), ("OS-release", "%s" % (release)), ("machine", "%s" % (machine)), ("processor", "%s" % (processor)), ("byteorder", "%s" % sys.byteorder), ("LC_ALL", "%s" % os.environ.get('LC_ALL', "None")), ("LANG", "%s" % os.environ.get('LANG', "None")), ("LOCALE", "%s.%s" % locale.getlocale()), ] return host def get_package_info(pkgs): """ get package versions for the passed required & optional packages """ pversions = [] for (modname, ver_f) in pkgs: try: mod = importlib.import_module(modname) ver = ver_f(mod) pversions.append((modname, ver)) except: pversions.append((modname, None)) return pversions distributed-1.14.3/distributed/worker.py000066400000000000000000001007631301211143400203240ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from datetime import timedelta from importlib import import_module import logging import multiprocessing import os import pkg_resources import tempfile from threading import current_thread, Lock, local from time import time from timeit import default_timer import shutil import sys from dask.core import istask from dask.compatibility import apply try: from cytoolz import valmap, merge except ImportError: from toolz import valmap, merge from tornado.gen import Return from tornado import gen from tornado.ioloop import IOLoop, PeriodicCallback from tornado.iostream import StreamClosedError from .batched import BatchedSend from .utils_comm import pack_data, gather_from_workers from .compatibility import reload, unicode from .core import (rpc, Server, pingpong, coerce_to_address, error_message, read, RPCClosed) from .protocol.pickle import dumps, loads from .sizeof import sizeof from .threadpoolexecutor import ThreadPoolExecutor from .utils import funcname, get_ip, _maybe_complex, log_errors, All, ignoring _ncores = multiprocessing.cpu_count() thread_state = local() logger = logging.getLogger(__name__) try: import psutil TOTAL_MEMORY = psutil.virtual_memory().total except ImportError: logger.warn("Please install psutil to estimate worker memory use") TOTAL_MEMORY = 8e9 class Worker(Server): """ Worker Node Workers perform two functions: 1. **Serve data** from a local dictionary 2. **Perform computation** on that data and on data from peers Additionally workers keep a scheduler informed of their data and use that scheduler to gather data from other workers when necessary to perform a computation. You can start a worker with the ``dask-worker`` command line application:: $ dask-worker scheduler-ip:port **State** * **data:** ``{key: object}``: Dictionary mapping keys to actual values * **active:** ``{key}``: Set of keys currently under computation * **ncores:** ``int``: Number of cores used by this worker process * **executor:** ``concurrent.futures.ThreadPoolExecutor``: Executor used to perform computation * **local_dir:** ``path``: Path on local machine to store temporary files * **scheduler:** ``rpc``: Location of scheduler. See ``.ip/.port`` attributes. * **name:** ``string``: Alias * **services:** ``{str: Server}``: Auxiliary web servers running on this worker * **service_ports:** ``{str: port}``: Parameters ---------- scheduler_ip: str scheduler_port: int ip: str, optional ncores: int, optional loop: tornado.ioloop.IOLoop local_dir: str, optional Directory where we place local resources name: str, optional heartbeat_interval: int Milliseconds between heartbeats to scheduler memory_limit: int Number of bytes of data to keep in memory before using disk executor: concurrent.futures.Executor Examples -------- Use the command line to start a worker:: $ dask-scheduler Start scheduler at 127.0.0.1:8786 $ dask-worker 127.0.0.1:8786 Start worker at: 127.0.0.1:1234 Registered with scheduler at: 127.0.0.1:8786 See Also -------- distributed.scheduler.Scheduler distributed.nanny.Nanny """ def __init__(self, scheduler_ip, scheduler_port, ip=None, ncores=None, loop=None, local_dir=None, services=None, service_ports=None, name=None, heartbeat_interval=5000, memory_limit=int(TOTAL_MEMORY * 0.6), executor=None, **kwargs): self.ip = ip or get_ip() self._port = 0 self.ncores = ncores or _ncores self.local_dir = local_dir or tempfile.mkdtemp(prefix='worker-') if not os.path.exists(self.local_dir): os.mkdir(self.local_dir) self.memory_limit = memory_limit if memory_limit: try: from zict import Buffer, File, Func except ImportError: raise ImportError("Please `pip install zict` for spill-to-disk workers") path = os.path.join(self.local_dir, 'storage') storage = Func(dumps_to_disk, loads_from_disk, File(path)) self.data = Buffer({}, storage, int(float(memory_limit)), weight) else: self.data = dict() self.loop = loop or IOLoop.current() self.status = None self.executor = executor or ThreadPoolExecutor(self.ncores) self.scheduler = rpc(ip=scheduler_ip, port=scheduler_port) self.active = set() self.name = name self.heartbeat_interval = heartbeat_interval self.heartbeat_active = False self.execution_state = {'scheduler': self.scheduler.address, 'ioloop': self.loop, 'worker': self} self._last_disk_io = None self._last_net_io = None self._ipython_kernel = None if self.local_dir not in sys.path: sys.path.insert(0, self.local_dir) self.services = {} self.service_ports = service_ports or {} for k, v in (services or {}).items(): if isinstance(k, tuple): k, port = k else: port = 0 self.services[k] = v(self, io_loop=self.loop) self.services[k].listen(port) self.service_ports[k] = self.services[k].port handlers = {'compute': self.compute, 'gather': self.gather, 'compute-stream': self.compute_stream, 'run': self.run, 'get_data': self.get_data, 'update_data': self.update_data, 'delete_data': self.delete_data, 'terminate': self.terminate, 'ping': pingpong, 'health': self.host_health, 'upload_file': self.upload_file, 'start_ipython': self.start_ipython, 'keys': self.keys, } super(Worker, self).__init__(handlers, io_loop=self.loop, **kwargs) self.heartbeat_callback = PeriodicCallback(self.heartbeat, self.heartbeat_interval, io_loop=self.loop) self.loop.add_callback(self.heartbeat_callback.start) def __str__(self): return "" % (self.address, self.ncores) __repr__ = __str__ @property def worker_address(self): """ For API compatibility with Nanny """ return self.address @gen.coroutine def heartbeat(self): if not self.heartbeat_active: self.heartbeat_active = True logger.debug("Heartbeat: %s" % self.address) try: yield self.scheduler.register(address=self.address, name=self.name, ncores=self.ncores, now=time(), host_info=self.host_health(), services=self.service_ports, memory_limit=self.memory_limit, **self.process_health()) finally: self.heartbeat_active = False else: logger.debug("Heartbeat skipped: channel busy") @gen.coroutine def _start(self, port=0): self.listen(port) self.name = self.name or self.address for k, v in self.services.items(): v.listen(0) self.service_ports[k] = v.port logger.info(' Start worker at: %20s:%d', self.ip, self.port) for k, v in self.service_ports.items(): logger.info(' %16s at: %20s:%d' % (k, self.ip, v)) logger.info('Waiting to connect to: %20s:%d', self.scheduler.ip, self.scheduler.port) logger.info('-' * 49) logger.info(' Threads: %26d', self.ncores) if self.memory_limit: logger.info(' Memory: %23.2f GB', self.memory_limit / 1e9) logger.info(' Local Directory: %26s', self.local_dir) logger.info('-' * 49) while True: try: resp = yield self.scheduler.register( ncores=self.ncores, address=(self.ip, self.port), keys=list(self.data), name=self.name, nbytes=valmap(sizeof, self.data), now=time(), host_info=self.host_health(), services=self.service_ports, memory_limit=self.memory_limit, **self.process_health()) break except (OSError, StreamClosedError): logger.debug("Unable to register with scheduler. Waiting") yield gen.sleep(0.5) if resp != 'OK': raise ValueError(resp) logger.info(' Registered to: %20s:%d', self.scheduler.ip, self.scheduler.port) logger.info('-' * 49) self.status = 'running' def start(self, port=0): self.loop.add_callback(self._start, port) def identity(self, stream): return {'type': type(self).__name__, 'id': self.id, 'scheduler': (self.scheduler.ip, self.scheduler.port), 'ncores': self.ncores, 'memory_limit': self.memory_limit} @gen.coroutine def _close(self, report=True, timeout=10): self.heartbeat_callback.stop() with ignoring(RPCClosed, StreamClosedError): if report: yield gen.with_timeout(timedelta(seconds=timeout), self.scheduler.unregister(address=(self.ip, self.port)), io_loop=self.loop) self.scheduler.close_rpc() self.stop() self.executor.shutdown() if os.path.exists(self.local_dir): shutil.rmtree(self.local_dir) for k, v in self.services.items(): v.stop() self.rpc.close() self.status = 'closed' self.stop() @gen.coroutine def terminate(self, stream, report=True): yield self._close(report=report) raise Return('OK') @property def address(self): return '%s:%d' % (self.ip, self.port) @property def address_tuple(self): return (self.ip, self.port) @gen.coroutine def gather(self, stream=None, who_has=None): who_has = {k: [coerce_to_address(addr) for addr in v] for k, v in who_has.items() if k not in self.data} try: result = yield gather_from_workers(who_has) except KeyError as e: logger.warn("Could not find data", e) raise Return({'status': 'missing-data', 'keys': e.args}) else: self.data.update(result) raise Return({'status': 'OK'}) def _deserialize(self, function=None, args=None, kwargs=None, task=None): """ Deserialize task inputs and regularize to func, args, kwargs """ if function is not None: function = loads(function) if args: args = loads(args) if kwargs: kwargs = loads(kwargs) if task is not None: assert not function and not args and not kwargs function = execute_task args = (task,) return function, args or (), kwargs or {} @gen.coroutine def gather_many(self, msgs): """ Gather the data for many compute messages at once Returns ------- good: the input messages for which we have data bad: a dict of task keys for which we could not find data data: The scope in which to run tasks len(remote): the number of new keys we've gathered """ diagnostics = {} who_has = merge(msg['who_has'] for msg in msgs if 'who_has' in msg) start = time() local = {k: self.data[k] for k in who_has if k in self.data} stop = time() if stop - start > 0.005: diagnostics['disk_load_start'] = start diagnostics['disk_load_stop'] = stop who_has = {k: v for k, v in who_has.items() if k not in local} start = time() remote, bad_data = yield gather_from_workers(who_has, permissive=True) if remote: self.data.update(remote) yield self.scheduler.add_keys(address=self.address, keys=list(remote)) stop = time() if remote: diagnostics['transfer_start'] = start diagnostics['transfer_stop'] = stop data = merge(local, remote) if bad_data: missing = {msg['key']: {k for k in msg['who_has'] if k in bad_data} for msg in msgs if 'who_has' in msg} bad = {k: v for k, v in missing.items() if v} good = [msg for msg in msgs if not missing.get(msg['key'])] else: good, bad = msgs, {} raise Return([good, bad, data, len(remote), diagnostics]) @gen.coroutine def _ready_task(self, function=None, key=None, args=(), kwargs={}, task=None, who_has=None): who_has = who_has or {} diagnostics = {} start = time() data = {k: self.data[k] for k in who_has if k in self.data} stop = time() if stop - start > 0.005: diagnostics['disk_load_start'] = start diagnostics['disk_load_stop'] = stop who_has = {k: set(map(coerce_to_address, v)) for k, v in who_has.items() if k not in self.data} if who_has: try: logger.info("gather %d keys from peers", len(who_has)) diagnostics['transfer_start'] = time() other = yield gather_from_workers(who_has) diagnostics['transfer_stop'] = time() self.data.update(other) yield self.scheduler.add_keys(address=self.address, keys=list(other)) data.update(other) except KeyError as e: logger.warn("Could not find data for %s", key) raise Return({'status': 'missing-data', 'keys': e.args, 'key': key}) try: start = default_timer() function, args, kwargs = self._deserialize(function, args, kwargs, task) diagnostics['deserialization'] = default_timer() - start except Exception as e: logger.warn("Could not deserialize task", exc_info=True) emsg = error_message(e) emsg['key'] = key raise Return(emsg) # Fill args with data args2 = pack_data(args, data) kwargs2 = pack_data(kwargs, data) raise Return({'status': 'OK', 'function': function, 'args': args2, 'kwargs': kwargs2, 'diagnostics': diagnostics, 'key': key}) @gen.coroutine def executor_submit(self, key, function, *args, **kwargs): """ Safely run function in thread pool executor We've run into issues running concurrent.future futures within tornado. Apparently it's advantageous to use timeouts and periodic callbacks to ensure things run smoothly. This can get tricky, so we pull it off into an separate method. """ job_counter[0] += 1 # logger.info("%s:%d Starts job %d, %s", self.ip, self.port, i, key) future = self.executor.submit(function, *args, **kwargs) pc = PeriodicCallback(lambda: logger.debug("future state: %s - %s", key, future._state), 1000, io_loop=self.loop); pc.start() try: yield future finally: pc.stop() pass result = future.result() # logger.info("Finish job %d, %s", i, key) raise gen.Return(result) @gen.coroutine def compute_stream(self, stream): with log_errors(): logger.debug("Open compute stream") bstream = BatchedSend(interval=2, loop=self.loop) bstream.start(stream) closed = False last = gen.sleep(0) while not closed: try: msgs = yield read(stream) except StreamClosedError: break if not isinstance(msgs, list): msgs = [msgs] batch = [] for msg in msgs: op = msg.pop('op', None) if op == 'close': closed = True break elif op == 'compute-task': batch.append(msg) logger.debug("%s asked to compute %s", self.address, msg['key']) else: logger.warning("Unknown operation %s, %s", op, msg) # self.loop.add_callback(self.compute_many, bstream, msgs) last = self.compute_many(bstream, msgs) try: yield last # TODO: there might be more than one lingering except (RPCClosed, RuntimeError): pass yield bstream.close() logger.info("Close compute stream") @gen.coroutine def compute_many(self, bstream, msgs, report=False): good, bad, data, num_transferred, diagnostics = yield self.gather_many(msgs) if bad: logger.warn("Could not find data for %s", sorted(bad)) for msg in msgs: msg.pop('who_has', None) for k, v in bad.items(): bstream.send({'status': 'missing-data', 'key': k, 'keys': list(v)}) if good: futures = [self.compute_one(data, report=report, **msg) for msg in good] wait_iterator = gen.WaitIterator(*futures) result = yield wait_iterator.next() if diagnostics: result.update(diagnostics) bstream.send(result) while not wait_iterator.done(): msg = yield wait_iterator.next() bstream.send(msg) @gen.coroutine def compute_one(self, data, key=None, function=None, args=None, kwargs=None, report=False, task=None): logger.debug("Compute one on %s", key) self.active.add(key) diagnostics = dict() try: start = default_timer() function, args, kwargs = self._deserialize(function, args, kwargs, task) diagnostics['deserialization'] = default_timer() - start except Exception as e: logger.warn("Could not deserialize task", exc_info=True) emsg = error_message(e) emsg['key'] = key raise Return(emsg) # Fill args with data args2 = pack_data(args, data) kwargs2 = pack_data(kwargs, data) # Log and compute in separate thread result = yield self.executor_submit(key, apply_function, function, args2, kwargs2, self.execution_state, key) result['key'] = key result.update(diagnostics) if result['status'] == 'OK': self.data[key] = result.pop('result') if report: response = yield self.scheduler.add_keys(keys=[key], address=(self.ip, self.port)) if not response == 'OK': logger.warn('Could not report results to scheduler: %s', str(response)) else: logger.warn(" Compute Failed\n" "Function: %s\n" "args: %s\n" "kwargs: %s\n", str(funcname(function))[:1000], convert_args_to_str(args, max_len=1000), convert_kwargs_to_str(kwargs, max_len=1000), exc_info=True) logger.debug("Send compute response to scheduler: %s, %s", key, result) try: self.active.remove(key) except KeyError: pass raise Return(result) @gen.coroutine def compute(self, stream=None, function=None, key=None, args=(), kwargs={}, task=None, who_has=None, report=True): """ Execute function """ self.active.add(key) # Ready function for computation msg = yield self._ready_task(function=function, key=key, args=args, kwargs=kwargs, task=task, who_has=who_has) if msg['status'] != 'OK': try: self.active.remove(key) except KeyError: pass raise Return(msg) else: function = msg['function'] args = msg['args'] kwargs = msg['kwargs'] # Log and compute in separate thread result = yield self.executor_submit(key, apply_function, function, args, kwargs, self.execution_state, key) result['key'] = key result.update(msg['diagnostics']) if result['status'] == 'OK': self.data[key] = result.pop('result') if report: response = yield self.scheduler.add_keys(address=(self.ip, self.port), keys=[key]) if not response == 'OK': logger.warn('Could not report results to scheduler: %s', str(response)) else: logger.warn(" Compute Failed\n" "Function: %s\n" "args: %s\n" "kwargs: %s\n", str(funcname(function))[:1000], convert_args_to_str(args, max_len=1000), convert_kwargs_to_str(kwargs, max_len=1000), exc_info=True) logger.debug("Send compute response to scheduler: %s, %s", key, get_msg_safe_str(msg)) try: self.active.remove(key) except KeyError: pass raise Return(result) def run(self, stream, function=None, args=(), kwargs={}): return run(self, stream, function=function, args=args, kwargs=kwargs) @gen.coroutine def update_data(self, stream=None, data=None, report=True): self.data.update(data) if report: response = yield self.scheduler.add_keys( address=(self.ip, self.port), keys=list(data)) assert response == 'OK' info = {'nbytes': {k: sizeof(v) for k, v in data.items()}, 'status': 'OK'} raise Return(info) @gen.coroutine def delete_data(self, stream, keys=None, report=True): if keys: for key in keys: if key in self.data: del self.data[key] logger.info("Deleted %d keys", len(keys)) if report: logger.debug("Reporting loss of keys to scheduler") yield self.scheduler.remove_keys(address=self.address, keys=list(keys)) raise Return('OK') def get_data(self, stream, keys=None): return {k: to_serialize(self.data[k]) for k in keys if k in self.data} def start_ipython(self, stream): """Start an IPython kernel Returns Jupyter connection info dictionary. """ from ._ipython_utils import start_ipython if self._ipython_kernel is None: self._ipython_kernel = start_ipython( ip=self.ip, ns={'worker': self}, log=logger, ) return self._ipython_kernel.get_connection_info() def upload_file(self, stream, filename=None, data=None, load=True): out_filename = os.path.join(self.local_dir, filename) if isinstance(data, unicode): data = data.encode() with open(out_filename, 'wb') as f: f.write(data) f.flush() if load: try: name, ext = os.path.splitext(filename) if ext in ('.py', '.pyc'): logger.info("Reload module %s from .py file", name) name = name.split('-')[0] reload(import_module(name)) if ext == '.egg': sys.path.append(out_filename) pkgs = pkg_resources.find_distributions(out_filename) for pkg in pkgs: logger.info("Load module %s from egg", pkg.project_name) reload(import_module(pkg.project_name)) if not pkgs: logger.warning("Found no packages in egg file") except Exception as e: logger.exception(e) return {'status': 'error', 'exception': dumps(e)} return {'status': 'OK', 'nbytes': len(data)} def process_health(self, stream=None): d = {'active': len(self.active), 'stored': len(self.data)} return d def host_health(self, stream=None): """ Information about worker """ d = {'time': time()} try: import psutil mem = psutil.virtual_memory() d.update({'cpu': psutil.cpu_percent(), 'memory': mem.total, 'memory_percent': mem.percent}) net_io = psutil.net_io_counters() if self._last_net_io: d['network-send'] = net_io.bytes_sent - self._last_net_io.bytes_sent d['network-recv'] = net_io.bytes_recv - self._last_net_io.bytes_recv else: d['network-send'] = 0 d['network-recv'] = 0 self._last_net_io = net_io try: disk_io = psutil.disk_io_counters() except RuntimeError: # This happens when there is no physical disk in worker pass else: if self._last_disk_io: d['disk-read'] = disk_io.read_bytes - self._last_disk_io.read_bytes d['disk-write'] = disk_io.write_bytes - self._last_disk_io.write_bytes else: d['disk-read'] = 0 d['disk-write'] = 0 self._last_disk_io = disk_io except ImportError: pass return d def keys(self, stream=None): return list(self.data) job_counter = [0] def execute_task(task): """ Evaluate a nested task >>> inc = lambda x: x + 1 >>> execute_task((inc, 1)) 2 >>> execute_task((sum, [1, 2, (inc, 3)])) 7 """ if istask(task): func, args = task[0], task[1:] return func(*map(execute_task, args)) elif isinstance(task, list): return list(map(execute_task, task)) else: return task cache = dict() def dumps_function(func): """ Dump a function to bytes, cache functions """ if func not in cache: b = dumps(func) cache[func] = b return cache[func] def dumps_task(task): """ Serialize a dask task Returns a dict of bytestrings that can each be loaded with ``loads`` Examples -------- Either returns a task as a function, args, kwargs dict >>> from operator import add >>> dumps_task((add, 1)) # doctest: +SKIP {'function': b'\x80\x04\x95\x00\x8c\t_operator\x94\x8c\x03add\x94\x93\x94.' 'args': b'\x80\x04\x95\x07\x00\x00\x00K\x01K\x02\x86\x94.'} Or as a single task blob if it can't easily decompose the result. This happens either if the task is highly nested, or if it isn't a task at all >>> dumps_task(1) # doctest: +SKIP {'task': b'\x80\x04\x95\x03\x00\x00\x00\x00\x00\x00\x00K\x01.'} """ if istask(task): if task[0] is apply and not any(map(_maybe_complex, task[2:])): d = {'function': dumps_function(task[1]), 'args': dumps(task[2])} if len(task) == 4: d['kwargs'] = dumps(task[3]) return d elif not any(map(_maybe_complex, task[1:])): return {'function': dumps_function(task[0]), 'args': dumps(task[1:])} return to_serialize(task) def apply_function(function, args, kwargs, execution_state, key): """ Run a function, collect information Returns ------- msg: dictionary with status, result/error, timings, etc.. """ thread_state.execution_state = execution_state thread_state.key = key start = time() try: result = function(*args, **kwargs) except Exception as e: msg = error_message(e) else: msg = {'status': 'OK', 'result': result, 'nbytes': sizeof(result), 'type': dumps_function(type(result)) if result is not None else None} finally: end = time() msg['compute_start'] = start msg['compute_stop'] = end msg['thread'] = current_thread().ident return msg def get_msg_safe_str(msg): """ Make a worker msg, which contains args and kwargs, safe to cast to str: allowing for some arguments to raise exceptions during conversion and ignoring them. """ class Repr(object): def __init__(self, f, val): self._f = f self._val = val def __repr__(self): return self._f(self._val) msg = msg.copy() if "args" in msg: msg["args"] = Repr(convert_args_to_str, msg["args"]) if "kwargs" in msg: msg["kwargs"] = Repr(convert_kwargs_to_str, msg["kwargs"]) return msg def convert_args_to_str(args, max_len=None): """ Convert args to a string, allowing for some arguments to raise exceptions during conversion and ignoring them. """ length = 0 strs = ["" for i in range(len(args))] for i, arg in enumerate(args): try: sarg = repr(arg) except: sarg = "< could not convert arg to str >" strs[i] = sarg length += len(sarg) + 2 if max_len is not None and length > max_len: return "({}".format(", ".join(strs[:i+1]))[:max_len] else: return "({})".format(", ".join(strs)) def convert_kwargs_to_str(kwargs, max_len=None): """ Convert kwargs to a string, allowing for some arguments to raise exceptions during conversion and ignoring them. """ length = 0 strs = ["" for i in range(len(kwargs))] for i, (argname, arg) in enumerate(kwargs.items()): try: sarg = repr(arg) except: sarg = "< could not convert arg to str >" skwarg = repr(argname) + ": " + sarg strs[i] = skwarg length += len(skwarg) + 2 if max_len is not None and length > max_len: return "{{{}".format(", ".join(strs[:i+1]))[:max_len] else: return "{{{}}}".format(", ".join(strs)) from .protocol import compressions, default_compression, to_serialize # TODO: use protocol.maybe_compress and proper file/memoryview objects def dumps_to_disk(x): b = dumps(x) c = compressions[default_compression]['compress'](b) return c def loads_from_disk(c): b = compressions[default_compression]['decompress'](c) x = loads(b) return x def weight(k, v): return sizeof(v) @gen.coroutine def run(worker, stream, function=None, args=(), kwargs={}): function = loads(function) if args: args = loads(args) if kwargs: kwargs = loads(kwargs) try: import inspect if 'dask_worker' in inspect.getargspec(function).args: kwargs['dask_worker'] = worker except: pass try: result = function(*args, **kwargs) except Exception as e: logger.warn(" Run Failed\n" "Function: %s\n" "args: %s\n" "kwargs: %s\n", str(funcname(function))[:1000], convert_args_to_str(args, max_len=1000), convert_kwargs_to_str(kwargs, max_len=1000), exc_info=True) response = error_message(e) else: response = { 'status': 'OK', 'result': to_serialize(result), } raise Return(response) distributed-1.14.3/distributed/worker_client.py000066400000000000000000000142061301211143400216560ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import from contextlib import contextmanager from tornado import gen from toolz import keymap, valmap, merge import uuid import uuid from dask.base import tokenize from tornado import gen from .client import Client, Future, pack_data, unpack_remotedata from .sizeof import sizeof from .threadpoolexecutor import secede from .utils import log_errors, sync, tokey, ignoring from .worker import thread_state @contextmanager def local_client(): """ Get client for this thread Note: This interface is new and experimental. It may change without notice. This context manager is intended to be called within functions that we run on workers. When run as a context manager it delivers a client ``Client`` object that can submit other tasks directly from that worker. Examples -------- >>> def func(x): ... with local_client() as e: # connect from worker back to scheduler ... a = e.submit(inc, x) # this task can submit more tasks ... b = e.submit(dec, x) ... result = e.gather([a, b]) # and gather results ... return result >>> future = e.submit(func, 1) # submit func(1) on cluster """ address = thread_state.execution_state['scheduler'] secede() # have this thread secede from the thread pool # so that it doesn't take up a fixed resource while waiting with WorkerClient(address) as e: e.worker.loop.add_callback(e.worker.scheduler.change_worker_cores, worker=e.worker.address, diff=+1) try: yield e finally: e.worker.loop.add_callback(e.worker.scheduler.change_worker_cores, worker=e.worker.address, diff=-1) def get_worker(): return thread_state.execution_state['worker'] class WorkerClient(Client): """ An Client designed to operate from a Worker process This client has had a few methods altered to make it more efficient for working directly from the worker nodes. In particular scatter/gather first look to the local data dictionary rather than sending data over the network """ def __init__(self, *args, **kwargs): self.worker = get_worker() Client.__init__(self, *args, **kwargs) @gen.coroutine def _scatter(self, data, workers=None, broadcast=False): """ Scatter data to local data dictionary Rather than send data out to the cluster we keep data local. However we do report to the scheduler that the local worker has the scattered data. This allows other workers to come by and steal this data if desired. Keywords like ``broadcast=`` do not work, however operations like ``.replicate`` work fine after calling scatter, which can fill in for this functionality. """ with log_errors(): if not (workers is None and broadcast is False): raise NotImplementedError("Scatter from worker doesn't support workers or broadcast keywords") if isinstance(data, dict) and not all(isinstance(k, (bytes, str)) for k in data): d = yield self._scatter(keymap(tokey, data), workers, broadcast) raise gen.Return({k: d[tokey(k)] for k in data}) if isinstance(data, (list, tuple, set, frozenset)): keys = [] for x in data: try: keys.append(tokenize(x)) except: keys.append(str(uuid.uuid1())) data2 = dict(zip(keys, data)) elif isinstance(data, dict): keys = set(data) data2 = data else: raise TypeError("Don't know how to scatter %s" % type(data)) nbytes = valmap(sizeof, data2) # self.worker.data.update(data2) # thread safety matters self.worker.loop.add_callback(self.worker.data.update, data2) yield self.scheduler.update_data( who_has={key: [self.worker.address] for key in data2}, nbytes=valmap(sizeof, data2), client=self.id) if isinstance(data, dict): out = {k: Future(k, self) for k in data} elif isinstance(data, (tuple, list, set, frozenset)): out = type(data)([Future(k, self) for k in keys]) else: raise TypeError( "Input to scatter must be a list or dict") for key in keys: self.futures[key]['status'] = 'finished' self.futures[key]['event'].set() raise gen.Return(out) @gen.coroutine def _gather(self, futures, errors='raise'): """ Exactly like Client._gather, but get data directly from the local worker data dictionary directly rather than through the scheduler. TODO: avoid scheduler for other communications, and assume that we can communicate directly with the other workers. """ futures2, keys = unpack_remotedata(futures, byte_keys=True) keys = [tokey(k) for k in keys] @gen.coroutine def wait(k): """ Want to stop the All(...) early if we find an error """ yield self.futures[k]['event'].wait() if self.futures[k]['status'] != 'finished': raise Exception() with ignoring(Exception): yield All([wait(key) for key in keys if key in self.futures]) while True: # not threadsafe, so try until success try: local = {k: self.worker.data[k] for k in keys if k in self.worker.data} break except KeyError as e: pass futures3 = {k: Future(k, self) for k in keys if k not in local} futures4 = pack_data(futures2, merge(local, futures3)) if not futures3: raise gen.Return(futures4) result = yield Client._gather(self, futures4, errors=errors) raise gen.Return(result) distributed-1.14.3/docs/000077500000000000000000000000001301211143400150405ustar00rootroot00000000000000distributed-1.14.3/docs/Makefile000066400000000000000000000164161301211143400165100ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " applehelp to make an Apple Help Book" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" @echo " coverage to run coverage check of the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/distributed.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/distributed.qhc" applehelp: $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp @echo @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." @echo "N.B. You won't be able to view it unless you put it in" \ "~/Library/Documentation/Help or install it in your application" \ "bundle." devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/distributed" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/distributed" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." coverage: $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage @echo "Testing of coverage in the sources finished, look at the " \ "results in $(BUILDDIR)/coverage/python.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." distributed-1.14.3/docs/make.bat000066400000000000000000000161371301211143400164550ustar00rootroot00000000000000@ECHO OFF REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set BUILDDIR=build set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source set I18NSPHINXOPTS=%SPHINXOPTS% source if NOT "%PAPER%" == "" ( set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% ) if "%1" == "" goto help if "%1" == "help" ( :help echo.Please use `make ^` where ^ is one of echo. html to make standalone HTML files echo. dirhtml to make HTML files named index.html in directories echo. singlehtml to make a single large HTML file echo. pickle to make pickle files echo. json to make JSON files echo. htmlhelp to make HTML files and a HTML help project echo. qthelp to make HTML files and a qthelp project echo. devhelp to make HTML files and a Devhelp project echo. epub to make an epub echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter echo. text to make text files echo. man to make manual pages echo. texinfo to make Texinfo files echo. gettext to make PO message catalogs echo. changes to make an overview over all changed/added/deprecated items echo. xml to make Docutils-native XML files echo. pseudoxml to make pseudoxml-XML files for display purposes echo. linkcheck to check all external links for integrity echo. doctest to run all doctests embedded in the documentation if enabled echo. coverage to run coverage check of the documentation if enabled goto end ) if "%1" == "clean" ( for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i del /q /s %BUILDDIR%\* goto end ) REM Check if sphinx-build is available and fallback to Python version if any %SPHINXBUILD% 2> nul if errorlevel 9009 goto sphinx_python goto sphinx_ok :sphinx_python set SPHINXBUILD=python -m sphinx.__init__ %SPHINXBUILD% 2> nul if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.http://sphinx-doc.org/ exit /b 1 ) :sphinx_ok if "%1" == "html" ( %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/html. goto end ) if "%1" == "dirhtml" ( %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. goto end ) if "%1" == "singlehtml" ( %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. goto end ) if "%1" == "pickle" ( %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the pickle files. goto end ) if "%1" == "json" ( %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the JSON files. goto end ) if "%1" == "htmlhelp" ( %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run HTML Help Workshop with the ^ .hhp project file in %BUILDDIR%/htmlhelp. goto end ) if "%1" == "qthelp" ( %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run "qcollectiongenerator" with the ^ .qhcp project file in %BUILDDIR%/qthelp, like this: echo.^> qcollectiongenerator %BUILDDIR%\qthelp\distributed.qhcp echo.To view the help file: echo.^> assistant -collectionFile %BUILDDIR%\qthelp\distributed.ghc goto end ) if "%1" == "devhelp" ( %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp if errorlevel 1 exit /b 1 echo. echo.Build finished. goto end ) if "%1" == "epub" ( %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub if errorlevel 1 exit /b 1 echo. echo.Build finished. The epub file is in %BUILDDIR%/epub. goto end ) if "%1" == "latex" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex if errorlevel 1 exit /b 1 echo. echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. goto end ) if "%1" == "latexpdf" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex cd %BUILDDIR%/latex make all-pdf cd %~dp0 echo. echo.Build finished; the PDF files are in %BUILDDIR%/latex. goto end ) if "%1" == "latexpdfja" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex cd %BUILDDIR%/latex make all-pdf-ja cd %~dp0 echo. echo.Build finished; the PDF files are in %BUILDDIR%/latex. goto end ) if "%1" == "text" ( %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text if errorlevel 1 exit /b 1 echo. echo.Build finished. The text files are in %BUILDDIR%/text. goto end ) if "%1" == "man" ( %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man if errorlevel 1 exit /b 1 echo. echo.Build finished. The manual pages are in %BUILDDIR%/man. goto end ) if "%1" == "texinfo" ( %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo if errorlevel 1 exit /b 1 echo. echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. goto end ) if "%1" == "gettext" ( %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale if errorlevel 1 exit /b 1 echo. echo.Build finished. The message catalogs are in %BUILDDIR%/locale. goto end ) if "%1" == "changes" ( %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes if errorlevel 1 exit /b 1 echo. echo.The overview file is in %BUILDDIR%/changes. goto end ) if "%1" == "linkcheck" ( %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck if errorlevel 1 exit /b 1 echo. echo.Link check complete; look for any errors in the above output ^ or in %BUILDDIR%/linkcheck/output.txt. goto end ) if "%1" == "doctest" ( %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest if errorlevel 1 exit /b 1 echo. echo.Testing of doctests in the sources finished, look at the ^ results in %BUILDDIR%/doctest/output.txt. goto end ) if "%1" == "coverage" ( %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage if errorlevel 1 exit /b 1 echo. echo.Testing of coverage in the sources finished, look at the ^ results in %BUILDDIR%/coverage/python.txt. goto end ) if "%1" == "xml" ( %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml if errorlevel 1 exit /b 1 echo. echo.Build finished. The XML files are in %BUILDDIR%/xml. goto end ) if "%1" == "pseudoxml" ( %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml if errorlevel 1 exit /b 1 echo. echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. goto end ) :end distributed-1.14.3/docs/requirements.txt000066400000000000000000000000501301211143400203170ustar00rootroot00000000000000tornado toolz cloudpickle dask numpydoc distributed-1.14.3/docs/source/000077500000000000000000000000001301211143400163405ustar00rootroot00000000000000distributed-1.14.3/docs/source/adaptive.rst000066400000000000000000000135161301211143400206750ustar00rootroot00000000000000Adaptive Deployments ==================== It is possible to grow and shrink Dask clusters based on current use. This allows you to run Dask permanently on your cluster and have it only take up resources when necessary. Dask contains the logic about when to grow and shrink but relies on external cluster managers to launch and kill ``dask-worker`` jobs. This page describes the policies about adaptively resizing Dask clusters based on load, how to connect these policies to a particular job scheduler, and an example implementation. Dynamically scaling a Dask cluster up and down requires tight integration with an external cluster management system that can deploy ``dask-worker`` jobs throughout the cluster. Several systems are in wide use today, including common examples like SGE, SLURM, Torque, Condor, LSF, Yarn, Mesos, Marathon, Kubernetes, etc.. These systems can be quite different from each other, but all are used to manage distributed services throughout different kinds of clusters. The large number of relevant systems, the challenges of rigorously testing each, and finite development time precludes the systematic inclusion of all solutions within the dask/distributed repository. Instead, we include a generic interface that can be extended by someone with basic understanding of their cluster management tool. We encourage these as third party modules. Policies -------- We control the number of workers based on current load and memory use. The scheduler checks itself periodically to determine if more or fewer workers are needed. If there are excess unclaimed tasks, or if the memory of the current workers is more nearing full then the scheduler tries to increase the number of workers by a fixed factor, defaulting to 2. This causes exponential growth while growth is useful. If there are idle workers and if the memory of the current workers is nearing empty then we gracefully retire the idle workers with the least amount of data in memory. We first move these results to the surviving workers and then remove the idle workers from the cluster. This shrinks the cluster while gracefully preserving intermediate results, shrinking the cluster when excess size is not useful. Adaptive class interface ------------------------ The ``distributed.deploy.Adaptive`` class contains the logic about when to ask for new workers, and when to close idle ones. This class requires both a scheduler and a cluster object. The cluster object must support two methods, ``scale_up(n)``, which takes in a target number of total workers for the cluster and ``scale_down(workers)``, which takes in a list of addresses to remove from the cluster. The Adaptive class will call these methods with the correct values at the correct times. .. code-block:: python class MyCluster(object): @gen.coroutine def scale_up(n): """ Bring the total count of workers up to ``n`` This function/coroutine should bring the total number of workers up to the number ``n``. This can be implemented either as a function or as a Tornado coroutine. """ raise NotImplementedError() @gen.coroutine def scale_down(self, workers): """ Remove ``workers`` from the cluster Given a list of worker addresses this function should remove those workers from the cluster. This may require tracking which jobs are associated to which worker address. This can be implemented either as a function or as a Tornado coroutine. """ from distributed.deploy import Adaptive scheduler = Scheduler() cluster = MyCluster() adapative_cluster = Adaptive(scheduler, cluster) Implementing these ``scale_up`` and ``scale_down`` functions depends strongly on the cluster management system. See :doc:`LocalCluster ` for an example. Marathon: an example -------------------- We now present an example project that implements this cluster interface backed by the Marathon cluster management tool on Mesos. Full source code and testing apparatus is available here: http://github.com/mrocklin/dask-marathon The implementation is small. It uses the Marathon HTTP API through the `marathon Python client library `_. We reproduce the full body of the implementation below as an example: .. code-block:: python from marathon import MarathonClient, MarathonApp from marathon.models.container import MarathonContainer class MarathonCluster(object): def __init__(self, scheduler, executable='dask-worker', docker_image='mrocklin/dask-distributed', marathon_address='http://localhost:8080', name=None, **kwargs): self.scheduler = scheduler # Create Marathon App to run dask-worker args = [executable, scheduler.address, '--name', '$MESOS_TASK_ID'] # use Mesos task ID as worker name if 'mem' in kwargs: args.extend(['--memory-limit', str(int(kwargs['mem'] * 0.6 * 1e6))]) kwargs['cmd'] = ' '.join(args) container = MarathonContainer({'image': docker_image}) app = MarathonApp(instances=0, container=container, **kwargs) # Connect and register app self.client = MarathonClient(marathon_address) self.app = self.client.create_app(name or 'dask-%s' % uuid.uuid4(), app) def scale_up(self, instances): self.marathon_client.scale_app(self.app.id, instances=instances) def scale_down(self, workers): for w in workers: self.marathon_client.kill_task(self.app.id, self.scheduler.worker_info[w]['name'], scale=True) distributed-1.14.3/docs/source/api.rst000066400000000000000000000032521301211143400176450ustar00rootroot00000000000000API === .. currentmodule:: distributed.client **Client** .. autosummary:: Client Client.cancel Client.compute Client.gather Client.get Client.get_dataset Client.has_what Client.list_datasets Client.map Client.ncores Client.persist Client.publish_dataset Client.rebalance Client.replicate Client.restart Client.run Client.scatter Client.shutdown Client.scheduler_info Client.shutdown Client.start_ipython_workers Client.start_ipython_scheduler Client.submit Client.unpublish_dataset Client.upload_file Client.who_has **Future** .. autosummary:: Future Future.add_done_callback Future.cancel Future.cancelled Future.done Future.exception Future.result Future.traceback **Other** .. autosummary:: as_completed distributed.diagnostics.progress wait Asynchronous methods -------------------- If you desire Tornado coroutines rather than typical functions these can commonly be found as underscore-prefixed versions of the functions above. For example the ``client.restart()`` method can be replaced in an asynchronous workflow with ``yield client._restart()``. Many methods like ``client.compute`` are non-blocking regardless; these do not have a coroutine-equivalent. .. code-block:: python client.restart() # synchronous yield client._restart() # non-blocking Client -------- .. autoclass:: Client :members: CompatibleExecutor ------------------ .. autoclass:: CompatibleExecutor :members: map Future ------ .. autoclass:: Future :members: Other ----- .. autofunction:: as_completed .. autofunction:: distributed.diagnostics.progress .. autofunction:: wait distributed-1.14.3/docs/source/client.rst000066400000000000000000000144341301211143400203560ustar00rootroot00000000000000Client ======== The Client is the primary entry point for users of ``dask.distributed``. After we :doc:`setup a cluster `, we initialize a ``Client`` by pointing it to the address of a ``Scheduler``: .. code-block:: python >>> from distributed import Client >>> client = Client('127.0.0.1:8786') There are a few different ways to interact with the cluster through the client: 1. The Client satisfies most of the standard concurrent.futures_ - PEP-3148_ interface with ``.submit``, ``.map`` functions and ``Future`` objects, allowing the immediate and direct submission of tasks. 2. The Client registers itself as the default Dask_ scheduler, and so runs all dask collections like dask.array_, dask.bag_, dask.dataframe_ and dask.delayed_ 3. The Client has additional methods for manipulating data remotely. See the full :doc:`API ` for a thorough list. Concurrent.futures ------------------ We can submit individual function calls with the ``client.submit`` method or many function calls with the ``client.map`` method .. code-block:: python >>> def inc(x): return x + 1 >>> x = client.submit(inc, 10) >>> x >>> L = client.map(inc, range(1000)) >>> L [, , , , ...] These results live on distributed workers. We can submit tasks on futures. The function will go to the machine where the futures are stored and run on the result once it has completed. .. code-block:: python >>> y = client.submit(inc, x) # Submit on x, a Future >>> total = client.submit(sum, L) # Map on L, a list of Futures We gather back the results using either the ``Future.result`` method for single futures or ``client.gather`` method for many futures at once. .. code-block:: python >>> x.result() 11 >>> client.gather(L) [1, 2, 3, 4, 5, ...] But, as always, we want to minimize communicating results back to the local process. It's often best to leave data on the cluster and operate on it remotely with functions like ``submit``, ``map``, ``get`` and ``compute``. See :doc:`efficiency ` for more information on efficient use of distributed. Dask ---- The parent library Dask_ contains objects like dask.array_, dask.dataframe_, dask.bag_, and dask.delayed_, which automatically produce parallel algorithms on larger datasets. All dask collections work smoothly with the distributed scheduler. When we create a ``Client`` object it registers itself as the default Dask scheduler. All ``.compute()`` methods will automatically start using the distributed system. .. code-block:: python client = Client('scheduler:8786') my_dataframe.sum().compute() # Now uses the distributed system by default We can stop this behavior by using the ``set_as_default=False`` keyword argument when starting the Client. Dask's normal ``.compute()`` methods are *synchronous*, meaning that they block the interpreter until they complete. Dask.distributed allows the new ability of *asynchronous* computing, we can trigger computations to occur in the background and persist in memory while we continue doing other work. This is typically handled with the ``Client.persist`` and ``Client.compute`` methods which are used for larger and smaller result sets respectively. .. code-block:: python >>> df = client.persist(df) # trigger all computations, keep df in memory >>> type(df) dask.DataFrame For more information see the page on :doc:`Managing Computation `. Pure Functions by Default ------------------------- By default we assume that all functions are pure_. If this is not the case we should use the ``pure=False`` keyword argument. The client associates a key to all computations. This key is accessible on the Future object. .. code-block:: python >>> from operator import add >>> x = client.submit(add, 1, 2) >>> x.key 'add-ebf39f96ad7174656f97097d658f3fa2' This key should be the same across all computations with the same inputs and across all machines. If we run the computation above on any computer with the same environment then we should get the exact same key. The scheduler avoids redundant computations. If the result is already in memory from a previous call then that old result will be used rather than recomputing it. Calls to submit or map are idempotent in the common case. While convenient, this feature may be undesired for impure functions, like ``random``. In these cases two calls to the same function with the same inputs should produce different results. We accomplish this with the ``pure=False`` keyword argument. In this case keys are randomly generated (by ``uuid4``.) .. code-block:: python >>> import numpy as np >>> client.submit(np.random.random, 1000, pure=False).key 'random_sample-fc814a39-ee00-42f3-8b6f-cac65bcb5556' >>> client.submit(np.random.random, 1000, pure=False).key 'random_sample-a24e7220-a113-47f2-a030-72209439f093' .. _pure: https://toolz.readthedocs.io/en/latest/purity.html Tornado Coroutines ------------------ If we are operating in an asynchronous environment then all blocking functions listed above have asynchronous equivalents. Currently these have the exact same name but are prepended with an underscore (``_``) so, ``.result`` is synchronous while ``._result`` is asynchronous. If a function has no asynchronous counterpart then that means it does not significantly block. The ``.submit`` and ``.map`` functions are examples of this; they return immediately in either case. Additional Links ---------------- For more information on how to use dask.distributed you may want to look at the following pages: * :doc:`Managing Memory ` * :doc:`Managing Computation ` * :doc:`Data Locality ` * :doc:`API ` .. _concurrent.futures: https://docs.python.org/3/library/concurrent.futures.html .. _PEP-3148: https://www.python.org/dev/peps/pep-3148/ .. _dask.array: http://dask.pydata.org/en/latest/array.html .. _dask.bag: http://dask.pydata.org/en/latest/bag.html .. _dask.dataframe: http://dask.pydata.org/en/latest/dataframe.html .. _dask.delayed: http://dask.pydata.org/en/latest/delayed.html .. _Dask: http://dask.pydata.org/en/latest/ distributed-1.14.3/docs/source/conf.py000066400000000000000000000267141301211143400176510ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # Dask.distributed documentation build configuration file, created by # sphinx-quickstart on Tue Oct 6 14:42:44 2015. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys import os import shlex # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinx.ext.autosummary', 'numpydoc', ] numpydoc_show_class_members = False # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'Dask.distributed' copyright = u'2016, Continuum Analytics' author = u'Continuum Analytics' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. import distributed version = distributed.__version__ # The full version, including alpha/beta/rc tags. release = distributed.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = [] # The reST default role (used for this markup: `text`) to use for all # documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True # -- Options for HTML output ---------------------------------------------- # Taken from docs.readthedocs.io: # on_rtd is whether we are on readthedocs.io on_rtd = os.environ.get('READTHEDOCS', None) == 'True' if not on_rtd: # only import and set the theme if we're building docs locally import sphinx_rtd_theme html_theme = 'sphinx_rtd_theme' html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. #html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' #html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # Now only 'ja' uses this config value #html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. #html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. htmlhelp_basename = 'distributeddoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', # Latex figure (float) alignment #'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'distributed.tex', u'Dask.distributed Documentation', u'Matthew Rocklin', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ (master_doc, 'Dask.distributed', u'Dask.distributed Documentation', [author], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'Dask.distributed', u'Dask.distributed Documentation', author, 'Dask.distributed', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False # -- Options for Epub output ---------------------------------------------- # Bibliographic Dublin Core info. epub_title = project epub_author = author epub_publisher = author epub_copyright = copyright # The basename for the epub file. It defaults to the project name. #epub_basename = project # The HTML theme for the epub output. Since the default themes are not optimized # for small screen space, using the same theme for HTML and epub output is # usually not wise. This defaults to 'epub', a theme designed to save visual # space. #epub_theme = 'epub' # The language of the text. It defaults to the language option # or 'en' if the language is not set. #epub_language = '' # The scheme of the identifier. Typical schemes are ISBN or URL. #epub_scheme = '' # The unique identifier of the text. This can be a ISBN number # or the project homepage. #epub_identifier = '' # A unique identification for the text. #epub_uid = '' # A tuple containing the cover image and cover page html template filenames. #epub_cover = () # A sequence of (type, uri, title) tuples for the guide element of content.opf. #epub_guide = () # HTML files that should be inserted before the pages created by sphinx. # The format is a list of tuples containing the path and title. #epub_pre_files = [] # HTML files shat should be inserted after the pages created by sphinx. # The format is a list of tuples containing the path and title. #epub_post_files = [] # A list of files that should not be packed into the epub file. epub_exclude_files = ['search.html'] # The depth of the table of contents in toc.ncx. #epub_tocdepth = 3 # Allow duplicate toc entries. #epub_tocdup = True # Choose between 'default' and 'includehidden'. #epub_tocscope = 'default' # Fix unsupported image types using the Pillow. #epub_fix_images = False # Scale large images. #epub_max_image_width = 0 # How to display URL addresses: 'footnote', 'no', or 'inline'. #epub_show_urls = 'inline' # If false, no index is generated. #epub_use_index = True distributed-1.14.3/docs/source/develop.rst000066400000000000000000000127731301211143400205420ustar00rootroot00000000000000Development Guidelines ====================== This repository is part of the Dask_ projects. General development guidelines including where to ask for help, a layout of repositories, testing practices, and documentation and style standards are available at the `Dask developer guidelines`_ in the main documentation. .. _Dask: http://dask.pydata.org/en/latest/ .. _`Dask developer guidelines`: http://dask.pydata.org/en/latest/develop.html Install ------- After setting up an environment as described in the `Dask developer guidelines`_ you can clone this repository with git:: git clone git@github.com:dask/distributed.git and install it from source:: cd distributed python setup.py install Test ---- Test using ``py.test``:: py.test distributed --verbose Tornado ------- Dask.distributed is a Tornado TCP application. Tornado provides us with both a communication layer on top of sockets, as well as a syntax for writing asynchronous coroutines, similar to asyncio. You can make modest changes to the policies within this library without understanding much about Tornado, however moderate changes will probably require you to understand Tornado IOLoops, coroutines, and a little about non-blocking communication.. The Tornado API documentation is quite good and we recommend that you read the following resources: * http://www.tornadoweb.org/en/stable/gen.html * http://www.tornadoweb.org/en/stable/ioloop.html Additionally, if you want to interact at a low level with the communication between workers and scheduler then you should understand the Tornado ``TCPServer`` and ``IOStream`` available here: * http://www.tornadoweb.org/en/stable/networking.html Dask.distributed wraps a bit of logic around Tornado. See :doc:`Foundations` for more information. Writing Tests ------------- Testing distributed systems is normally quite difficult because it is difficult to inspect the state of all components when something goes wrong. Fortunately, the non-blocking asynchronous model within Tornado allows us to run a scheduler, multiple workers, and multiple clients all within a single thread. This gives us predictable performance, clean shutdowns, and the ability to drop into any point of the code during execution. At the same time, sometimes we want everything to run in different processes in order to simulate a more realistic setting. The test suite contains three kinds of tests 1. ``@gen_cluster``: Fully asynchronous tests where all components live in the same event loop in the main thread. These are good for testing complex logic and inspecting the state of the system directly. They are also easier to debug and cause the fewest problems with shutdowns. 2. ``with cluster()``: Tests with multiple processes forked from the master process. These are good for testing the synchronous (normal user) API and when triggering hard failures for resilience tests. 3. ``popen``: Tests that call out to the command line to start the system. These are rare and mostly for testing the command line interface. If you are comfortable with the Tornado interface then you will be happiest using the ``@gen_cluster`` style of test .. code-block:: python @gen_cluster(client=True) def test_submit(c, s, a, b): assert isinstance(c, Client) assert isinstance(c, Scheduler) assert isinstance(a, Worker) assert isinstance(b, Worker) future = c.submit(inc, 1) assert future.key in c.futures # result = future.result() # This synchronous API call would block result = yield future._result() assert result == 2 assert future.key in s.tasks assert future.key in a.data or future.key in b.data The ``@gen_cluster`` decorator sets up a scheduler, client, and workers for you and cleans them up after the test. It also allows you to directly inspect the state of every element of the cluster directly. However, you can not use the normal synchronous API (doing so will cause the test to wait forever) and instead you need to use the coroutine API, where all blocking functions are prepended with an underscore (``_``). Beware, it is a common mistake to use the blocking interface within these tests. If you want to test the normal synchronous API you can use a ``with cluster`` style test, which sets up a scheduler and workers for you in different forked processes: .. code-block:: python def test_submit_sync(loop): with cluster() as (s, [a, b]): with Client(('127.0.0.1', s['port']), loop=loop) as c: future = c.submit(inc, 1) assert future.key in c.futures result = future.result() # use the synchronous/blocking API here assert result == 2 a['proc'].terminate() # kill one of the workers result = future.result() # test that future remains valid assert result == 2 In this style of test you do not have access to the scheduler or workers. The variables ``s, a, b`` are now dictionaries holding a ``multiprocessing.Process`` object and a port integer. However, you can now use the normal synchronous API (never use yield in this style of test) and you can close processes easily by terminating them. Typically for most user-facing functions you will find both kinds of tests. The ``@gen_cluster`` tests test particular logic while the ``with cluster`` tests test basic interface and resilience. You should avoid ``popen`` style tests unless absolutely necessary, such as if you need to test the command line interface. distributed-1.14.3/docs/source/ec2.rst000066400000000000000000000051231301211143400175440ustar00rootroot00000000000000EC2 Startup Script ================== First, add your AWS credentials to ``~/.aws/credentials`` like this:: [default] aws_access_key_id = YOUR_ACCESS_KEY aws_secret_access_key = YOUR_SECRET_KEY For other ways to manage or troubleshoot credentials, see the `boto3 docs `_. Now, you can quickly deploy a scheduler and workers on EC2 using the ``dask-ec2`` quickstart application:: pip install dask-ec2 dask-ec2 up --keyname YOUR-AWS-KEY --keypair ~/.ssh/YOUR-AWS-SSH-KEY.pem This provisions a cluster on Amazon's EC2 cloud service, installs Anaconda, and sets up a scheduler and workers. In then prints out instructions on how to connect to the cluster. Options ------- The ``dask-ec2`` startup script comes with the following options for creating a cluster:: $ dask-ec2 up --help Usage: dask-ec2 up [OPTIONS] Options: --keyname TEXT Keyname on EC2 console [required] --keypair PATH Path to the keypair that matches the keyname [required] --name TEXT Tag name on EC2 --region-name TEXT AWS region [default: us-east-1] --ami TEXT EC2 AMI [default: ami-d05e75b8] --username TEXT User to SSH to the AMI [default: ubuntu] --type TEXT EC2 Instance Type [default: m3.2xlarge] --count INTEGER Number of nodes [default: 4] --security-group TEXT Security Group Name [default: dask-ec2-default] --volume-type TEXT Root volume type [default: gp2] --volume-size INTEGER Root volume size (GB) [default: 500] --file PATH File to save the metadata [default: cluster.yaml] --provision / --no-provision Provision salt on the nodes [default: True] --dask / --no-dask Install Dask.Distributed in the cluster [default: True] --nprocs INTEGER Number of processes per worker [default: 1] -h, --help Show this message and exit. Connect ------- Connection instructions follow successful completion of the ``dask-ec2 up`` command. The involve the following:: dask-ec2 ssh # SSH into head node ipython # Start IPython console on head node .. code-block:: python >>> from distributed import Client, s3, progress >>> c = Client('127.0.0.1:8786') This client now has access to all the cores of your cluster. Destroy ------- You can destroy your cluster from your local machine with the destroy command:: dask-ec2 destroy distributed-1.14.3/docs/source/efficiency.rst000066400000000000000000000065231301211143400212040ustar00rootroot00000000000000Efficiency ========== Parallel computing done well is responsive and rewarding. However, several speed-bumps can get in the way. This section describes common ways to ensure performance. Leave data on the cluster ------------------------- Wait as long as possible to gather data locally. If you want to ask a question of a large piece of data on the cluster it is often faster to submit a function onto that data then to bring the data down to your local computer. For example if we have a numpy array on the cluster and we want to know its shape we might choose one of the following options: 1. **Slow:** Gather the numpy array to the local process, access the ``.shape`` attribute 2. **Fast:** Send a lambda function up to the cluster to compute the shape .. code-block:: python >>> x = client.submit(np.random.random, (1000, 1000)) >>> type(x) Future **Slow** .. code-block:: python >>> x.result().shape() # Slow from lots of data transfer (1000, 1000) **Fast** .. code-block:: python >>> client.submit(lambda a: a.shape, x).result() # fast (1000, 1000) Use larger tasks ---------------- The scheduler adds about *one millisecond* of overhead per task or Future object. While this may sound fast it's quite slow if you run a billion tasks. If your functions run faster than 100ms or so then you might not see any speedup from using distributed computing. A common solution is to batch your input into larger chunks. **Slow** .. code-block:: python >>> futures = client.map(f, seq) >>> len(futures) # avoid large numbers of futures 1000000000 **Fast** .. code-block:: python >>> def f_many(chunk): ... return [f(x) for x in chunk] >>> from toolz import partition_all >>> chunks = partition_all(1000000, seq) # Collect into groups of size 1000 >>> futures = client.map(f_many, chunks) >>> len(futures) # Compute on larger pieces of your data at once 1000 Adjust between Threads and Processes ------------------------------------ By default a single ``Worker`` runs many computations in parallel using as many threads as your compute node has cores. When using pure Python functions this may not be optimal and you may instead want to run several separate worker processes on each node, each using one thread. When configuring your cluster you may want to use the options to the ``dask-worker`` executable as follows:: $ dask-worker ip:port --nprocs 8 --nthreads 1 Note that if you're primarily using NumPy, Pandas, SciPy, Scikit Learn, Numba, or other C/Fortran/LLVM/Cython-accelerated libraries then this is not an issue for you. Your code is likely optimal for use with multi-threading. Don't go distributed -------------------- Consider the dask_ and concurrent.futures_ modules, which have similar APIs to distributed but operate on a single machine. It may be that your problem performs well enough on a laptop or large workstation. Consider accelerating your code through other means than parallelism. Better algorithms, data structures, storage formats, or just a little bit of C/Fortran/Numba code might be enough to give you the 10x speed boost that you're looking for. Parallelism and distributed computing are expensive ways to accelerate your application. .. _dask: http://dask.pydata.org/en/latest/ .. _concurrent.futures: https://docs.python.org/3/library/concurrent.futures.html distributed-1.14.3/docs/source/examples-overview.rst000066400000000000000000000001131301211143400225470ustar00rootroot00000000000000Examples ======== .. toctree:: :maxdepth: 1 examples/word-count distributed-1.14.3/docs/source/examples/000077500000000000000000000000001301211143400201565ustar00rootroot00000000000000distributed-1.14.3/docs/source/examples/word-count.rst000066400000000000000000000213051301211143400230120ustar00rootroot00000000000000Word count in HDFS ================== Setup ----- In this example, we'll use ``distributed`` with the ``hdfs3`` library to count the number of words in text files (Enron email dataset, 6.4 GB) stored in HDFS. Copy the text data from Amazon S3 into HDFS on the cluster: .. code-block:: bash $ hadoop distcp s3n://AWS_SECRET_ID:AWS_SECRET_KEY@blaze-data/enron-email hdfs:///tmp/enron where ``AWS_SECRET_ID`` and ``AWS_SECRET_KEY`` are valid AWS credentials. Start the ``distributed`` scheduler and workers on the cluster. Code example ------------ Import ``distributed``, ``hdfs3``, and other standard libraries used in this example: .. code-block:: python >>> import hdfs3 >>> from collections import defaultdict, Counter >>> from distributed import Client, progress Initialize a connection to HDFS, replacing ``NAMENODE_HOSTNAME`` and ``NAMENODE_PORT`` with the hostname and port (default: 8020) of the HDFS namenode. .. code-block:: python >>> hdfs = hdfs3.HDFileSystem('NAMENODE_HOSTNAME', port=NAMENODE_PORT) Initialize a connection to the ``distributed`` client, replacing ``SCHEDULER_IP`` and ``SCHEDULER_PORT`` with the IP address and port of the ``distributed`` scheduler. .. code-block:: python >>> client = Client('SCHEDULER_IP:SCHEDULER_PORT') Generate a list of filenames from the text data in HDFS: .. code-block:: python >>> filenames = hdfs.glob('/tmp/enron/*/*') >>> print(filenames[:5]) ['/tmp/enron/edrm-enron-v2_nemec-g_xml.zip/merged.txt', '/tmp/enron/edrm-enron-v2_ring-r_xml.zip/merged.txt', '/tmp/enron/edrm-enron-v2_bailey-s_xml.zip/merged.txt', '/tmp/enron/edrm-enron-v2_fischer-m_xml.zip/merged.txt', '/tmp/enron/edrm-enron-v2_geaccone-t_xml.zip/merged.txt'] Print the first 1024 bytes of the first text file: .. code-block:: python >>> print(hdfs.head(filenames[0])) b'Date: Wed, 29 Nov 2000 09:33:00 -0800 (PST)\r\nFrom: Xochitl-Alexis Velasc o\r\nTo: Mark Knippa, Mike D Smith, Gerald Nemec, Dave S Laipple, Bo Barnwel l\r\nCc: Melissa Jones, Iris Waser, Pat Radford, Bonnie Shumaker\r\nSubject: Finalize ECS/EES Master Agreement\r\nX-SDOC: 161476\r\nX-ZLID: zl-edrm-enro n-v2-nemec-g-2802.eml\r\n\r\nPlease plan to attend a meeting to finalize the ECS/EES Master Agreement \r\ntomorrow 11/30/00 at 1:30 pm CST.\r\n\r\nI wi ll email everyone tomorrow with location.\r\n\r\nDave-I will also email you the call in number tomorrow.\r\n\r\nThanks\r\nXochitl\r\n\r\n***********\r\n EDRM Enron Email Data Set has been produced in EML, PST and NSF format by ZL Technologies, Inc. This Data Set is licensed under a Creative Commons Attri bution 3.0 United States License . To provide attribution, please cite to "ZL Technologies, Inc. (http: //www.zlti.com)."\r\n***********\r\nDate: Wed, 29 Nov 2000 09:40:00 -0800 (P ST)\r\nFrom: Jill T Zivley\r\nTo: Robert Cook, Robert Crockett, John Handley , Shawna' Create a function to count words in each file: .. code-block:: python >>> def count_words(fn): ... word_counts = defaultdict(int) ... with hdfs.open(fn) as f: ... for line in f: ... for word in line.split(): ... word_counts[word] += 1 ... return word_counts Before we process all of the text files using the distributed workers, let's test our function locally by counting the number of words in the first text file: .. code-block:: python >>> counts = count_words(filenames[0]) >>> print(sorted(counts.items(), key=lambda k_v: k_v[1], reverse=True)[:10]) [(b'the', 144873), (b'of', 98122), (b'to', 97202), (b'and', 90575), (b'or', 60305), (b'in', 53869), (b'a', 43300), (b'any', 31632), (b'by', 31515), (b'is', 30055)] We can perform the same operation of counting the words in the first text file, except we will use ``client.submit`` to execute the computation on a ``distributed`` worker: .. code-block:: python >>> future = client.submit(count_words, filenames[0]) >>> counts = future.result() >>> print(sorted(counts.items(), key=lambda k_v: k_v[1], reverse=True)[:10]) [(b'the', 144873), (b'of', 98122), (b'to', 97202), (b'and', 90575), (b'or', 60305), (b'in', 53869), (b'a', 43300), (b'any', 31632), (b'by', 31515), (b'is', 30055)] We are ready to count the number of words in all of the text files using ``distributed`` workers. Note that the ``map`` operation is non-blocking, and you can continue to work in the Python shell/notebook while the computations are running. .. code-block:: python >>> futures = client.map(count_words, filenames) We can check the status of some ``futures`` while all of the text files are being processed: .. code-block:: python >>> len(futures) 161 >>> futures[:5] [, , , , ] >>> progress(futures) [########################################] | 100% Completed | 3min 0.2s When the ``futures`` finish reading in all of the text files and counting words, the results will exist on each worker. This operation required about 3 minutes to run on a cluster with three worker machines, each with 4 cores and 16 GB RAM. Note that because the previous computation is bound by the GIL in Python, we can speed it up by starting the ``distributed`` workers with the ``--nprocs 4`` option. To sum the word counts for all of the text files, we need to gather some information from the ``distributed`` workers. To reduce the amount of data that we gather from the workers, we can define a function that only returns the top 10,000 words from each text file. .. code-block:: python >>> def top_items(d): ... items = sorted(d.items(), key=lambda kv: kv[1], reverse=True)[:10000] ... return dict(items) We can then ``map`` the futures from the previous step to this culling function. This is a convenient way to construct a pipeline of computations using futures: .. code-block:: python >>> futures2 = client.map(top_items, futures) We can ``gather`` the resulting culled word count data for each text file to the local process: .. code-block:: python >>> results = client.gather(iter(futures2)) To sum the word counts for all of the text files, we can iterate over the results in ``futures2`` and update a local dictionary that contains all of the word counts. .. code-block:: python >>> all_counts = Counter() >>> for result in results: ... all_counts.update(result) Finally, we print the total number of words in the results and the words with the highest frequency from all of the text files: .. code-block:: python >>> print(len(all_counts)) 8797842 >>> print(sorted(all_counts.items(), key=lambda k_v: k_v[1], reverse=True)[:10]) [(b'0', 67218380), (b'the', 19586868), (b'-', 14123768), (b'to', 11893464), (b'N/A', 11814665), (b'of', 11724827), (b'and', 10253753), (b'in', 6684937), (b'a', 5470371), (b'or', 5227805)] The complete Python script for this example is shown below: .. code-block:: python # word-count.py import hdfs3 from collections import defaultdict, Counter from distributed import Client, progress hdfs = hdfs3.HDFileSystem('NAMENODE_HOSTNAME', port=NAMENODE_PORT) client = Client('SCHEDULER_IP:SCHEDULER:PORT') filenames = hdfs.glob('/tmp/enron/*/*') print(filenames[:5]) print(hdfs.head(filenames[0])) def count_words(fn): word_counts = defaultdict(int) with hdfs.open(fn) as f: for line in f: for word in line.split(): word_counts[word] += 1 return word_counts counts = count_words(filenames[0]) print(sorted(counts.items(), key=lambda k_v: k_v[1], reverse=True)[:10]) future = client.submit(count_words, filenames[0]) counts = future.result() print(sorted(counts.items(), key=lambda k_v: k_v[1], reverse=True)[:10]) futures = client.map(count_words, filenames) len(futures) futures[:5] progress(futures) def top_items(d): items = sorted(d.items(), key=lambda kv: kv[1], reverse=True)[:10000] return dict(items) futures2 = client.map(top_items, futures) results = client.gather(iter(futures2)) all_counts = Counter() for result in results: all_counts.update(result) print(len(all_counts)) print(sorted(all_counts.items(), key=lambda k_v: k_v[1], reverse=True)[:10]) distributed-1.14.3/docs/source/faq.rst000066400000000000000000000035121301211143400176420ustar00rootroot00000000000000Frequently Asked Questions ========================== More questions can be found on StackOverflow at http://stackoverflow.com/search?tab=votes&q=dask%20distributed How do I use external modules? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Use ``client.upload_file``. For more detail, see the `API docs`_ and a StackOverflow question `"Can I use functions imported from .py files in Dask/Distributed?"`__ This function supports both standalone file and setuptools's ``.egg`` files for larger modules. __ http://stackoverflow.com/questions/39295200/can-i-use-functions-imported-from-py-files-in-dask-distributed .. _API docs: https://distributed.readthedocs.io/en/latest/api.html#distributed.executor.Executor.upload_file Too many open file descriptors? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Your operating system imposes a limit to how many open files or open network connections any user can have at once. Depending on the scale of your cluster the ``dask-scheduler`` may run into this limit. By default most Linux distributions set this limit at 1024 open files/connections and OS-X at 128 or 256. Each worker adds a few open connections to a running scheduler (somewhere between one and ten, depending on how contentious things get.) If you are on a managed cluster you can usually ask whoever manages your cluster to increase this limit. If you have root access and know what you are doing you can change the limits on Linux by editing ``/etc/security/limits.conf``. Instructions are here under the heading "User Level FD Limits": http://www.cyberciti.biz/faq/linux-increase-the-maximum-number-of-open-files/ Error when running dask-worker about ``OMP_NUM_THREADS`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ For more problems with OMP_NUM_THREADS, see http://stackoverflow.com/questions/39422092/error-with-omp-num-threads-when-using-dask-distributed distributed-1.14.3/docs/source/foundations.rst000066400000000000000000000106411301211143400214250ustar00rootroot00000000000000Foundations =========== You should read through the :doc:`quickstart ` before reading this document. .. _quickstart: quickstart.html Distributed computing is hard for two reasons: 1. Consistent coordination of distributed systems requires sophistication 2. Concurrent network programming is tricky and error prone The foundations of ``dask.distributed`` provide abstractions to hide some complexity of concurrent network programming (#2). These abstractions ease the construction of sophisticated parallel systems (#1) in a safer environment. However, as with all layered abstractions, ours has flaws. Critical feedback is welcome. Concurrency with Tornado Coroutines ----------------------------------- Worker and Scheduler nodes operate concurrently. They serve several overlapping requests and perform several overlapping computations at the same time without blocking. There are several approaches for concurrent programming, we've chosen to use Tornado for the following reasons: 1. Developing and debugging is more comfortable without threads 2. `Tornado's documentation`_ is excellent 3. Stackoverflow coverage is excellent 4. Performance is satisfactory .. _`Tornado's documentation`: https://tornado.readthedocs.io/en/latest/coroutine.html Communication with Tornado Streams (raw sockets) ------------------------------------------------ Workers, the Scheduler, and clients communicate with each other over the network. They use *raw sockets* as mediated by tornado streams. We separate messages by a sentinel value. .. autofunction:: distributed.core.read .. autofunction:: distributed.core.write Servers ------- Worker and Scheduler nodes serve requests over TCP. Both Worker and Scheduler objects inherit from a ``Server`` class. This Server class thinly wraps ``tornado.tcpserver.TCPServer``. These servers expect requests of a particular form. .. autoclass:: distributed.core.Server RPC --- To interact with remote servers we typically use ``rpc`` objects. .. autoclass:: distributed.core.rpc Example ------- Here is a small example using distributed.core to create and interact with a custom server. Server Side ~~~~~~~~~~~ .. code-block:: python from tornado import gen from tornado.ioloop import IOLoop from distributed.core import write, Server def add(stream, x=None, y=None): # simple handler, just a function return x + y @gen.coroutine def stream_data(stream, interval=1): # complex handler, multiple responses data = 0 while True: yield gen.sleep(interval) data += 1 yield write(stream, data) s = Server({'add': add, 'stream': stream_data}) s.listen(8888) IOLoop.current().start() Client Side ~~~~~~~~~~~ .. code-block:: python from tornado import gen from tornado.ioloop import IOLoop from distributed.core import connect, read, write @gen.coroutine def f(): stream = yield connect('127.0.0.1', 8888) yield write(stream, {'op': 'add', 'x': 1, 'y': 2}) result = yield read(stream) print(result) >>> IOLoop().run_sync(f) 3 @gen.coroutine def g(): stream = yield connect('127.0.0.1', 8888) yield write(stream, {'op': 'stream', 'interval': 1}) while True: result = yield read(stream) print(result) >>> IOLoop().run_sync(g) 1 2 3 ... Client Side with rpc ~~~~~~~~~~~~~~~~~~~~ RPC provides a more pythonic interface. It also provides other benefits, such as using multiple streams in concurrent cases. Most distributed code uses rpc. The exception is when we need to perform multiple reads or writes, as with the stream data case above. .. code-block:: python from tornado import gen from tornado.ioloop import IOLoop from distributed.core import rpc @gen.coroutine def f(): # stream = yield connect('127.0.0.1', 8888) # yield write(stream, {'op': 'add', 'x': 1, 'y': 2}) # result = yield read(stream) r = rpc(ip='127.0.0.1', 8888) result = yield r.add(x=1, y=2) print(result) >>> IOLoop().run_sync(f) 3 Everything is a Server ---------------------- Workers, Scheduler, and Nanny objects all inherit from Server. Each maintains separate state and serves separate functions but all communicate in the way shown above. They talk to each other by opening connections, writing messages that trigger remote functions, and then collect the results with read. distributed-1.14.3/docs/source/images/000077500000000000000000000000001301211143400176055ustar00rootroot00000000000000distributed-1.14.3/docs/source/images/network.png000066400000000000000000004737421301211143400220250ustar00rootroot00000000000000PNG  IHDR ó bKGD IDATxw癎edMdUHv*) "BB${{gy~|ιxonޯsK_=fX,={V׭[}z ʛ7orR۽{vؑm<ʗ/ *$wܓJЮ]tn`)cߟ-}7cg*f|r9 4IуPwssS*U4n ϝ۩^CRﶲX,Z}XV͛ʚ)Cg>yRo衣O;t$,\XsƎUjQQ2~xC+[6MP$ܿ_65vjU55jBB+6mR 5k"{II"<"BϚkv۶8{vRI~CNJbfZySu\͛2a+Wo, i,pRB ׏>蘘$1L^*Z@w<ٖY;cW{z:8}ܝAGNа0{zu= 4Tm?P/vIĖ-.[kl7sm[i|}^I]to;[}SuۯJ\)^I`)̂11:Զb6]P|Jn'׈ɓմkWEDF&y ж} TZ57N/׍ݻuvF-:U]ZOTPy`Ѻ~_#>o˂%ܼsG :vԛzիհwb3gԢGɑCuӦsuzz]ߵKSƍSW^u11jޭn޹c QzS'>%̝a'VbII2H>,X*/~Ja,-Ypw5J`:{~j5SGVUR9Cѝ<~\+7ou'$Ȩ(}0tϟ`/I-9ܿ_\kUP!,\X% R 6;{v`jM{'](2Ob? ЉjӻLϾ|NՓ+|:r:dM…6M]ySwӵa+RRٺRڽ K_ڿU5 I Kx_Uz4 ЦsB'GmW/Yp6Mj̒1& b&۶i1j۴ #i2ʕ5Ouz:5SZ~}$zfVn6Piuz 5K7wֺٳ5_?լTe˪xOsء?7n4q0 *T_ ԦsUX1)SK;ӧkݬYiHըXQ驜YM&ڵp>ژXu_qqqپR R'eϜYҥ33jŢk-Y'Ξ5}mE<٣rMg%ymý#G`޼6 1cj/:&FMu^WOm4uTѱի5gO Htmx5]5s,ĖﻙͻwN0,p'U*}ޭ.n٢)_}*5y`y-_}U7N/Tõh*{tDa ß^^s4UPZwww [ul1`zuTJ@@=|XM&-vA4K,l۷OMtQ7о#Gg2\jX]έOt1ܛhBMŠ5~ ̚U6M3gM6tZmj)}7ry5EQQuTի5wou1d&v'oΜjk{n95&}hc}/n7rY,{tFJ@A77T]8],X7gN;Ux,\J5K͛kզMZ? h},mPw|S^V´b&jikL~Od,~ĔRw#11jѽn~m9mrgϞ(=uJ![͙SŞޡ׫g~q9b̙빲eӇ:\C>p_WKJ@By0̒kM,[T}`i)kOnڶoVo٢EWkMsi8 {׎x_SD ^H4H>+:&j=TP2^ƍ6ɚ)C?5Lߺt}{Ȑ|G;>s#M./5(`~Mk#"2R\iף];v|U`/?g{,  .Vt/իZա\QxjӲC%iI²ԊMfV93/*}/mZȒEEW ԰V-Ȓũ/mܵK;ui8sF<@(TH/..Zp.qw3 9Իkh:p.\TȡREfJzF N/O][Rw}էiRS]eׁ T*ooV-_^ &$=|ᚎo;wI)uႆMh׸vm7{i~x|z\\5nvϫf;TY_v5z/`) ˕KiÇV{΄k S___qt \x²~FNS޻Љuru4HUSnTT)}d޽h[K׮}}ڴk6ڥWkLԀ㝐UjL'Ϟ Ӧs =}Z_}-_n8266V~v߯Is/MumZtt-I}_m7Se׀=TR%G*U7gNx@A7nȓǩJ6\?r򤢢T}[Mk$=<.hY2fԴ#Lddb2wMpM{޲pT"ʓ#C5_[We~=9tHKtR R1XO)eZb2YqZyC`bфV5͏?5,..NׯWfz4z/Ԥ9s 7W)''2*JMt1 ɓUP!jEĨ*٠f/^l63 4T_ԭkEb_Ruڷצ]а0mܹSuڷ) zU+I=|7kP^I2l5\ёSmOC|Rǝ8sFs-3 L>2=;а0j:qpDճצ] k5 ͫ3u^bH2HH% ^]ÇڸsM]z%)QsL W=} = uGf-ZNqVάY!]DmXyn=7775J*T֭wUeK}NMsvt쨟.t+}#ހŽU$ݢ6̙k֨ONL,f(#. ts眪mO=HgqgϖbZ/?6id}7 {FĘ_~vI08:&FO4+icL… םG{LK\$G:YzaX(dBe&}UxkN4?>̹KTU$<.1K{cSW^έwUeKvI_1zo_rp+}߷ۧn^s :p 1G@C,mϔ`3fS=xtm[M-Sd̘d}<.%GDhuoVIT@ýa'*,<ܮzBCV^zaCzDZӧMCN6cvcǜkt/3xZ,sj]~5VpaAƍjv!ڿkʕ(+[LJ+ׯȩSZi#" {u.xsTbEU(YR TlXvHN;ǎiΝzti42xx9pSl];S&MT-S&{@ϟעիfVÉj:jU߼};sn𚧅O9t&K*Xzu-\pzn-!bo//n8iٰmׅ q0k ;kkݸ2OTӽy:Upg.^b1D Hlٔ9CɊ{zuaZP{>wR7х+W *,eR͌j?W\ i܌ '^~]oE'O|LiP*oo PܹUZ5}ҥ4gc׹eK Ck}wڲg^_6ګSkF[U>^sm [8~̸;{ɓLHOf>z/Mk1\O`7,KFf-[p֋/*/kZ_OV{SSj}7`@+VhVi|}5ĎFgtp=]@Ry{;U;K vfT}WJg<ʕ(anotUZoZ2K*eZ-]Naa6a6K/)W_WM'|Ol*7Ӯ]wG\/^F~-~RCL׫I\ٳ>i^,YkBP^*SF2wn޹csO\$ϫ ,_.uOTv>Ӥ9s zo L}ظpnժj`lZyVURSҕ*5jvȑZu'(0P>t̙ުakתNQneϜ٩mu u?_p͛Np%.T /\bף]j-_\*Q$5GBkڵ6a,}H&>tן|ڴ hՌL`5I|zW_1cn_=Z1ߏ>>Z5cmSl3kw,FoHg͔I 4=u""#]~r'$Dmz6=+[D"v.^jﾫ eʨYn;}ء/+}EFE)}ulݪ&v픫reM;שڴQM7ڥbӧ[,2EMv5Tۣ];ulu'$p=sN׎/jv.@r!X BJ6uM5nݽu۶Y?yn] el:C2TLPpz sgxxdž{g/^Ԃ+쮙ݶMͺu3@R2Z-[ -թR%~:q"QkhZ+6mrTZ4i>9 ̇O԰aN=o//}7p䓈͂ծ_8;1 8`) U GToƯSG˙SeךcMY>•+v-KzUΉsww{mܿ_\qvRstа0 h̙vOַ5sxdE3kV8HZijAWU_km~떢cb |z 0A[RW_3%{Pwýk䧟&ʹO WNL ן|C8CjԶIwFTt+&OH.K\E`W_]֛Hկo8mΒ% h̓#f!N;mk b:qZѣӾTsk?+}`uoܾm:Q2GPۢlGOv.OƍժQ#;!!1yjm,*Ȼ`Ay(%F ک;w&l |.85dqޜ9AgūWKFj& IDATzL߼Y!+IٷOWִ#Ԩvm1ǺP~Y=¡k%iɚ5W~^23l cwF&~4/K[Z6lh#K֯۾]_&Pz!M >!iӪT"{t~b;pjmY̚Uk~YY2ftnKlx+[6S9U맑#պqD?'{,.٨Q9e^Ơ 1CҧwOI Ԝcus-]ewYOp_ڴ˜!feifS[ ԖyYRݪU5s(]ڶM{V@֭SCmYfAWIFI`)@"^x-2\/X9C08ɓ 6JҾÇ eΐA3=$'xjݺseg$Xb?rt{a$?ɝ=K[jjӤJ- )<<<6rgϮ+W;jIw~8tb…]߰5pXý>>Z:e^,]%g=Q*T0VbA~kG+gPEM Sntlj`$9S6oNg\޿tmI֏H= /Y>^7nVnX __r٪Q#ٺj}Ƃ*UպYZ YyF̂Qъ{"cr۴k^j\KL1}-#I}HZY2fTVԥU+զݻ ̂5e>3[ 'N4 RJeۿ6e!xFbQ/4 Տs)V̦3iӑ#5|$|$Hʓ#u^yzxqԨvm?k&C? unyM?n޹czfl1>~մٽ<^5\ONp5nirʆ6_GDhZAjۤaf+tziݸ}u3gVm:ˬmf2ᡡ~2ŊMކS޸mh˼yʗ+]\NP]p;\wjМcմN?&^qéK&نGDh֭VnnnNAjը͜i=8Ặ(/ޡ'TBk<~p`޼H,)w{ի xPQVr2 ۪I0i϶ʙ5OCmu Ŋ9U 1,H$5+Uz\\ܿ3~5ҧ+5j|Z7nlI3-{fMܥK6׉ϾÇ ׋(I|ɒ/?{v`*6c?s"϶cO3 6mjz&c߾vwZtT"'Ϟu٤he`+~9G? SDdSD 9\3:&FLew.@b!X HҤQR m&E7zpۦM C`,M,ussS^ŋpoIp>Tѣ{Kt~R*?6͝Y>yR-zPLlMȨ(mر>lT>} Cɞ׽CuZeתQ# CB*UKtM랿|p$eɘpAhoHtL✮/hdž mۦа0ýZ&$'nժ+6m$haPf>x*m:}𠎞>'OV% Vm>'(0tLu2ӉtuTq~R˟;VϜ &A6_T+}`mx{ij'..N)zKys活nhX>2pT"6bi0 +/ipsjׁҥ3Lw;$>cR#K1sݻڲgC5͂E7D =Թs:sf<6Mr%J8w@ 'JR>EvԑSH\\&ΞmWxq1y'5ܟ8{̝kS*wݼs숊V>/KȓG?ڣMӥk׬jJP]ժe7É`+ ׫+gzY cOm$j6=ՕJ*B\zaZ`}W$REoO[wlZ5j$o//Y3%^Ys:l)4i-hwG~;WG~meo1/L>mӦa0fS=wYUMZp?>>v׎3 #igMtE]zUKL&p$*ZT>RYGD+q/6\LڪkZHgO3G޵ZwwwW&M ,Hdf͟ JNL0 Tڵ֯ݼ ҩBɒvw|poΝ9e58Op/OSR$Mр= "{%e˪Lb{1 d= .\ Q-TE }0d.\m=޻ Qz=85nnn9j/-ҍ۷֋(nm8TYO=SJeJj=..N]GÆ)"2j=]@Z5jdz]*ooӠoUhXݽH;tZý6Nv}eK10.bTܥK ׮9sx/`)@"khb#׫@{E _[zu;WD;vڴ{ ?ls#Nvt~ݺá>S={/]vMM޽M:|Lp)Uѣ*+;}ݫ{j*հ;HQZ~0>hVlڤo>L?kj܌r8EFEq.nnnyhU|59ujɚ57szkgZ~jjޠ~X:T VЇ:QZ6rcbc}>-\JQQ\\|B߷ڻv}herssKO\\^zŋ:rꔎ:G+~5S^"iný???z* k{go:z|!!ѷ->֩{N'Ϟ[f~+*R$:i?xYI.?+~ݻk…v?okL~? [ IzY3.Z4^)RD*Uj:sz N[c}]*ooճa[|Z5j6i…֭ӲuTX1U)WN2gOTzㆎ:7;.[̚:bS=Di}}hdzUGÆ֋/H5~:T_.?j&ʗ,RE*}`Ȩ(ݽwO]Ӊ3gA= $Ȓ5_>x0@|Að0w".I/~kծnd ͫW/P6m >zT-]V[Tr+[6Y,]zUۃ5ql0#?|]3FT7PLlCTQ#կVM kRbŔ1(Hi|}u'$D^պ5or2I*?aG\IÇN ?-X+VQzpa[qNmعӴng%x#O R&Iy K;6oc7ח) H^Jp-& W{>lWtZ:e:b?wn}7p:k}hr UM]vZmnfso]]r˗|͹K`UB)OP`wnm&L=SbE N;޽ՠzu__T) c6pb dž v"I*Uuޠzu}{2aP^X/oԚ7nMG{<5 xzx믻Y^ժ浺u]roZ>uI~>(VfjVޞC4|Ҥxϗ++ގ:ʘ>}dk6^c3g?_rITnܾ:3zG;ޠ^ԯ[7S'}էǞ3kV9}˙Ӧ'>wouaZjB|S‰UdI{Ve,Hi}} z6QCY3er陝[4P_vYSk n@uVW_%KTpauM Le:a\e~=e}VTxSY3e;ejilԵuk>]g6lgｧ@q'$ĥ,3_򉦍N|f =i׮ZO6@Ҩvm_̮|b?w39i\թRk{qt$yZz^-\~+U޸aW|y?bM=ڵSz4bdM?_>ڗ+WVUlYx7uu<ٳ;U7dJ^>TڋTA|xk֠^ԹeK:U3]u"Y_]%'U*Mxe2YlgĄ  z]f\ߧMxD_u۷+QpAmp-ׯw^7&bIs~]\K֬I'թs4o fHfL,ăPٺU[Gue]uKaSj(cʟ;9/YRʗW>ȩSط5+EϚGPXx2[۶Har9 8Q,]HׅGD(<"BCBtYwssW}.]!4,LͻwWXx$ v6M1B_zիطlڇ_}JLbHiܓ)66VGVѺu5}xCX,ʑ%St0@NūWU}{2ũnϡ]m IDATC*^BݻރHX [Dd^Unz^xkZ.4RU < J*<9rcV{g.^ԇ_} Kت{ qoXlz-`N~PiP` е>TALGoJ)V]B`n`mpn (Zmm6I˹}MJۜw1 #J'lmmB8k۵ o4rUDDDDDDDdXJDDDDDDDDDD:z8 z˔ ݇cM^(=)Y07NcLVĉHLJ2rUDDDDDDDdlL]QQ)WZ5n+WFYJ1qq ƣgpM:sU˖իXuƘ͞ BDѐnݰqqӀ\7AeDDDDDDDdNXJDDDDDDDDDDZ[s'"lY`˂hXA[[8:8 Kƌ(_$~jIɈ֫Q3f@Ri>lrf˦׺DْɓQ JwW_N|Y8+S@DDDDDDDDDDiJ²͛qw77ݺUsS 2\gq] cxϞ:Id J-]jceF KH+OeP0>}(,\8P՘~ 7"2b<2gkxQR""""""""""5PȞ5+tl:v= Ǐ5ԨT"K KJNE\~dpg^ETtK1BLnU˕3u9iƵ{[Q꘾t06i@GZBCm9pG{1VR%ԭZU\Ax5d[ؠ@<( _c'$FԪT 6Z6x;9:bd^0ow1J[q.6Y0/M]IT*<Z ʕ(al(Mz۷CVnݴ>~p׮z5##ݹS%Q`etT|_J lWydΜ߽3g4 ˇƵk˖ٳT 3Й3i9in<0wsqAool9p@҈( ac)i%2:Z6Fjr6(J^:@PȖ+"*JҢظ8$&% Ǜ>>M - S@DDDDDDDDDA(WHgGWL]F%j괶6J wo(57KD%w1?<| 5AeDDDDDDDd*l,%""""""""""w{1ִN̖-kYVVs#~RinN몕oWgwS*n?Ȩ0ݡN116Y6u0ֶIPbc%{|-6Wٲ2u4iDXQL>DDDDDDDv u8qB@uMDȕ-ʖ(O|{x 2"""""""26Aj5޼\{'UPX%CDRK, """"""""""r-aRkR˷o"""""""256Yz0Vre#VB~U)[V8A/j+""""""""Sac)5ҊF(A5B4 VȧOx1/W.dvw7rEDWҥ1o"""""""J_lL]QZlU yЪaC`og'ORe; cRMpD;m??1b5DDDDDDDd*l,%"""""""""ґZFDT$&%T0Y+!Jʖ(!L&""""""􇍥DDDDDDDDDDd^J4˕ˈysƤ^DDEHGza2Y G{899"˅9:#|@|||>݇Xh=J.}G/#V+K洆mDDDDDDDDDDdކcl,%_ޜ91# %œKj*yyBFhwoɶ6i/,4/1b$4y̷a̟RDÿk4/a04'lP, şsWAR|v}eGkTIBB"vlݏ*ea3#]EG-cș,ʗh~=`gܺዠ7sbc!l}>DmѺYɦRD~˷@?7)JL45+z:?|MK*U)/K洊Kt`ou@˰ѱ!J[F""yjZS&CdDT:~,N=gAW3WTGع ƏŪ֨h(=HLLBñ1s: ^+&&ƉcޕpϘ5@eoT*X4o n^="gӴ#]ܺcHJJ>˒1zHHH4x-Z-vغ{*W-'C򋉉ECqY:y&kp*It:ϟB ZśZYNv6vG||BHm~i{hT~;ԩ>}4to,MmZ4K˺бu?Y,&7肀)'K洎DDDDDDDDDDdbJpZ)Q*zmI&M@Ԯ-k|~leD$[ Nh>C6z3f@)b(W4+[>~-{#..^?~—ϩ3lrI ~i/y8Uoo!-ZפxB@QDax-lٳ:bbbӺ=5]g*?|F-چkNlL]&7;X pX*$"2''統FPj(W )Y3 * 1~/1n 746~) %%v#HOJJFFH6G٢гo~ř|z\pCƭ,_2E_"11Ikkk4h\ A+wvd䎄|wo?cw'_7to5+\y3'/OQmcoee'gG"::..N:q55]򘒥bhU9rf*cGbmSI##at llRnFMm7֍{$ɛ/M <ŇOv֮چ+n }!cK.]oxK,I4ٳ(U8H^DDmP0vתY&{ EV1;J. /FD\8{wƁ=T*!w\ E;r5wOz,/a3\π.$88|Y|7_.t7Ӈw8N?A3ծ/K"9DDDDDDDDDDD&`og'%&&"ˑ ڒzM8i?Hggg_'1|L_ɦR{{;4nVo[Xr ODFӖl&o|y y٫{M0])lS՘0fͅR(Xq>2~x.TlYZI3'/![4&szIQT=wL8?#3k lٵ~MQH~:T [6O5Ƭm K}G/?9/ <,BK,Z1oص v vvcƎdk%^szR""""""""""2KN7L$"Hmw/$"2FM$ɛ/.< OA[..Nݿ3<>ʲ&Z]B 0 ƴrFa3]lٳfV 1z@aέ8sNk@9ҩM n˟ȖE`ъ\1kd#)Nʥғ_=eS4 f1eʕLz.\'&B%O,Z1C{w꼮\Ƶ;M`ZC50"ԩFY5'l,%""""""""""("!ݽkNISʳ.:ϲ%Rk89{%Ú[PX}q޼<>QꊋdžčWCFFEZ{QTQa|ҍzݿ3ڂkY[[c_%~A9d#) X7Kӫߏw"ϟ#?1 y SkPZ?}kj٢HLLsttU3upWTV^U:'7K􄍥DDDDDDDDDDdaeRoD?mN,%"s#JSP>9Yf2RUDdNEAo KVL>qacv=vk`Ȉ^Sx Ԭ]YҾRf%(W,y e#.c¸=6KV̈́[Wt aEѵgaջ/Rr鼮BoF >u^Wx Kl&$&$&"!1'kV<);8c۞وUYX'/{!nw{!= @dDK$#<, .E6<Ԫ2SXn.6h/.5i^>[ HNV3/j co9?4 @?CVk8a h˨u?y!UQ0x}};}?|Db轶!,S@DDDDDDDDD$$&"W.~\leٲF!!(W<Նt,̖͈i'9YsVH,[HW8w2nŋgg..ț//5*^pϘ!j#?ͼRU۰1]fdt/߽ #gqe\p_>kXOwWMB^C\5s, oG|\<ȃ0~pvv2zѨi](^qBVm`ܤ(SIj{X{ eQFEac˷ea(ɬj* F{<݇ؾy0>iH4i^ψeߓ-><>Ν,n[7Y |^Hх%^szR""""""""""2[%G߼7ҋaa˓#lLF>&kVl2a0lmmqZaIphIm;4%S-A5 7ޠܽ-{˻5dHDGunLL^۽W-n^d,l*]b F= )pY?IKg@,-Zk' FE^Ͽ"#$ʭ+_|:˒K_R$LT Hcg:+`Fc6Gٲg5x}]{xf?|EOdȌO5OpQ,-#+S@DDDDDDDDDD$R xM6K4~-w!v0=|:{"Eǻ/vW#:pK]BmUK9KJgO0fda3~4vh[(W4f/mL 9nSWHo1>q ԑVcHP|R*V.pт''$'+ϟB2 =µ+wpMRJU^J6-{Vd˞c5uK8|R 9IWg:#o\^ƌGb؀_mX:jL=V4I`3HhҼiE[.SW.ݒkkkb ]wzTx!<y7A<X||,9qM8zNy`tKޑ>^FDȟ K`IpusA;Zv6Lk-T8kx*H˖=lyRZ #6Z5G4*cW5b%{CQ {DO1V\aLE [.SHყXwo&iLz >GG6rVE͊-1~Ll߼=BRDtt,޿ wy. 3ՄWp%]ldɚIWT1y!jT#gV5r&qOuMӉT%xd"[.gg'8 _d%91uDDDDDDDDDd^\ݏ3W Y"ӧbCXD2f`H^2n[[-40b%DD zN2+wv#U߻%re˗k_~=;Kh,-Z5Dp\߆*^j6j* ~PTȖ;z#?i6ō./9-V<w=5] !iVmX\\bל^̚6i[|S*8q:y<+ElT4/[d'8:V#yKx8nܿ1?wnTm1DFDIfUN+lƊ/{꽶'j9xsj&:v6>~R̻mSxm Rd8i1-^ލ1qO0XZlmo7:~Am9dKm?aL4]Y^E-jԥ5Ǝq+ׯrfP94jZW >˺Ux/cmm-:qŸvlݏop4/YDuJ%ΝMv Ɉ3o^. '`[fO+&&]}c9ڳAkUZoDcpy&qdĊsѢUCCRRdIߒ>(k.K1%""""""""`5u &P(P OSAZS2e\)GF"+=Dt/4%u`c)8RsxDEFk٢}pquFr%75O6INˆY.y̥ 7Pٳ)|b͊-?|FRR2sıghtl3wسdow 9 jT4J=MЛ:- ʖ/Zua:R< +->V1x?]3hU4+Rj4$YsX5WXJDDDDDDDDd*.ڵq5$'K7qG&M'GSBZBͱtӦb zw*[>'GG$Xd ;;;}~L\d<^wnnx^7S Bw77yKniȓUʖ4""Y[K7ȹ\^-ժ[..NP7flEn{g#WKJ p놯J#OӲΙ+޺VM!2"Jx=pT%=CXhxU䉆M~Ú,[3h<.2" _NJ%%GJ%z4Fw~O_18ykk8z &N!#z-0IZGzd|?;-zn\^ލp9<\x>|)=w_ FRE_DG`ܨ8v,6l[d߲fB?$HR-rkNXJDDDDDDDDDDiB…ѠF zؽǏqU˕%Wڵ93^ɲ^Z;{vT.SƠ5߽J)kk)7//_ZYp|: {h99:KE۔sXREe,&B,({I%S8uq'vCO%5˕j߹ Ŧkcͦ.hh_~wsep.ԯѱYz;NL3kwѪI7 *V.A288hׅWѨv:ySy.bk+ݚ "$ֳ5%^szR""""""""""Ҋ:m]cc)XW-W%QdIֳ$RWL3d@S5Gzn.<'ood̠y[L""s1dC'.SHդ,9Y)9M`7{]/FȢW7A9ཤ悿m0ql{bz66Ȗi^p)6ClUnر5fxɸ!@)bd Yc<66>i#?x7.l*3 XNfpwC֍Ѳuc,\> 'kpmLw uEDx𘚵+cRՒħXImԣ=,"?rN뵢"ѹ1aъ+lѲuc gg^|?Ӻ>dJU Req,+N,%"""""""t?~g7Q%D0ae˰kR#WEzRVVVRy>ap Q|yrJ~ƚG`bS ammm{Idggn|Э"#q=<~ o"K8aeeWgd̘GREQBio s$O=C].Zj3AP!!d%9XZ-+6QҫCz5޼.q}J2Ae3a_cfŊȚ)+o%$&b[P`ȑFHwE?|Hh'Am_lIi%Jw74jZmP0ްqCKʳhV3|Sr8..NdS ..Aw5p< E\[]c,_Xju`1Ȗ?"qQc,:*wn=@j\f69'ɛ3swT*lٰ[oڢl"#ݴtԲK͒Y2J#"$?-B#"Q*~?*Y OmfW*ȖU2m0 g}wpsO;di..NڳfO[1.jxL-%J>-'Xdˣ KI: |)S\ӧc~зv;&l͕-fc䊈 Ҥ] W3 lL Zi-K*Uɒk qa8rYe7k|r2Wz' KL+""2\EPha<y؈IF$}oԚZCl\Sr2wexL8p|#%+^H{ly$fɚI/]ӷ -(lrUT9T\)XZCTDeƝmלҬlY`ر{#Vd) /Zc 4\5BF\<(5uL-,[B;}+I_y&}]7z0^`^teʉ'4^zW< cYSh,,Й]FakkY\##d˕Y2x[NЅ%^szR""""""""""Jt&uhGFb"`oKphh"jrlZ4뾾xXLl#WDD$=;VV,^oĊĪ׬$ݽ퇏>QId΢cѱu?DEF 8e$ll 5~A_q>j^H^%ַo܃RiK70w]$&&p5WJ7-w=.+Q9se78,6QP(y|ʖ]fLL5DY{;;xկs"M޼g0uYzw/J.VVVز`r~!"JKd͔)㑟*װ䶰kVl1rE郋0Fb!x/\ơS~ASks=NJ%CO ֫.. X-RWo[ai ":ֺ]Sa,EA[ëT*hԔʝ,6Q5S&x1lSZ5;>M(X[A(SK!KC0OXYЄѸx QZ1v8HLOw_O̞Crr҅$fyƞLdB?e8w Xv,fO[,YʳfU<=lYPFEa\cVMb c^ի&MN7ɛSPz(W_pksކ㝺HK􆍥DDDDDDDDHxw+d&:z'a#.>>cǰjv/L_ ֮Cp}ʚҾZ*aqX8ilق|MDƏmнm[L>%"Gl3~1_E.Mеga,2" #nj҇BE '}jjy/SVl2,‘Q_#< k7\  ثyZ5V ;;[Œx cǏÓG^Kq06)&q?U۷/ U)`CG(z@rҰHbbN/׬]E%yIkL_cc)$+p+SB]bzWu~ >WR}LZQ4 +YϞ5v,MCQ ykˆi+>!Μ)SPys8.-ZÐ!0q"̚I `̙}!s-Ȓwuixb21%>;wk cJ}Ar{'%66&(LqF!q57bY IDATk֦u;q놯0Hz.cJzD֜<=:I+LlQA%%'㶟k\u+c kOF:\ARSAXa4h˖ARi]_JЪ_?,ݴ ~P)Ru) z,Nm̙;~ԩXu+#>!._F= K=DDDDD[[lضHؼBЬ~g +BuP&q{~4J4DllX;;[L2Rr ?Ĩ!#11I,AUK67l5Woݻ!X< jPl\jw@ʭpϙ2g ]J;G'KCYs CGØaۿT*zv =n RnCsϘ-Z;pJxg֫`]F >}&kahۢd31`՚<5v)qzKN NJJc Oi7zݘ54S@DDDDDDDd)45^ycO\'Oc* WA:uR<_ Tu [R{;;+YٳdAL\~wwɓh=`[Q@-PݡVsDFGB:_oF!CO;:8К5(S\6BB 6"""""2Lv>:ŭ~ulݸNT-|sHJJFBB""g~M;D֍y m]q_b؀_/PzT^%JEƌn''G$&&!::_-wkШ4W/jԴ.\\×_jn AZh˸yj5ը(9ֺR:m^]TU41i'Mȃ6/@d7?$Zk]#ꧏ_{a̛\r·-V kkkRBims9y ]{vw#w Jwn=fPd !|+5GXn.Z6&|:~ ՅOVhؤ. [[[|kWnc]x$7ȡrrZzݘ546gb+w@R8Qܵktv7o~5*V?~ E#k*usqQУ];|ΙW|1{J80qس|FJ'OP`AOOtkk^ *t)7/L?twxTe{ @$4 %$*Ųb]_t %-ބ 5e?\YBf$L$纼d9Ϲ'gIOWow z~Y閯ϷdTW}A}GYmÕibZW<[ߺs/KieŚ3>nsa8sЁv$X\]]u93zUZ;$+쀍u9)Cq([},ǜgf UWIrE74lay1Xx+0?:{|zڱh3HS+BόWo+ԩ^M__MZݴ!ך#Ǐ߃*ߐ0L7տW/z .Q~W!Z}Z6s/'%k;l?}ujiCw3!7klscb˽,)08B,|LJ%h{򮲿_:QRcϟ*>noJ'M~Z?6TټݏUv^^.C~:}*M>>Z'e5Ai^G&?Q{!ZcZYX @Q:o\F99ZwÊضgOHK~Zh+<{V_/^\hUK>H7yIzsDYh,/?_㛞(ORK?!a]kBե&=Ͳ8ڷKjql1*_M1v"$X*}nY۶VrWmڹTzt.,+I3xxz _͹eڶouc<==24i*oRWfu=1ϊ?y>*+iWt~{;ɤs zyjky8$RF0_=ºܲ~ Ku%ӊ5ߨ_RϛaԷ{"!ܰ9 m~|ٲ7_? @mCB,;cȑغ[Z?y<= oJjϝSQ:}r_|Q3lΞ5_o6WXGs?q_@ff̙vj{iƜ9FG؁&M~Z;jizGt/yAu/ꃏ~`]Hԅn~`%ZaV.ݶ%D;|rssy̘Fȧ w~jּԃZJHܦiＬ-a.*F_...YW?k}{KټP5|4{۪[.sJ7}Oj_GzWUVR!~gX;WK=#>osm @5WW @1.NY78ڱukwqƍEV CNn~ٺ޽[n]G[=ޒ WsXfVn٢%˞Ξ?>#GSc J㕑)Ij&}2u"u38]u*1Qc{*uپ}~P~5jw>&M~Z~H .d6oiZ~5uW Ӏn&4L7jP_~@7ƕ:dRP5TM/2⸛} 5zj{Vݷ>Ҍw?Qҥo2鎶~?jh_北4oTo~eϷ}6ԉ$\]]-$O/OxmU`P=5iPڷB2oI9zX! C}>K;zD ք?/ýܭ糿_kån=7j}b4K:oPEFik̯uJzsqq!znxp߷:H)rlRJtT7ˋĮ`rŌ_hlƒꝗ^xiU5F WZIuB\?Lgќ/2n24q*vݫQ,z[ڹdWf@ 1iaǗ+G)q%]J֖Mqڶ%^N:xA̖{jTz fM24Xmچip2˗x6ۦ[u1.%+#=C99R>VjTM2$X!]VRNN֮ڨ7iw.\,L&V5|UwlwU.vYzv/׮ڨfj]VAg@=GU }qx-Z}֮ڨvǕxUԩf-c]94GnmߺK GO3JM̌LͿ\WpԶ]uQNm=#_GiV^JKM~;vϟ":T-iQyxx_ V``6mG}TK,QVVih,0Ȉ =!ܰо} ?a[%5&jpնX6ϱTbbJ-(4zygӛ}d@*u?^M;wZkŸD[ĨQkYYzm '_^zS թUbEXԎrsso>͝;W ĉR{j޼yrwwcZ83WWW}%n.^ffbcc5qDiӦoo>1-0C7TAA$@.XPa mԼI^ͦ{*VF>S~~Bc Sb.46f0լ^X;̘ܣ_OIK=O< &+gd}VZc5klѾ#G,ֺwꤰr_Qx{c[o胯қ'r8n٢'O*11ۃGц :/ I=.\+W\ JHHƍzaÆNCZ(K T? @}w׊u뮍ʸnaj[k=TM~B?x\]Io }5ܹjڰM}bE1WWW=3n\HU}|hL1t.%'_ux͞2깿<~ks͞2E (ذ!??_knՉ3gw]ste`ѶQ@VkxI*'FҴYV6{$xt;xPUezUViժUez t):uJ7n_ZjޢZQ͌X `G(XqölC\k,t{-Qb7nԹ9^kOVJsiE{u5kܸDIZ0c"G.'[ǎc1itDՕ1G,?S~9 IDAT=-Res]ZGS-aJܰjSp-9˃GkPxxl d*IVUCQAA1 et87KXwuu՘aÊ=ϮB:t2oy>4TBC->}GE бc:s"GV4G4k\J2v<>b?Z40Mo>xh*S}'ȑrsu-2=?[8.vm 0ԙ´oKt^~I}6.ofO"5 ۧ3gkAAEwqqQ.]4#l߳G'ΜXS=׍Ψ8$-Xܞqʽ|mرC/_8xBiq2238,}%ӎ ojyftH }jPҮ]5kRrGW]h.\o,ѬsqY~~@=uoFSO1 $I&IV(}զE fm^ۣxMvN~^NZy6:qR\QFVLժV [Wڬڴh-[ڵ~eVku_Cӓ'[_LNDuEXNE+7lPrjёXa6gu@oUxdwEFot/խotD/cd-v󺐔$ooխSG͛4QLR]|\Coz\Dn:rZLY];t(uQ6\]ԼIVܭ(22Rҧ hyڹc󕙕))J8uJ;-TwuuUhp:~lN:uRHӦrM&J5'/ϊtI,XKf`@چhEjGR_RҔ4efe@ def)%%M))iJMI@X bY]*Fzrhc}zu!).sk{>?_44bϵa?ub]HzPZu])\\JH^,6J+WVҖMeӦzzXedfj͖-Z/Z/:vܠqKQa sT1: R;y})m/kR~v{6a@z-6ъ呏}v%~~\Z+7lPJZ ["Ppu6krJQlڴsZΝѶ<=ZҦ ^Ү'777*#+K._֩D;CÇ;0Mj]zUtq?~\v^ǎkӮ^zI۷o뼞jҤ4i=:jvJRpbfYWCkM=f$M_2LJeݱ'ڴh$ΫmHZdaYU-I*((Й3gta%%%iӦMi\\OEIRVV bRɤ:( @ZR``IMw`6m(::ZQQQ޽ LeRsZҭV*J}c+,,Laaa2ezjT߾}l2UZ(2335d߿_<"""ԧO RSә;i4ŗe͖ccK[ SLLbbb͛7+..NaaaFGC9c֭x+s!Sڵ2m4,zXP^6>޶777(x}bt!=3ffJR2Vk999Llo[ "l}dXefg;0 lmwo>T$4m%)%)*6^3n +v@EBc))lE))JR2^rq|[?[X * KNv͚rwsZ?wӔ0sr4+U5V,*u1$INAe6S,7ܩ~@'$X>wNM6tp"t*1j~@Kf%--ZXm,]uk_nkjcDKZc:8 @p1:rG۶Vkԡcn8zY&q^e֠^=:8@٠4{Z3͚:s”@ 5KjPX pCCmnբEڹOF9$kVkVhh,8'FZv\[g>4ԁIʖ1c8#F?˗-֏<ѣhLխS'^}v^EqP+…"_MiWʷjU^j,ڱ-V,8jUh_jmw뎡C5oRb{y22fbiuҼh6Ϫe@X[pp"Ê}iҥZucN%&jSO3tOt"wW@ծUKgjZ}Voڤ2ܵC-XbmΝ ҥ̮v:k]ڵKNE_6LϝyÇa;JgK%iڵzƹ,]j}\OgtbԦ֩c9>^ɩN]ͻՑb_Z~]/ZRhk:WVAA=NPh,P)=F@9Ro]Z94h11z4uL-g2Jg ][/]*RaJo_b\LsRVk14JR#Q#͘rRI=o-5:OϗlXȑe ps?NUw{K[Wdit PXrZZ[ت^B޵ZںkW[-U>>V99,|e PX R}jָ_dtkĊ#Gւջ3F6mX>Yn:GL/Pڕ+!Cl6 2/\X{`լ^RFPݺ [pr.^T©SkU||T1O5i@-\gɪU}چ؜aJxQ0L ][Vԙ3֟3, 4`G߭XaֶeK5\\\ܣI,_1C󸺺Ҍm׉3g,GDuN8+ I׿f϶Zuvؘ5mb+s~]˙d5+M&I|i'p,Kznk ,fxf]YL5K V-h,84ΧjڬYVmZPvz-x}wv^ewYM-\]?V iug:xhٹ_{q{װEo=kN+W~ﯿLGTpFNxnFђRRtI;yR%IգsgjLjޤjVjUZ*QիJIK_Uܾ}yzwuQ T&ϥ_Ϟw@" IIzrd}[erdZ|Z`@g4={%^yKZjWo-dK/iźuJIK+RлF PfׯN;G'MZdURŁbtRciY뫟>L:kksY?6iN=[**٬1<))wGFH0͛k…ԦM_K1BzXKIKGs:$GEe.۶bNZ1yRٱwo]WS-^Mund24_R`@@jU_?7eHj^[.Thppq}9}h,8-}'k@ުY-g2ԵC:q{Qy!iԩUKs{Onwީ˗#"غv.Y?\\~W= NxnF֯<2:m{pBNT©S gf*+;[..T __qPZ6mNmڨGQ%IީM(=5fL&ё*O `KNd2eӦjٴQcPaY=:w6:a\RH܌t]zU999FSUVUftDc)t!;w(.^Ǐ^zjٲ\]]\fvMS)\bbv-lt@c)TN8dc|N:et @c)Th.lXJ(Æ +\t-YDK,$\k2e5SPl7[Jh֚KwAYsjܹ>]*:,Laajht<J#++KYYYJJJ*RT:uTNիWOAAAR)h,J"';[6Ӫ˵e:a#NjFjql$)48Xѽz):,L;u *l>}ZOV||$f͚ TÆ ըQ#yyyR$223]ftIrE)iiJNMUfVd۷O&IuU&MԤIrR$jԨCl6k]ZBbWhM3:"4ԭcGE)gO 1:$f%&&*11Q6mRժUլY35kLF\*!ɤ6W{^Yںu~YV,CT:[VDnM:v7[{W^U||6m(44TFG04VX˓h޵S;wPff Z* Lz[FG0j֬)|qAAj׮}O>V$a̘1 믿nرcf̘!WcĈ2d6mj Ѻukn刍۷Z=BJJ t邐BdLDDDDDD_,,%""""""hڴ)NSϟlj'p ǣ3f̀;N8pofuQ aÆO>2pDDDDdl,,,Ю];kiii}v/Ÿr bbb0h 0c"jHdTh޼S!"""jPXXJDDDDDDZgeeU]}ȓ'Oѣ1c6lΥU;vE0qD9}"""";QTT7oݻjoa޽h׮ SSٳ8{0p3!!!NIR: "ؼaODDDDDDs...LNꁲ2h|NQڢO>ڵ+ܹ7nTwAZZgggfJD D"ADD0Tբ| Snnn7oBS!=`a)镥%o4D~ Ҳp]ffia!B!^^h S#z.W ͅ5\77huҲ2\}5i'NDTT""" R|ٳ/2<==%w&&BoiDEE K DDDDDDDDz2?QXT~ݺiSB!9=wbbp/>VV03SPa|d$[C6pmťx4>ѡ8};ZW 7!JQRZt{EEض¹Νô%Kn$УcGrs矈OJOB%23)%#CGLtJ]σalxŘCp$'': GA,4R""""""""=C2fMbǺuнyYYsz߶5k0qr JmmɢE>nmmenFGcڵrTD|͝v/6ZD"_أh\簾}JACOcֲex_O?k++ʍH1?ooM`4}^z)K"""""""HN_(5.9SWi)ϑ%ssq9q&, K֬[Ȍ+6b|ZP+FJJ;u T8/_P {klCSǼuXT;u˗irsтxoRJs'n I''L;Vy q10$@Ν;ǏGnnns]2 8G""""""2:,,%"""""""҃ 5{ǎȤf21P߶lQZT/8|LuL\Yu[⛏?v ݏeb݃ZO.Ģ>U;tKfR8gɬYpqvcPRZ^!!XZ@YYZbĉN0.<9;;^ӧZ{. """""""j&/äef }c*bؼɓѩm[\&((*JٲgmhW/YP'J&v!29Ǝűmpnn|hQV277Lj#Я_?E޽Ǐ}BDDDDDDdH,,%"""""""bmTjl;!{{Tpoc^zI&VZVW)7Uz1x4k~]<۵݈JnDDD0cذa05qqq8u}_HDDDDDD  K`0ykc[o!#;@Yz2~~V{qC~sSUzV)*M̜)7"""jX1~xX<&&.\CVDDDDDrdff"-- %%%NHoj$Y 77&?xN\c`Ih u h-E_\5JubO qnDDD𸺺bҤIؿ?>}Z㵷n݂wﮧ숈TH$ ,--뼦X,s޽{ΆH$%\]]>}_~077^eeep.]h䠤ְC-Ю];:tc]!deeٳ|2J4jԩz䘝={ܹsHLLz & )S_#dXXJDDDDDDD'_._d?^&^ZV?? 0eh4rpk~9yyr1-4ZˣY3B՜Ǐ5MU CFvV>-,DDDcؿ?k~UXZZ"88XOڵk?^hNZ5###' 55Un HHHѣGꊥKjSRR~{AAAxaa! HlٲXhjp]")) 7nݻwt OkoÇQQQ!7LDGG#,, Xp!u͇T#T~}D"k222o>۷cǎ{gzPeR%DDDDDDDD ffjBX)ynl,Z={bʕH[~<9i mmeb 6]QT\~øq"""C=dEDDDT{iW_}9s(,*U$33 .G}$96.]ȑ#eEbܹؼystIgP<|=z4FUVСC~:QTTXj<::'Nľ}*P|wrwYDKbÆ JJwA̘1ֻ蚾#0HB!>xm;ޛ=Jn^RZowrxkx :[Ԕ7!"""c„ pT_DǏz""""mJؾ};>sn:O jxÇj*^cھ};ϟT~-ݫ|m(SPP/CŦMX~n1{ld*y%llli&> OV{bXYLjXj84kk.＃=ǎ;p˱d>z-W4'Fb]"H&X[ci.`}DDDDފJEE>_VVVz̐+V ""B;w;v쐋wo&affrܻw;v.sÇѮ];;V>%%%رc`Z q==zTa!_~~E\:e.]>OT@ @~~~׉D";(~N@5kF&Y\\/ￗuVk׮*>"9sӧOG۶majjxݻo`䧈#c)YZX`ر!D<1/{`} Vri4Q[=/%#C{Ek`SRt/lmm1n8X[[x]AA?^/:ыܹs0a֋JKK'ŧO]tX~=}]96l|Lo߾صk~L:=z@PPz_8x BCCD|u3PfxO>Xx16mڄ'Nʕ+y&ZڻvBs/vѣJCŞ={0h 9RTg]K_'NM6K.9ڴi>+W~ǎHJJIs&.o3Eg**?_.*߽+kZhR._:ߗH;v,,,,j.99W\SVDDDԐ%$$ 44 .Tzߐ'۷/y4o`2|ڵ #F}k*0a]SuEg]\\b ?6l)SУGZ7ٳGc'''|5v733ի룷oFrrJΖu9cǎɓebb[nz~2s.SC'@DDDDDDDDʾ}0bR22RRO(vUmU@__D?xPC|RZyzSbwW;'uy5o7d]3hdd$<<>3gT8fnn9sk׮x7O~~\[pp2d~}tܾ}:tP:NR:teYYY:),q13""ëtE4u)NմUXUX?Zj@DGG{{rcرcG֭La)Z K55Oj}@g,D+Wte˗kbUUcH$SNϻt\l*4hLai՚,,5s`a)RTڸ\^^}Pf7C[#꘸#/\IJ/'ebB걣YΝ$UT`th(~{tjV=HL7ă0Z[cۚ50|Vi(*1[j>'NĘ1c0eʔ:tk׮H$QFXn <}T.qrrRim K[p0|4D?xL<)(@H lml-[MVC:^+HNOG_!'/073C#S۶*O>տBzV?\47:@ si u%0apL>%%uⓒrX.hӪ|abA={"z,JJKѻsgi5k;ƍJEPPZḧflŮuĪnkm]b YPf;W_ŪUdbk֬'ڶm[|DbV R)V\8x@@O^ųU>||X]vظqJ&MBYY6l )X}a߾}CF)3yꫯ OD.faa:*Maa)y]x4kfʹf# Wkꛍ5zWHtwSQ=CRϺp&O\7U ^ZPPTZ]l*JQPPBTHJ!QQQQ]VՉXպUU1(--EQQn`駟T+,,̙31qD+hٲ5ŕ+WpIdeeɬiYnN:%_|aźR&Oyd̟?N9bڟYŋUW {u]v !!A5j KKKأG^Gu'ba"wt*jcLlڴ C>}:-""""""""""""޽{QQQx!"""ҽ۷O.+F^^,Y AbŊ:puueffJeee!##C&^пX||("亖Xud> hѢE]#2870prrB^^k5knjy?3/^,ZoCDDDDDDDDDDD"jٲ%4i7n[eY?YYYiȈ[2Ɗ+jڷoիWC h%šK.*NNNr1E߯6nX}^G;"&&F&vM[HKK\Kd,Vbرغu1c1z^ZZ¢R"""""""""""gϞ8|hCDTTVqv "ҩgrzK5N<+W_ܼ9HZn-ux xO31X,Vy*l`mm:w={VRT&*ыH_1>3_ 88K.sFDDDDDDDDDDDDy{{MX,FTT3""""Ҿ˖-3*QQQLMMpttJ/???4iD&vEƹsb:uŞe*>"$$vvv2guUD*СCrqR}R%\\\fffￏ/)QTJܹ#9^\0333tecccHO***}vD2cxwU^Ζ͘1ZyѸquG]`21H]v4͛sL,((Ha<<s~JDDDDDDDDDDDD'''O>EBB3""]211A>} iD":u *6lD"3쌍7Ug0sLd3'Oƛo:٫.Ǝ P(۶mkWPP?X.+3x`E"-[ZJXf 222d]tQ fΜ O?˗/+իWzzQ'D`jH=8{[nUVz̈tih޼9 yxxiD" 11}6.\7o~-7oO@iؔ|3g4=gggL4 ?cu.}9&&wcdd$V\uv튐%G"C`a) M6|2"''M4sfD+7tDDۻw/֮]J 4+WZ{ݸqRT&v1;vLuPxnn-ⁿ/_1ch.̛7ΝCjjjuХK111pܙYZZ?)|^ƍtRZJ&7xmڴANФIGƍ+ܫg0w\DDDÇձR|ذaڴi333$$$-V\Y/󄈅DDDDDDDDDDDDD/SSSi7oTz͝;w0p@=fEDDDP(Te˖X`^ӄ76f(VVV1sLS\\joaa7ǧֽƎtlݺU&.Jhrի/_tgPOǏŋJ皛㫯{{h"eDDDDDDDDDDDDDd:vXcb=fDDDDe4h[߿_Rxsf͚MLLj54k ۶mS;?OOOl۶ ;wVyo> 5R7MBbÆ  J#,, *-[x'԰c)i(((@ii)***PYY KKKX[[ N޲D"$$$DDDH$BJJ [n!55NM)+++vvvpqq;:L+{5~!ϟHT7<ݱuV8q_lƍkaڴi077W{Ç Çk(ꊗ^z 'OFӦMk]_[g憝;wb޽عs'^qaƌUiyB ^ KSSS/"66=ӧOQZZ +++ANн{wA #EɞqqqGdd$␕T===ѺukԨ_U%%%{.޽(Źsp9|' ~#Fh%We6oތk4w˖-߿3"""""""""""" OOOۣ@xb8 IDATb"acc̈6Fvv\hzz]7UiFI&: 9::W^zZ_WgbyB N Kϟ?#**Nddd믿Faa!mۦ쌇T*Ŏ;tZ[<+"pa>|ǏƍUj\ٳgi~M_#GbԨQغu+7n}󑟟b-gCDDDDDDDDDDD[JI$ƢcǎzΊ]GP^^JJJ H&5LZ-,… i&J51h e,ȑ#qEm- 8pV4d <qqqZ]7==Dxx8tյ->>C իWdt^XpqqAvvlnrDDDD"AVV\hUhnnA255;Zh!>>VVVɇbСz6k(۷/u~aa!F۷oM'{h˃0g۷Щ@\\ K0Ʈ  e GT} s}TTvm\saT*ŴiT**uvvK/ M4 "::ΝÇ kF#G 'OѢE //III8~8vڅsrr0}t:uJk7ڷo???xzzvvv.jCjj*"""pQ\ruߏSNaȐ!Z˕ K JǣGzΊ8 !!Aahff&D"A򲵵Q777$/"":nܸ|駘]iq駟~RM >;틌 \zwEPP#"""""""""""?@BBÇFu G5kS:|#""һ:rJ+?>[7dK,QZjiip2e DqUtM|}]Eba)Qx{{8CbQ"""qaÇsN{fQ?n޼ӧO+5!?݋X KW^A(?{,V\笈[[[4mYYY SSSQ^^ =gFDڐ$ue4 yN=RX< S,ʮDDi?JcVVVؾ};JqFc[Ƃ 4^Νp__צɓ'+-,s6DDDDDDDD8BЉbtܩ qyIIIsVDTWgϞٳg Ac̘1 1t*J]QKKK*,uqqAҕr: "' UOQaT*EXXɓ'UV'UD"@ 0u DDDDDDDDDDDDf x{Ai@R}zW~1$ADdiv]G]]]ѩS'рۛ."""2ZKquuչkS[[k ,vuuVd2AAA1b}޽#-A "25T۷o]Q@۳(D`S̥K9hX%u7"""˥h~~>bIrssCpphv&@DDDGgg0էm̑iӦut{yS,Cyy9;]GuG|iiX6mڨej5CrrBDuݻwQ__oĮDDDdVqttD6mp=III3fL FII K ΫzI$ܹs]G@ ^tEY0+KII1r5^\CC+!"""""""""""2OpssCeeb#WDD22E%"""kSw8sҹg϶ kӻwoҸ8_2rEE\UU+!"""""""""""2_*%%%d1rUDDIOOGii)|HRacc???i۶mk I`СC*s0pf-u)x" } )R,DF|y{{#;;[\CC*++n䪈4"ccc:(,3f lll ɔoذ=z?+Lo˗/7~aU\VV !"""""""""""2c>>>j狋,%"ꪷ`3BBBGࠗ}NR___DGGرcJgѣGKj Xl0l0䄚%%%FHsKuW """"1cO/9sgφD"u YXXRנ3<@[lʕ+XsrrBT3KDDDDDDDDDDDtʹb#VBDmb7n;W⯿ڵk駟СCѱcGJH`>l޼O_~#"""""""""""2 ꂥPVVfJ<7nDqmǔ᭷1vX 23!PZZ4$%%ԩS̞=3gT[СCU޽Hk}Ø0a޽%夤4{ȑ#˗/7{lMM Q 8<(**2>qo7nzuuu8<yGDDDDDDDDDDDdj...*FH3"HANNN>pGSi``џlJDDDd,K0sL;vL_)ɜtQq?f WyZ}R"""""""""""*nnn y8Xڜr\~ׯ_W W: &"""xz @@@9 6`ɒ%K{۷c֬Y(//W{)ǏcÆ XlܹPPP???kzSNaʔ)z\̇;ΉD"HRv#"mTXnDDDDklll0w\<3ذa~G\tI"##1c :Ts&N+WwݻQ__p[!Uch5k6oތOL&yHL8f)Tl2vvT rG}Tuq5\_5ݻ666̙31k, """""""""""2Gnnn*R)D"ݍXQ*ա`{v8m`uރpB,\8z(q dee 􄇇ڵk0!"" {cǎ(((/ʕ+GEEJu xjK(bܹ;w.p #55YYYw WWWݺuCǧE,_\?_ Æ ða #,YwyƑ#Gpedee555<==~!** h׮A"""""""boo:A>Ƞaccc2yK.]: 77hNNrssQZZR\vM鼺n:tqDDDdq ,}PHHBBB /c&~~~7o͛g}[]v1cf̘aR,P(DLL bbbL] Y ???䘺 """V%XBlADf)쩧MAӇ^_]SPӐ8ìDDDd,%"""""""}a"##M]sqq-R #WDD@DDœn7n@mmAjM{>}ZivJDDDR"""""""_~HHH0u)DDDDBdd$:ud2@ *++Ϋ'"~~~JC]t"""֍R""""""" 2r `ꒈЧOtnnn*FȼcӺ:v;%"""`HN:sYgggsUUUFȲ4J;.]`ucP,T*+""~~~S:-// >|8>lRLѦ.ު 5L&kuTTMX Y"$%%DDDDDDDDDDDDD& qq_NN˕ BܼyhF@ر#M]iHn7oDUa>hFDDM6.DZZZSZKZ ;;oG׮]ѵkWSAt;}6RAҵiNjLR"""""""""""""#5u jk0T`ANDDDDDDDDDDDDDFϛuLTj=:wlȺYbS{{{)Y$K`ذa(,,ٳgMyggg۫wppP;+WWWxuS@:xa \`2Ν\=Lsjkkqq:u III(**Buu5:W^:t( BAA>ӧO#;;%%%MM<XXX۷ѣjs 6 ӧOGhhQ벆,DDDDDDDDDDDDDyv,%""2DDD ""BaN]ӌ (]PJWX-³>ۢ5ϝ;_}\UU233{.F>駟}vTTT(WVVyyy8w{t-u &ׯ#77yyy@mm-8ooo*++ڵk{n+#??IIIشiv7| 痤L&O?N}ݻwsNܹSN;c𐯵\dY,%""""""""""""j%`)v*qmiNN!^ˎ;_bƍ7|'OҥKakkyN²eP\\U}iiix饗0|̟?_s :%%%߰o>dee(|}}[P*N‡~RAYYYljb _Dtt4UUU8y$֯_7o?{+2gFϞ=agg ر)իXv-^ud 1Y6KZKHGbŊZ=\555׿0>{lrHH/Xr9Wȑ#t/ggg|XhQXtt4͛p ׯw}'7'?wm C[~9PAa AN'''Meܾ}[nlꫯ$7cԨQxqС9Lo Z| 3f{'7֣G,_z,7yfL<AAAzүc| " YLdggkxbkP"?YϞ=())oȐYVV-[}4ȑ#??8p-[{L>X`JJJc#G%g@/6mRży0a8;;~?\!T {{{|駸y\ҫW"'';vԹe~wʍEEE}ͧNLlٲi?,:&`)Q+`)9|0>l2LСC2e oRҥKi&?~\#}~̸ /^ѹo&bccQWW4c…jf>}:zq .Į]4;w} c_Jy̟?-Z?55UiBgҥKϝ;`kS=3,X*++bccxbtua)1Y "JHHH)"jZ\t?$"""]7iR""Jq Sa22UDǎS4 B >])####5zxx`ܸqrcyyyzܴ '0^PP:2kTZJa> ZC0{ג$&&"''Gnlڵk纹'khhZ%\d,%2X~=&Ndeej"pttDFF|||0qD_o6uYDDDԊ~ DDDDDQ,H$F4!Q__o2LJ$"ԥK+6 Ν#B}jA)5,U@@XyyA2k TKu>Ǐ>%%% cNNN=Xܢv)#Gj|ѣ5Zyu3uDUcX۷/^D"@ ƍ>@RӧOGUU^xݻQQQ0abbbl+w""""M~DDDDDSwoDDDdM$Û ,P%Z#""Ba,55Uy8p p5xО={:N<'O>ʂwk+--Unq]xСC5>?<<r͋/B&=-c_d},2XZ^^F 2go>)<53ϘBvs H$8{,Ξ=?~~~7nbbb0vXߐ.w4ÎD}jQ͒H$$$2uD㑗'jhzmUWW+iYj׮ܽH}y1^F2 6lsttkڷo0v)ht~bb˜d2RRRܔvTrժ*lc>v6iiiHLLlO"`޽.]+11ppp@mm#R.MpMv2lJMشi6m@"&&111۷?MCDDDdxKDDDD}o߾ի֏) """Ѐk׮!>>/_FVVV~L%=zĉ1zhDGG% DDDdL=z􀍍\ٳJjKpT,^o?Ķm1k,ojR'O DHHn޼ӧO#%%Dptt7ڶmAaذaJϪ`;vLkiZZ̙> ]tQ8… XdXhh(~inν{|||^G940kx JOOkB*&MիѾ}{ClIdTŞ={pQؚdjO>$hfҐ۷ {ƠAg׶*HDJTVTB!Gښ .U(**+WVuxKnrr2bŊOO8&L@``%""""2KȘѽ{wGs!66=hBV]]}$2ؾƔR1P'|7nP{ݻaggz ̓F{`ѢEXbxFFf̘)S`֬YM?b cj*7(//WzeOR1똌GҌ <%%%Mc صk.\ .6ȔZڕT'''̟?zDrrrBeeN >>0 jz鋞={A `7ScH$H~W._+rTV 돐Pt CXԹÔyyq!ğ=iξr ..AǠN}a@`t KmE8~ş+ yee B^"1p0Ϥh4̌t?v !3#E^W77tаn0pGwG3'p#=99Y@}F[_?0CFcАGn:uWiHG~~D"TWWɽ ݺG^/ ;NkЀkp>.ģ ^.D"ݻ{nDDDDdj"""2iӦ?={6{T*Uhekkd2,]rݻwٳ[x0HKCClق}a͚5nOWZЀ;wbΝ Ν;'wn`` ˦>UWW+988hP(hmCk-1>\AOf}oKwJNEE$2lfc脞5m}a[k} nY0aԧF/ud2w÷8u7 Buu"=-En?LлO?k44`c"I-bܺ[73{o`qX·8DU[S_7CRUǦ&X|KB;D 6xǏ7kV*\23$pptDL$y% Yz%}ҹxK?{ujbsypqdnU^mǎęQ\U**ʑxIWo)ЎOGݻwp o?w%Ud͔1tJDDDcaƍn?3f`ҤI )++CZZΜ9@nMG==lժUsww@=́_ĺuЫW/Ι3gwKPa0$p?n:i̓]QTVF ZZc2Kߢy"S1TWR֬/x|+tOD2ݝA,/Z_MUT#.vb (FN:~)ɉ-Z .}5DJ|>_"̩:/p~>z BL6e _b1ݍ}{wOנdakpAί?v`;0pP+RƑ6m}*ՀX,3'}/xKRfJDDDDJ]Ԑ%""~)̙#X,ƦMi&8::2 v}l۶mZz5~1@>>H 6 }App0aoo "..{D"i:|v '''CYY?ӔTr/lϭ:&kT,C$=Xy'"S(..Ɓ8 "IKMFZj2Vm}0jx#ǚޖM.u~Q-:K~FeR]D?:Zokj r9;~-++ C\8w46oݍȾ&oxmt{ş:^W@}())Ʊ#p`,cf#&&cǎw|;ooJjkkUiio Ɨ.]!C $CcwO r[lAxx㽽1d 2'O믿.۷o_~/v_De˖5}@Qn0|p\xW^ 6:t_ 22R/Sc4shmCim1^B())QyLv%VR).]X۷.\P^Qam݈m[7B A5& W$xٌ{2q$uݠNiDğ9Ǫ0{x(ī>u<Ɲ[pvx| 23w0e(0, ߯[}]aa2KMR)]Ccq>\]"S)((ƍqA!&&111.;Vyy9Z /JQ__" TWW?T۷믿Vxl*С;us=֯_0h"<:kuUs###|r,ZHnܹsC.Yrcfo28p[lQxKKKK/~@Ϟ=5W cK5ZZuLƣ`)L:?)S{K"!!! HLLDfB" )ڴE۶h߾|ڴ7_{@b"Q%Oon=zDM[_kxW-Z b"U궭* r]S~.l++ z%)ɼ%2׮]/|}}ѡC3͹$ჱHK.ٳKa%""""DD=8-=io{c߇ӧ{n۶ HS} IDATVR;w.͛zN1&V__Pcƌ @bb`iII V^-76tPPlmm1m4 Giii\aa!6n܈^zIzPZ<==uIS:&{ΝG}]v۷nj3d{K"B5 Fʕ+QTTcǎaϞ=uǂ=b4=2E\o69ww|7xə sptrBGlr" SM߯[O0m3j8=z섦_gf?bt u_Mʁc\]СC7qc B|᯿TקW._T9w[.V?|ѯ@|gXc?,8zb1iz,x7&_c`L >d%2nM?v!J.\z^(P //ot wB`mB!GQ(WS!.vR[@@&NѣGcĈhӆDDDDd}R""t89{!55UnZ+V(?3xW[T%pssS*Xڽ{w۷믿uŋkOXX>S]0|ܳ+iν{Ƃt)I#:&{իRDm۶x'OB"ʕ+8tكgZ'{́ uQē31xh4(B"Q%9r~æxt}"T9⋯QCaG7ڰU*Ue̸ǰ{LJ9s8n"4Vk'%]7k?A>}Ｗ KU{O~njxi%+>| ^=|ZDYuo jΧM[<>I<>'뷭p,RSLLDlmm1x0idd$w"{ƈm'nݪ3ѣ{aBB.]O<~5ZeKKKNUQ։S٣`޽rGΝ5k8p Ν;4VTTLC4ùu(;GSɘ ,%tC~ﲛ)1 j0z鋈pw>gDɮ)ɉ*GS>RTWWqt!XwӪs{V._oC՚gXR:E\i^]m-V:kcc|h?m Cz#)Nki$())ozѣGcɒ%-R+6Gǎ5^NYAAޕo>F!, Kmll, 7//UUUZ=;==]:thq}uL`)fzQ8qqqqxΟqrr/GyDi3gbNؠk׮߿?zЮ},2@h) w=znb_TņJ0 JO;UKj~ikb;q#Mo8c/WTr9T ;{As]rС1tpHRd_ŵkpE͔dZzkkkcܸqƈ#F;YVu1鈊x e}VSHy]>߾}{2Qo߾rRtkΝ;rcp똌R"- =Gܢn5555kك#Gj":^xA7}}}1`{G\UU4~֘6D霣֬`7Ӧ?2X|]_F\hX7u{ |I}ߟ}Z۷mR9צ/]V2,K2"22xP^^WٳHHH@iik~LM]v%%""""k.XR9.0USUhXN2ÇW5^c8sX_C(FKҭ[7˗/k<1etV^[ʍ>|X`#Gs1Bo5uL`)Q =ʹ'Oo>4{~uu5LcF输'Obʔ){쉘?=z@bb+$e}T%\7b%.|gЩs#Vd8FW9W\|Ӧmbk,nqwgD[ȿvj(*,Tο[puukQ?r]1|pDDD%ВaÆqDDDDd=R"2';ǥ T+&ˑ#G+O2=_|ܘzӒm۶rDO< kѣG00#5,ڶmjpss{a,^nnd?,uL`Amm-xppphfrJdeeaÇQUU ?qqq4%;y$&LH7775;6iәK]p+;vlی3f˗. YϽh:!S|QXt>=-58~JB!NѢ: }pQERyL֩ͅǧNoQ_r]u.YYzsHDR8L7o.:u[oZCaa؜9sZ&JZt ...GWx?7blٲ/ut]&74D퍎;"''iԩS(,,F?C޽Uk 0i$lٲi?^{5!--Mn,22ٟ5k[cF *d666ppp0Ėٳgqq>}yyy(..Fii)lmm@tC qfZ#-- ˃H$BUU?FN^z!** %Hpu\|unݺTWW555 pvvgc:?3& "b-%ľ+j+KUK.h}k-Ric+" }_&̙LfL^Wss瞙31squ|||йsgt Zz&5"00%%%z*N<Çܹs/=֬,AAAZiMLpp0XsD"A1p@>>> T5n ;;{/3;N ;lj?cu=Uk_   &$ ؾHQR^ҧ``LL^u|`E֨]ڹU'~%}ϻDDDDDS H$hѢ `bb333OoH$066~4%  H/nnncc`ffULt!!!Xf }ZaGgd2d2pM=zK,-&O9sȽ@b<F*&M 22R#00Pnm۶xܘ!Zn֭[c޼yr;=[&XDτ%z.MU mC[o`p"k-+Z~jr*o/𴫣_*$E0{NGވiNWT.~%""""Xqqj!鏺unݺJXp!" ),,DVV߿;w ""gΜA^^ܰvZ _{Uzz:^/XTY~KyyyXx1ZltN1c矟o+..Ɣ)Sdtڵ1qqqz{{O>s`ǎ/l^|ƍg}&XC| -:u|xkII Νc/Օ]p .DV+ M6󨓣1ы_%%%6l߯.\1c|rlܸ־}{?/T222waXlO^z>#XXzM$$$hPSquwٳ'>,l"u:u  B˖-٥;3+rs3búuV8&MC:u5낱-[k|>]sppd+5Ɲ[1ƚ[J0(uEwy5jVN=U弢?yH9XJDDDڳg,_nٳ'.\kkk|ri9#G4΋}rKO OO٘1c._7}!NC oBS/ܹs(((72d2qMvijժgG~YXofߩFT*tP0^X*}Qqq1ƍ$2SMjJxaa!Ro锨yEϻmKDDDD"q*]TZ^={ܹZs%$C͚^Y̰aL8tmܾ-܄xbʕ+Y\CE||<6m2DFF*?>cxe_giHN~WjjZ!6 T'G1cWmǿJJJ`L:ULai2O?_+cKQfMx{{ U%|(Zzq*C&͚=[wRgFF:6_ v0|po VcE:mW)CEyyʤr e|qEˡyEDDDDDDDKHW`ll yׯfϞ7̙3о}{Tjb%h-ZWL4 4-UժU 6mҥKѺukC1c ݻ͚5Sy7xGA`` \]]+= WZkPF رS؁&MѢE V'G1cWU[k׮}pqq h (( &~bԨQͯ~h߾=ڶmMaÆ9dܽ{ ChhJK Ѽysq􄧧'`nnsss@FFܹӧO㧟~;w{_~KUǀwɍ=~׮]C+={DD0p(:_m+REC?ѮC'|hOc+*"TzE,sQQwBqLP%U󊈈GEٱ^ԴiS\zU#cuFƪ{ｇ{Oӧ!ЪU+ +@~_~7T 6DF*]knn@"..111HHH@~~>$ agg&MF*@LL R١~h֬ZTQEU/CD#Gbܯ[n6mҰpBYF2SNE׮]W)bbb&&&=пO wY wF^j'''os7|ŋ .~ #Gpleh#RRRѣRcw5f??m\7!--Uϟ zua·VxPzEQ sGNN&E?UuU""""""""DDDDQF : _q54h@]gyB:`ݺuV{{{Z Æ Ñ&w,̟?[nPQ#F5 >Sc;wXa=0`kv̛7O-7fooTgY""mrssϗaނm߯شaE\QR,znGͿTX\jff0tޯ Ns 4ss9ddk2Q*V_U󊈈 K"Ujv0=|0z|b;v`…LU;＃3gয়~?}4*Z "XXzYQϜ9#֭R`*b =v.^8MV{+\  gW,Yƫb9+g{UkbwpD"AiixFrtD'yy"f"r^{BQ300P:`͚5XRsT5K,|^._zss]JJJ=z=.6uͿw0?3`bbR1֭K 1J \r՝T ;;{x;"f#.57o^1YZZ rsrDD\U""""""""QQR#*ۗ4jʸq.a믿*᭺puuE׮]%333=Ҟ={=.j׮߭¹KxcaْY~Cص+*:Pݾ%b&yy5s񜈙(Z0QWDDDDDDDd2¸27Q?'%%!44TČtcǎx3QO@@`ĉU}9܇ vkmР4hDDPN]l۱+V@`*شY ꚢTq3QO?΢Dlcmm#{綈WDDDDDDDXX }􁍍p1NQƍc"f  RyLEATHF%~gV.ܬ IDATNF{wp=gW/!9)QluU0#AKFIu Նr^~PTX*JĒ Kc0=XFFCݻW18P.z@0έҵsVVV-׊|J0a Q'yEDDDDDDDDDDDDDT]vk׮YZZ }٠OF-;z(rssUԩSr_􅁁;w$+<:ve+nET;ŵ n͏M&&&xs[g‘׵ISoر U>WDDDDDDDXXJDDDDDD1uiӦt$%%YČ4͛7Ldee!''RVVVpssCzдiSԮ][a{Y&q!ŧ/;ʕHlm*uw&!rcxt/H$c&ٿƲ21oL۸Cʛ ~w2Zk"fX6~p^~c{wcYYgbb"Tk̪r^z())ŋs]@DDDca5j0_e Kqa:t'OD^^:99N:ZNǏ_-7m6 KؠAʏH_V\08d(,[ʍ_p35[X#`ْ$7oNx{ČYsEe[BpXxzzF]\EN>bͪƊӹ!*V0tyEDDDDDDDWPP ca)QQRRP]ADzʬHKDH)_ ɊXl֭[*X1qD,]eeeb'OǏQV 9r&&&ӧO$"RUnn,,,+=΃|\0^۽ g_a13UaoVՉ }9ޛ9IpE}vԶE/GHq?n\~:`ǮѸI3+}ΨS`󱣿cG"g2kkH$ʍ :c'4fU:Hر۱Qp]*JO]عs'<==?իW]t+--+#;;ҥ +#:`蠞e;}zr6h_5{(.!mYkVA,VEоcgl޸;4Ƕ-!WyԔdeF 틐S'ӫQ|8}HJLPkMH$2=l\CDd5*  Ɨ| =Syܪr^X0R"dbbccc]SNH}BD&H`ff4H$X333?ҥKu^4iN>-7m6,X@'N6hРʦGD޻{wz&s?NKמhu=W>eeea}_ǃp())g015U:붡G6p@ff~:˗.FŵQ\\dfN-DGG{ `x {k93ߴ;FJr~wbna@,#m6ѨqSff(**DNNRwoE?y#yz[jq|N>زy-o݀6mۡg NΰGIi <~1q2"KȄذnܿ'O`tͼ[[hаammr'xpnF^GFF(8 o M!T|?XU:^w:iS5/m4[H$,Z%DΝ;#88Xש脁`C!"uE@@5w=E5kd KUVV 5bZz5J6lf͚%ӭ[pykN  KH\υ>y '&&O>\]˞2txJs܊fM7lfggTq >\ob=:'.r·0,SS\. vΝ[~:uL&C^^6Qu///]ADDDD|MT EL(\&;bbbtR 2Z!H`nnGGGx{{cXx1֯_/оӧem_{noȑˏHu cccY#Q{f-qet調G4>m]NZCص(ޤ+ZR33jTyEDDDDDDwNftmSRRhu""""""<.:t[wʘ [np 8PĬSzзo_ 8p@n,((Hp#Fh$7""UEGigm31JZNǕ⪁_u-dY +7FZ?//I_I"`GLX裁1DYf ww$aM-am]rT,,yEDDDDDD:)--k=E+~vS"9w`VZV~Q19rD0fffÇVČ̙3_v-m߻w۶muj4?""e}>qIĘ'baiAw&csOر}3E\Q{,>:lun7PvstԵ?{Fz{v]lŅa(++S{<>o FzNΚը ?qJ :Ji``zu?눎]Q~Jکsw' ۶n+{N422WlZoXXX5V~q:*>#=rPKK+wSU:^WƦZ]ELbXXXitBktL"""""">+44T0ֶm[3QիWcǏG5DFǝ;I/\v ͛7-""rbR"%kk1!Dž ax,nߊB܃X$##= |-kFIڭ{ㄩTc{{7Gff 14-mZ7D Ѹ7|Z]s}`VKh_sO ~Ǿ3 cߙ'8w69ш墨R,,-aii{{4h QhZ͵EXv+ ,ٳx> ׯ_Ń99(,(D"1,-,aemG@uP7n-v>J8 Sgj%+/"6.W)UW}/6" AAq%IDDDDDDRQQQ wMlcX˖-E̤jH$={6fΜ)7zj_ϻwFT%53C=ѥkO]Q7YשT5jbȰQ2lSQ]BntJZAKq.ԨwO^^^9rY)AXl5j`رrcرc#7>h X[[k3=""""""""""""W733)""""""wD1$$$G8|j:t _-b6U˼y tϝ;>0a6""""""""""""$7 ~'1_| ܹs 3f  $Rf֭lmm-œ9s-bVUCÆ [vm[iiME׌,,,DʄQǎ |||0k, <ނVPPzM4A:u [[[HJJBll,?T8vfͰfR)O3-[୷B߾} ݿ  Vz?\^ &͹ODDDDDDDDDDD, K9#GbÆ ;aXx10w\|"(޽{wFѣGammqO?ůX}ӱaÆ^cSSSHҗ+++CNNJKK>iѢx 9r}% &N(BVDDDDDDDDDDDDڡc9 ĔQ wϯƱcPfM]333l޼S3*_]-\PҞ={N:"dDDDDDDDDDDDD999|:{.vu <Fjj*;ܸqCff&T sssXXXprr+jԨZjN:􄇇څ~>1zh4^k[/ ?Rn!,XH?8 """d2?~Ǐ~~~NZSTXjll SSS!"MH ѢE]EE2D L&F닜HD/^DllӨ222cVJDDDDDDba)fΜ;wnbb  #""""""W^u DDDDp]P)*,1"""""" XXJe˖arcgFÆ EΈEbbS """z$%%:j+''łqѫtrss1|ZJnf͚DΊIQQQm(++A6DDDDՔK? :JRԭ`a)RKڢWKDEEaظq#=z$w D۷R aQ)UU ,,%"""""WM6!..f͂`ll,#//1/^=z-8c1"""""" XXJU^||zL pGF`an=[kK|xxvn.ss!N\ ^X0m˗QXT$!5ln!phxyzj,]uWXzWnGpc$"/?2r##aT#%%%q\w<Ν/fƘ7Dޱ{Er144Dm!0167I%O}TTT,W&$W IDAT R`O_ض *{=Cݹ;+KjF[\劖JJJ(oV +/F:u*.+ ΝoǎHHNVrj*j}y̝nތ_~MW+m((,WV aOr<Xߵ8VuǷS&!&&F0쌨 ID K9.)) IIIJcjjjE]QkԨ+++r#7uUnz^XJU1<=)}DT2RϵhXƍĒz9C+,@ӄ\7NK.\ !Cpd&Qk va7 =3Se`wMySR*,{(&̛ܼ<}QDT: =+WwNK1hsIє)XG000Byߟ އfff"eBDT9;=R"""""""JjިҨ;wPT\ c# &BM49'1r,DW ?gS齟w5`9s```1oqE]Ѳex&|P̞9Ց'̮]hf&$ȑAfgeaÒ%Ly|D"<t%yXTEDPTjff;;;888vvv-v]ޔBDDD7ADDDDDDD7j_.QwyF:J5BUիcc!_N 3.+W зsgL>^ܼ<ܼsAf)Sq,+!#+ ǍS2&͟zjwQد"ii7w.8sF1SRTֽ{EnnݩjIsoǎ=o͘ 8۫X*&OtQ3wB]77,6M#)OӰaC\xQ(*,J:+466N&ӧOGff&Ddff>Ygff&RSS5vË#??JîDDD+,,%"""""""$EQQUE"gBT5%b:uJ9brcC_do77N7n.wݸ8|w…!+(@Çq3CzFΝ.0Jk8_زo22_PX,}*Elxp*N EFТqc4o.ԪҼrrm700Y0?0&&/ŜЬaCտ?ߺ|krg?NLĒիg&,[\~]nL"1n0lHHNƙ -|k'Zsiٲ7\4olllD>CCCEc233O_,H}ViGծbT8;;ypuu+ӈUˈh9}K7R%%%r % K*--O{CrZ\Hnvpnnkn?#"܉wǍC:uu_}3fs&OƬq`'LIr8#!!x[7EW"'0$09yy`X߾8]4$wQ7-]oU^^ݵ =Ǝ[qn|2c]K#-rc8nv;x0F?%W;> W~]%_300a0l0qM?~\0޿xyyim~""}RY2yirrŽHIIы*((@\\ŮBQ_,ReWT"" K* 쐒^.& Teee8,7ɓ.,+w9liڴRcUg_W"#EHH \(>ckm?#_&{)VT\nڄ_~Y8A@N娥dG);] æOǡ'E._w!LBq)Lp{>jJžUдOr&ϨQUs.[(bbl?~I_} ƍQQo*=?:?Ǐ+R+1B*B* 6TW"մ4kQŮ111JƆ]Q)iOӦmDa7X`YC'Obi_QXXJT+W 8:a͎r{Piu`~` c؎݂J#Zg.\}ߏ?V!6.].|Ji>yCJ{w K2KJp$$DnW:d*óvm:\ϰ0 KDFdXg+,*}?Dtzط7b000TʪOz٣Gc1"Mݮv?}(YmY~R+(݉H K4@airZ$%c. .FDTz|RڵakmD́'͆ 8p?s33駘'Z#/?_ko5Ԁ|j^頔cַocXY}doqCbb))zj}y)*.ƈ3!7Iv|]r=kF3%|jX߾r KEG5:V9fM̙Z_vzir'$'=*dػc#.tCMq%2\lCI* i)\\\.D)bjBBǮ'== HJY\\SN!$$7oDRR !Jƍsڵ+LLL*5L&Ù3g#22˃9Pvm4o;vD˖-+شA.$&&"88aaaxҞm:88uԩڴi#*JJJݻ?FLj#@oXmd CXX.^Ǐy\"^zA>}кukrꉅDDDDDDDDЪY3Xxd$vָq CYҫ+*UBCq2, ᑑ!2Eoi 7WW4_]0GVj>yřq><7{YE3[kk4o-7Fh\zQzf&SRl*J'r<"\ nni۷ݺ̬%UyWǴ1cpX_ϝ)FA")#"ni%.u[Xz T%SD)}y_u7.K׮{gO,7O9$ F*_HԹs֘[X{p0JKKTA?(Jpww7|dEYYY:+*˗?>;S1/\/=*ݻwq]CnT'//[lݻ%oldgg#>>.\ƍ>~~~j=6mk˸~:>|xdeeA&ĉWz ^DQQQxBB۷ >ڵkɇPYYlقu֡P>O<޽{w^ :}V|?СC o(--ELL bbbg4m/i@=wwX[Z"+'\2EY)zթs琛 5az$bYESRƍؼg`Uzf&33q->r3/F.]hּτ]")OwE[}:ugڸmxox̞0֢$)I}VrG޾¢"tR0V ￗ>wurO˗E9^2pTbeer4i5 郏L\~-Z5.^VZZ6'T TRWL$&&>|jGT/ʕX+f^Ĥt 2ϝy ?[ʔX *TJǀ$/G bep{D5EĠСرlt\`30f\=Z(üѸNޯ*p]ar\|z.3ު*gEHs891K$IjR%B)9V^6UQ+NRW(J䭊jee3+$*jrr2f͚7n(uޫW2J7naÆ ZZZīW+W<]6z437UVppp!RRR+>}ѲeKXTz ܺu gFBB $&&,oI>|8Zh͛زe ޿/m۶Ah/^đ#G$}&"'7CM5k9eyQDDG~P<4P&&%\ȕB\VV*IbTKx}WWX/_l(L^}(sl܀[B*Cy338:%ZӒѷK/%5XV&vZ ekk$՝:IK,Ԅm-BQ,UQD!.. GBBbccwʇ!,,,`jj MMMƢ^z իXlL-HOO… %ĉEdutt777ٳ˗/f͚5hݺ5L8c٘|\6u0q` 6M$sׯ[۶2̸,myGϞ]dRiڵѼQ#XZBOWQQx-߸ 6 II:|8Ξ:8:e&h\\kԀ t%&Ǐ @+x!uϏ96sL<<99>jԤU[7VKK$ 'xnfVŀǫQ{L\\񘲂j0h]^\QQ8vgŲuw-fKO.rD]X&,XmKk_`in:x9VjUvrp`|TR6:>[zz:bbb8mmmޖRrssp^JkQ_ߓDbb"8$x&%%!..N-ߛ)))HIIB76lP޽{___SÅ:u "-Z~BCCE*D&&&" @jk֭[?ss7jhh`ԨQEƮ\R,~ XV\ ?y*KKK1:u>ǹ>8 o333J$ŋ"K;;;rdFI}{w! {,''۶mc&P+ϟ2o޼'ܸqKL$!B!B‚YYg2͕nIĻk.m@(J\L1 g5X);9+?T8-- kw5kmFEaشi8y3Hn|dT5 Scc8ۣc>jcٴw/Zާ;Y[2Lzl do׬G>R0njѣ QiĥKqU١g1#Qg$C5T5[ָw/hմ~}\߷aVۺ?Y85]NZ:xg \;u #{n] rםǏi^A`2י͛}[pCHJIAzFtuanb**AZh?$LLd79^K9?N]&1/32\WD8GQex8Q KT޽:vBȏHWWvvv222'&''KTDUEUTYBCC1tP昶6Fƍç;EILLHtss466FqpnݺʔT &&&[.z Ľ~+V߂ AI >~(sss=}8Mz΋OQFgϞصkXNN.^^%| "  ) %B!B!(Quaሎ3jvv]&Gi8y2vk\uyUf[s'gÿO/20GG{7͜1GBs3WI˓kUdJ욵r%rtuE&5kиGefeaב#2|8u/##XQ*H+KKKԪQ߼[};vΙ */E5%>1>'3;U.m"cb9"9_FĹkט7&׉xKocM 0W/?cZ` ;q>\;tOnpbi67l޷9ֲqckըS2~GΟw2[.߹c^Mlj͛eJ ^[op/ajah^& ;|z4V؁GÆ^EȾ}"߼ɋaÂr͛+;Q\8>fo]H,4ר(ۖ,A#Y̛+Z<µzutiw?IXCUKLNfV *@ t|߿g&poB@q^JhѢEI/ball ccc^E||<O޽{2{b&M0j('k0:TXQ9jժ%{k}v) ݺuCnݔVbhDD疐 S SiiӦwqqHÇ_7Agdd$w K !B!Bq: kOJ,=r<eNGDsu$а byT~ NN^[/gB[ÓW$VQKo?z_FDFfX=پ]愥3_7p\ݿ?3OšrŊŞhtY+W26 #3ϙ;+6!~G۷q=^@ut\dT߆ AfT҆UVsG6l@e^G[7m __ؾ]jۋY@[K gx9?XҒY#ChhqJ,%lhӦ *V^JCʕKzPˣ $ZzѡCK.[*--M")[ʒ97HeamllT/x^رcHLWW&LPl7ߺu  ykOKĪT"1%Kq 򒺏pD2E*<{oդ 9Sb,SN'4t(RO\񏟟\ GΟgƇxy]U *U b;//OR220y3y2s|8qp^f&&jh[Zb;<} oƝG8ۧ@yPhv=kި|gP~KpΟ.;v B5}:WC*9|eqVF^ h G[,'qcccB[5PF^!رcUXJJLsB#::{URQ111x%v^JWx JJ ---ds9Wis%_HMM9::ݼħOǏEX ]==9S(crm '/]BVsȋwPIC---f< JǧOZ TZWC!j]Tu===V???:"Tڝe,/d_hٲH,''SL)2A``HlJ~_|sSDׯrIQɘ4iފ?6l*UT"k"U,%B!BQcD<7|Tw3O.pWOj镻w5* RNr$hs䛖}Q ;K Qx $1VB\md3f\[K ?)Ίۏ.__82.(bݺ9V vRYpUUfsm7561jT!^^x9z iQQcX0zllٿ9ӥKocҥВEZqfee)eLyEg"޺S\Ջi%B!ܜ!D6޽?,X'0upU,mި6^0&&-r{>깺2K.~k0+YYX= ޼ό;VpbarxDsDhij 8V6!#3S pPb˿9Vaƍ1#ݱC+K2{wiP9Fݻ#,<\b|ɓktX9W~~>~1;ƅ 1迊Yٸz.Nı --s^c7K6heW997ep$YÇPB1B!NNN"-q9tܙ<$*333fIÇÔݻwHHHikkW^.?~ݻ7Fccc^V&Oeb7.O>޽; mcǰl2LMMb %Hbn"G!333^s+KFFzccc+8PAa̮WEuB!B!?֨PqwĒ@ {N,ԫS'f (i-sX 8Z9Pb՚sEH*531E??8V\۷T\9w~N/o.2DvУ}{x*U8`jڵXns>܉j)e_eQ:Gc XYXorXfVv:kE60JMLpII7rN s)N_ɋZ qJ6G97:>K΄eߋtB!K"tRxyyyHMMihh(%0??sA NNN}BTT3~~~0a}' rRB!B!h٤ 3.^]=Ǐ3K,,Њ!!x8Y _`V_@j8U8 &FFEn\Jۇ*NkRǫB'zիml\>ݺ ,LM P#ڢ& c6!#ݸ39RַdF]9S[½^=쫬RE+sV>z{[" w Ɔ5v,/^Llu~~G]*q$&$'+<7uiŅ҉PB!ubSRR0tP\OLLD`` VZ;ӧO"㺺JYۊ+p|}}eyS# >Ϟ=͐!CqFXZZ2CBB(srr޽{zd1c˓JFɑkMʒqa۶m"Bc!B!B~@-5%⡟>!:."|8!uY_׮tD|#psvs%hX~ p%Y1+4+;ѴW/ںv66rSIʨGuկF{\x9ƕ(/[b&GeLmm۸$ODqTNVՠ=p}ϑn2??,`&կPՕ˛Ȇ 닥613u2k֌|hoܑ11c%V)>>氲*B!HŋcȐ!" }YYY󃟟tuuabb|Y| k͚5سgHL(bҥXkժԭ[+Wׯ_… 8}wt̜9#QMw₃`̘1bRUɓ4lSn::u +VCI/2&B!B)%b;߿ls>["ճC1*4;u 9dRu%ABEaܾIZׯP(^ Rv|b;)K2Ǵ45qpZo\}( qErscSRS~fv j89aD>2cɔ)ؽ;s,(8XjoeT3)p(:>K`NJ6M!L...ظq# 9ddd 22QQQZ+XaرC">g+4:e| ?Fƍ zzzԄ1|l۶ Fb7E")%77f‚ Dk֬#F^z7_t ^^^x䉌϶hXVӢd2:CM>}G֭[8z(VZJ<ӧO>|D_BBB!B!*PB8* DLKS}t)r_FҦD<:.]s%JJboxmGRJ 3U`Dze9f g4roQUG[&&*_ela}*+Pŋ7osLS(ākS4\*4oU9?b9qq ArV553g䚓/zzFd%"771Bsˋ+??_QF1B!h߾=m444`8VJױuVlٲE">yd/rϫ%b|ntss<#G //O궳fӧEb @@@ƌ;v˘9s&uNHHѣ ^kK]Ll^s+P(VVZ?SpB턄L:9 v}GQ&͕ #1^7ThI--  ;԰vmfrϙ jՒ{^yY%''"~mmm8BO!RZ^p<ѹsl2x1~xXGULaž}Dk?իWŋ%Gev^Gʕ%b1112+gIJ.QӧO OXJ!B!"ʡsʝ;mj%<{wfR49ZV:r* x6m{? jՂ@ `]H\EoxsQ: _pc~Tf 5)3+ ;)bָqr{9X]E$"Ņ "Uׯ`]<3ʌ+ݟXU”uKkwwfkY-ʝ;HZ T/((ZZZŸ"B!ۻwJt IDAT@2#̙3Gb={bʔ) 4(WD,!!A9X8Yt݁VZ*}OhܸH,&&8:"rܸFJ??IڄpRB!B!j߼93~ D!^^2VH:z^EH"%89ܜ~LL8+1Ӝ7k`(u L?\&&pxp%H~KPdddpWXfw!eڻw'd1*ΗU:::pttDR]+V %%E$֠Aptt`5kEbuŋXmCBBШQ#sھeཟD%X^z<?~fݺ)NYY69VbExw${v3~޻SI3yVVuѧ1.Tfegc̛X>~k0~) X礎'͇ա 0/Bʆ<`|Qϟxrssܾ}GΝ;zj̟?'Oڵkƍf͚R [nʕ+1cD{pss5Gvv6Vu_4^:ʋuy&yqUX}wCkF |Yş"4hCCC˗%رcquI,btpp(Ր҈*B!B!X6mgϞ:H sx=c˖ |L:[GocS..koߢH8i;kXB֩nN`xQbKd$N7Kc4t(sl)$ɫWhݿ? ]Õ={8+Ȳq-C||HrʨXJGJ,%B!BQ?r Vx<<`gcë Aoce IqX1s&W|LJj*ލk"3+_<=sQ+WIxK䜣cթ=+1ɋ_ѵM^I91b|lOL3iHRUD <=}VWr|KMEJj*}Çϟ[|O_LZ>W@v^KfVnph664W[jQg& UhV@\b"MMc?//sWƢ VV֒۵C*U؉KЬwolYnRŮ#G…rQ020:2_Y1DHe j, _f>0 x^VsR.<<?BzqEBȏ+''qqqFtt L(})[VV^ 6ӧOXvLIx1Fܰ0?1o+WpwuE ace]DDG۷p&s++l_L5#}}߼n?Ld ڸYJk-xWCa=G`Xթ7ZZBBRDFxRRS&J҃gϘ t~'^N*+Aj\\9t7GJyըZ +-MM?m||mWnߎS/cx>hް!ll 60A lYbY444zFN>]cҦ 껺#>1#"p]6.\p4^/dnKTKX3w.2)gfea#Mpan_"#9w0UsoߔPO__$Ayզ\37c ⧉Kiդ Myۏ1uR]8y2:1.$ri憥S%K8{^&Zl";l?ׯg" f䢯kJ-n'M\\޿9.Z)!?( !QFAuADn)))C||</66 "8İLGGB%NIJeˤ9s-@GI$v9sFy 5jF%h4xmt]}˜1cpU|{,55WFiiix5_.b" ,,,0m4,X@$kG! tK6nMu Uҟ`inΙURdBOW)RcgcέZOOZMD^oI3M'L?-ofwrchPFΚ?=Oعld]}ܹZf[R%Z]]eVIݎO_dd$޽{'1 6,B!IIIADDC F OJJ*%+LMMaeeˋ$N-H 511n7={I& u)-lll.(5zI1w\<|vXh\e~3n8TR˗/GBBL >|8F!|2^[[[āNKKKrVRUdQQQB4443fT9ȏK !B!BIN8KV:-1ּQ#'n 8^:`y0}m[7 pѷ/£$╋PR,ͱ| 6Mb,#3ÙۥΡy'bx>X};E' 8;8cGtl![[MUuutqB7yyy"cB.) ]mm QnΨ+|uG{ BfͰpZ?t^iki[7,6 J.\}zN|,V tm ˼UE1._UѣQN__*9I :nggj!X$%%!&&JeP(DPd E%SHmmmQ\\\Pn]h)ZNNRC `ܹ?~HaXl됕-mۆsx)gEY  zΝѪU+EVFvпXYY9^ *`Ϟ=8x ك_2gbb={bȐ!0005k˗q-ܼy=߳ѬY3xv!E̾4B~x1s,E:alBwB!"1!/coӠAjH~ׯH8//&32pU\{O^0$ '7hm gGG4o]==agcSK.3rrsq=07, 11HMKCNn.uuah`cCC8j@FPB^>Kocux+N?T+#G|$˸zy_ ۷6VVpqtD~Bv`|~., HKOG~~>L`jlfff YÆJY_? o"6!YY0G P 5/mˆƢÇ%qɶƍW&Dy(22RъRTTڵkg;::Zkaaan@K?ټy.hQG|XVY,R``&u9r䈶l٢ edd~ffpp:v쨔IKHOOÇ뽦k׮JLLQG9Ѷmd6^ǎnTTF%&&gϞ2^p X 4{*33ktW`0+МTUUCfY[nUyyuu릔L&L&z[`)Є;+VѣG+22核"MOOW^^jjj4!!AڵR`)jjjl2{]v4vXu|eJOOWFF233~k*11Q&IIII2d¼-`)eVU˖-Ç.$$D]w|h VU9992_m۶I/腎R֮]]y啊AG-٬4egg㺝;wÕDٓ )^۷#FW^>xRJKKlVnn=۫W/4RɤX/t g^vZeggj>4ɓ'qFedd(33Sf@@ D%%%d2C^!X x  vYcƌi:͛7l6l6k޽Z H֭[{cR6Mk֬ymttz2b?8m_PPqp%%%ä PV1W͛5b\=X,Ztkt뭷0ǬVrrr m߾]׍҈#d2d2ԿF/t ,4)ͦ,^ZK.Uvv/^L^JkN 4iƍD \YY-[Cڰ0] AglEȐlֶmTVVqݘ :TIIIJNNVll!X JmذAZ|v-I g}B3~x}۔.I۷M0A]vafx|rUTT6m[nQTT:)))w}',٬`P\\L&RRR;zc7׊+|r[NC&MS|ZpLjܹS;w/֭[/ׄ 4a TXXedd(==]6׶mVr"""|F}m4Z@QQ6mڤ eff*77;JHHk?x`oKp~!X h3j׮]u^KrM79KO+++ҥKtRIR~!S_vڥ˗+55~TWWkݺuqpx㍄Jx]mm7 g84{t H4dy[Kn;TRW )//OyyyiW\q&L[KYY֭[gQt饗j>;^>??_WVYY[wQ7pfiǎcf <a&''_~ $6+ SYY233i&} nV_+ln&}WUYYYeeeꫯW_Iڵk>}O>ׯBCC}2pT 6aߗҚk֭ln믿"88XAAAudKкuk&TSSm۶l6+##C:~u5|p(11Q={hB\l6~ai&ܹSE6~x}nKtqmܸQ7nT@@z>}'.Nbbcbhٲe~:UfiΝڸqݾoРA3f ^QFi͚5n/ F6EUUUl֭[:\bccl?>66 2,L[f*Mڶ-K'NxnEE*j*??_(K}zRjO6MԎ;cϲZoZr"""{j*,,t@;VM+P׮]uQsѣۀN:e?>==]a 㕘$%&&cǎ^,fil-Y2ۤݻ?hiNiwiwi…IW\qJ$@P SN^~AS%)..Nolٲ&*))i=4iMO>ӧڴi=L4x`L&hjӦ:g#X -@q)[V.uTp[{5JTzf$G׮*%E㒓uQ l6ehUڑfkTݻ;TqqqXM0Aaaanؽ{233L߿㚡JLLTJJL&ߵR@֬JUJ۰Nen t^Po/\6aa|HM=ZFVENiڵZvoެ'<ٽ{w͘1C^{BCCuWwnZɱO#5*((nddF)dF/t `)\ ,UUڸq֬JժkbիՒqq0f&I!~{vkZaٲEUjwUw}w SW\qZnVhRΖlVZZUZZq.]hذaJJJRbb7, DyEN,ҩS'URҼZS%%:Y\STQYI\0jkkUtWuMW7ieZȑ#5h @(--Ֆ-[HsssU^^QM84RɤX/u fr eee+p}jՄ]3l6ؾMkVК+ej/*91QGQ4_?Oo uY᧟/̃6rݴI֮U7ԵkWs=jժǵ+88XCՐ!Cx믿.VkpFoI~DQQ6mڤ eff*77W Ѐ,ɤA颋.Ry}gE6篵GWp2 8hf=ʊ mْo֭Kkg[.8qJNLTfv\+@*7߬?چ-Ylzܹ*EѺoiIW\IW\!I?p@viϏ?6*rA~ (>>^ h ?}"izzq`%%%d2)))I jݺR@1,-de)+ˬ ώZ֭5~h{neJFc_ֹޝ;W㒓֝?`/ZxddX{^SvNTK.O?퓠J@@ԯ_?&I1@Sl4fx\7<<\III0:Yzu(Jyy9ޖmYڻ4[bb'.N}zҋ?TR83gCT%LwkѻSgo׸SuabNYKzmoppS|~_ھ}rrrgϞsN3׫UV)*blVU999JOOWFF{;vQQQ1bL&L&/腎@sEZ |T]*>yXYYYʒlVyy㦙F͛7Ov;j:rUBCCշo_b[gBCB&}ך.4~(w4U]紗yy?7ɳ۵k1ch̘1*))Q׮]b }:~:effզM& EUUf222d6uF/L]ve]fz[p>!X pйsgM8Q'NTmmyfڵK۶msNmmM&@~јP`PnԻwoǫG7eQ\Zy?}!  GNUUN.Zi7ިQC6i:uMZeddhJMMUVV=X*I~¥"MOOW^^jjjh~^N{6M3fV c45|p >\> rJڧ~8q.\N8͛7+--MfY>@%$$ا&$$]v^\(֯_?OfRYY֭[g综b[ng}F>dM2Eu^g3FaaaիWMlٶMo.XrMX} w4O6MW-[vݫxC<::t蠩SjԩNLZ-[p)p)##Cڿ5CBB!Ҥ$ 2 ,4J֭uk$=zT+WԲe_ԩSk-nF͛7O̟?_ӧOw+R^{jtbs9P>3rq1 /oNWL>^z/fiǎ2_m۶Xd_F `)+:viӦiڴi땚TٳG>}Ù\&xbu]Pi޽5aM0AFRV!]YY.}N+WQY?xY7V:Cn|b޽ڷo,[񙐐K]vw+Vvvf222m68q㺝;wÕDٓ )R׵j蓤XB˖-ꫯ2^5|͚5KSLѤI4zhЂl6=չ9 +x!}t w{Ownlk֬њ5k݆_^Z7xL&[iJeff*--MfY*//n^GۛL&z[\(l6*++ Zshr:tO3ԦMTUU @eeu릃{ Itrmuwȑ㎀Gp~Lz9=+CgrYVmذmfӆ }ɓڸq222\*؏7L -T\\_]%%%nsκ{B,ThhƎ6p ՘1cp^KٸQ%ng ##uqve?{wa'imoB~zZ >dXT]]6~r!m޼YfYfYI4x`L&hjӦ:ƅJ4ÇC Ah_ l:^T/Vl.7/6o֎ݻ]L=l۵>~E9uqn< ٷOrsUYUV|h4sJ11aZ;ԓ.&ZV`^=~ݻw+33>t UbbRRRd2Կy[D6C),,TNN$?kpa#X `݂_YVOK/ }A{:[mjz`unrrڥMJN_~['Ã*U+< pAFUfj`ZcFBFEEiĈ2L ) ) 8P۷w.vKr,Z0k _/T~Y_Yr/{w]uenZvm JLYf.vd-6T*I;vұ"VD6N҇_~Ns`Fyb(##C2ڶm<C*))IɊBM`)Gޝڠ䩒ou^*Z!nA**+݂ߕWTH^J]' J?#X 8oﾳO$QEEG5 d2DuKRywMb/5'm|tuѨo!+mu۾kvޭ޽ STTM6)##Cͭ(99Y&Iqp X 8e|˽FER@@u#oezh;Vm&|z8yIq׭?gKIӵ~kd2d2)))I jݺh~dٴg>e)wٷO֑ctD%%TWFAAjn߻F&&jhӦI\|y{7]}W12*_Gûkk 6X:v6Ćk`W_]gtzfӎ;ڛfx\7"""5LWhh:h Wzf6oݪﲳRXTeɓڹw_/I քѣuϯ~.Il AFng-ALǎܯrQޞ=-UMMm&٬ }:vu4b(11Q={hBCzfŢ/VVeII^}.x˽~ԹC= h3eTX`)IYV:tHe6uV{\[nJIIon. K>QSSO=V^8Nی:&q}8v.fwh)jjjBhʕՋСC$7TUUHRXX[|`)^P}R}W.ȣZdT:xBUe8m޶M6M.d:ujjj|oPPr( IDATlFv5A@QTT$*IjݺBC555Zv֭[\b(44T:uR~4j(3F=R|ҔB+,,L֭ d%$$x5ѣGfk߾}:q=ٮ];%&&.dOn:OԹsghի_; ="X mJEiѢE馛c5#Gtwȑ#۶mSG8_,nd8P}.X]}tZj`)=vL?Vnؠlzkݿ_K~[npOG .Ĩ]ddkp2X*&X h ]t`iddFiHڿF7~C>p#G 7ܠ9s( O o׮]{4c ͘1A6ĉϴ|zǏw;XGy)X]vi̙9sf̘KV~믿v%K(//OoڶmۄݹH3fhg.K,U:6$IW]7nPHpׯwyD 7[}z0<_|}eQ^b||p'kdvh):t #*%%%)11Q={$H 4ӓ_{5}饗4|{AmڴQII/BO3d^;;j7TTT&ߺM1Kqq~m}uN<: ~Ϟ=o22u[}wgykP<=׿@E***t}i߾}~pͦSukÝ/:wW~Z~nؽk?XD^s&6hý׹7OPչ_`< &cǎzbkڰaW[etذa뮻4dJ?֮]p_|A馛n9?P!C4vX)<<\%%%ղe\N_1c؃T驧RQQ[ JMMUUU}z?&OAޫ%K振Y9Y1M]_} k޼yNwyf̘ܹ) {-nppɠ[lQllG}ߛk9s+RSS#(22ҫ=zno^oOz`)g 7߬wz }{3Ņ`)]]9%#K^WTTdNO?䙧Cgku;5Nnth*{ѝwr/88X3fаa_ڣRXV?a*bsI8aWW{(''a=%%E=bcc#''i祗^ڠÇwJ:g\#~)?GLVUsq*IO>Bt V :TQOgKKKsZݾܸq5},<111;w9'"X }hĐ!ڴuu7,XT[[r^=}^UT@K`ist!]z+ڦMF_[n\>|fΜfaaZ׮]TcNk;wltO9;p){)cpK:Mt 7x9v}+**rZ3=Vݾj۶Cl6f5͛7k…/m۶ML<K>7j:V:JbZ)ƪ뇈-I8L ?&?^ҤI>][10a.]h4:pv8[>/>l6kz4M0w*`X6a篫u1 իC@Lheqq̙0Syh|nD=8|X%ee oݺ啕ur1gT#N:)((HW]u_;555QYY$TX,bl6{fXTQQ!ߗ睃W5)AAA $EDD`0CA~9<88X $nZ2jӦMuu=Rjժ>o.ꩤA5Fڵk*,,TnnZNԽ{w?4_| N۲e6ydOxھrJ͘1íen߾_AAz:qqqNOd?&I~{< -R ۷i9^Щ'= G+::Z mgVVV̵FW{ ];3 yug^?FQ'OҡC={4N&7y9Lk˗;]M6mh̘1Zj}F>y{ڕZ[lqX6mF;xZǎ\SNNk?sz:4ߎs=w^MI`)##?z9!uY,FnU|^$44~y۶m/Nuו;LD\e`plnڴIVA=,X,VPPWܻw.\SzYV}k]vUϞ=t*--UhhuEiJIIiDܙ3gjvڵKӧO /8{2224{l^zy ;֮]quOaaazOEE?9oTpRυn-ѨZe??Rt';Gat z2)u~M5`iDDpC'])߿*=N[QQQM\_ڻwւu뭷~/B/uϣ={j֬Y;w={4ex㍚:u?\suaѨ(K^ v)s+n믿#G:t5MR  өnC[5L kw_#4'Xz=הئ˛DŽ{-ܢg}a^PV6l3gvޯ_?qo. 555Z`/_^{ͭ$vmԫ0F-ҢEԻwoEEEi˖-vUSo*w!!! W=g}3,E}!5W/̜-4$DZ q l3頫$]eaSͦbIQ-jkkUZZ*I*++SMMaӓX\;]t٥fĉo>ZIINM2E]wz)`?yvڥ7jŊ:zCsOKZrZDD^|fmHSTT[?5h >}9s樠iPK"""<ٕ3֘0jMMMzܹs3q޽W`),ղuZ9%uV*9kj$UY,XF@{~p8uuǎ裏|̆ ?ӧ;,͛y)44Tl*))9Nj.^}U}kFQ/vqc4p@(!!A=zPDDT\\;wjʕZlCBO>-ZVn~-p!Du Pnݴo>/}~mYfot欢iO>n;d=35kŋu]w;sJMMuX:uz!FIɓ'_kN碢"s=zwV sZkLi,j7_|o߾꫽R8 9w׹ݪsI=5/+*REe˽n;+?/ҥK͝;W_|~a 6LݻwWxxFڴiK.DW^y|I/wӻwF=zם^M6Q5&k/\sX;rvY= .t Μ9S<ó###5yd}={S0RfRyygiii94oL,?O{Gڀ;X ۶չ7͉)tq׮u,iR@]z=zo?絛7ovZKHHh3.\^zio;:ĪtW^<;vNמ8qBZrrfΜYgr-4hf̘"^AA>s=n[mۺ칡?٨δl2ݻFq]]K>U]SkԹl2]gu7@صν&:vTvکW^ s:WQpKϞX*Ity4aGq9{?"]df̘ѣZaaa;vi{=߫pѣGu׻uSξ>J.ZaNԹ?.9Z .^:-+'A}[u q1>H6amܸq 9s8չ裏zuNkEEE.uqDtu3 0@\r1b -[ w]ԩBCCUYYi_\= y?o>O:20z`)%4$D{fL,ѵ:wgw?REiӨ^#} Zr-ZisƉ'裏a}ܸq={=/:uv6@fGÇsN;vCT``P=:tHeee.Cu}i)FQݺu?|%K$zۤI2 RSm6ha ZږrMX5nI0A ^z%udҀܮ1{l8qamze01wСCݮh5WGʷovש˥^,,]ven_PPamnpslU`Cgڴu3}ѨnuG^z`$m"X mե&չ_;nه>;wްQvMj͚5Zx}v+Vhƍk111zWq瓾}:mݺUݮqvSzVSSsNsaaa SѣG룏>rX[fҵkf9;+m۶[z뭷V-g1gTW/}E$w|e&77ܠq\\r=p/_"ڴw#չWXk׮O<~7jȐ!nը//J޽{뢋.rXoU\\vu.6Ht5AN>2g0L wX[fӔ\Wl6>suoKcb)k6~5wh6oݪ!&&NCӧkdqyNS<6<\/^)DP=kjţN53<-.zt*Mt5j֬YV`0hn@X,Znx ۷i]vzOn۶N{uegUW]$knIw8b_FUoමX{J_mXO7P9~f^ÇևZ~ w߁QǟIJR0H-&`D@E.\A`Ae v`-"UBNzH(\L䜔篛7=亄'Yf)))_ y9|ƍa}.UPNb)PY-#Ɉ9vI}7orӃյb$]-5~KM`Hq__kΝn۫(y 'E]wO.} ӧ͎b0vmfFɊÇ~m۶M+WT_J G},… [kѢ|w3ƮW_}U F Ӏ4mڴk)))zoM69z{ :uSNNտM>ݮyf 4H/"c4i$EGGg 2rv'jȑ5jzaT_V\\>M4QF'73P,bXlg"55|0-[X,  5;F}^>7VP`N;a%R#Slszz-3΢(Wիrlƌ0ai/,l}nJOO[?ϟ}n6tP :4s2<"^4f; ?b 7֮^zJժUӝwީ+>>^wʕ+3g>>>;v]Vzk٭޽["""԰aC-[VE.\жm۴uV_#}Uݺu+aXKHH+_ԁ秗_~= @`,<`kH˖ZoJI)Zxޝ:)BI6M?FVvSf%&%sjdybJJNvKK^yՄ~_'rtma`XZVmp;.frr1bZj{:u*GDbbbffՔ)S4x {j޽._~ޫO>:q>3tܹS;-Z顇ٳgfϟתU^{O.QоN@fXԷsg(&FM^[TL .ԬYD@O Wխ[7mVSNՏ?鉲˗Wǎ5p@eރ *h1cONZT)W?}ER@ծVMKZeV_[:.\$ժŋ+BUW ձE ը\9s RIzӔ)Yޫ^DED+nqޥKTs:3g%KfUbbbܲWvqF앙#FhĈK4is_Ubb4hS,#]tQ.]tEرCϟeXTLUVM5jS-+CjС:zۧSN)!!AVU*]"""TBlf߿ۧX;wNiii*]Tڵk˖_˯]<AJ ڱw٣{(p/ookT65;)<4T4:o2mUݚ5MH:m>*RRS5OΟ44@PB9VT)h{),,=ZO-_'(:(o"ycolH&%`ժ 1cߘc^1HȆŋ@!gZkv b)RRSU~}]zƚ.m&_,1q=?aOkT^=ש^]/Xg.X)}ZJׄV>:!CT\,{]bR^ūWkڱw]^cZ5O}q$7ޘ1C Æjf)_`F^RnAWgg(V(WNKNP*>/YZ]~X;-[2dG:u̎($}Zf2ݲ%Kצivk;Pu}2+.YZ-7Kқ]Kﾫ{WbRRɉj躛_BmV_RdxW=.uĉqT,ްA1pZZZnެNZez>P;t_ׯWGq{o//ՋP@]MHоÇu x:T B%ӌlj*S#'Nh׾}rE j7[ixB)7|}|4OuG9~Ύ<<b)AT 37lެLڴ)k >nְsڳO?TןyF+??ٖ; Zf/=lXoֱeKCߞ}o-6V}T 7,kr1~>wn^^s=vڦI>\sLɋ#ϟ.5;FaK ԱE l6%WfvkYjjV[[5k:|it-%na.qƎڗf)ʕ,W8uJx@:e/jkP>Z7zg\ Fڹo?z L/==]MS55y4#b)J, ڭټYIINY|ˉ<]x|V_oֽm[_atzGԻcGDI68}Zԡc_==xI+' Ijl%ӱ!;yR zqڼcE`=ڵ81)I6ntO[=t~[J|۷w/j=vkUSp;ӷs k-3gv@;|n}a3|9  {eov09OMJmھa7)9Y>"b);CO;vYÆj״J,ycmCNN}}|ԾY3tFu8Cdu19+N;jCv#zHF4%9kڜ9g^ܪ ;jm!!gӳy%%%EVʕ+͎|GJJQ4K VUreWp+Vؕ=;j%ͦͳte>ܪ|}|fX  ūU(b9+~]^{[)y /]'_}|ϫN*`<=Ϳ-oo/fT ޽[&LPw}W͜ ЬY3}'W4aŚ R`&өDZk#^vwN-[| k;uyߛp!Z%*??s7ˢ+YfyuiNZ韃REFr8KOOא1c|O%}ԠAEDDhԨQZtڴioVfD>eW_)22RK.ըQTV-իWOGVtt4((o.YbqRr^} իYSԵMA]NWiR<=գ}\gϩO^5m!!*k|?#S2;GÙ_bb Hñ'|Pm4q8}}U'Oԗ_~{We˖UV[oi֭77H.]4g89KfR=$cmۦǫu TTT>s8q~KLРV-~cEF1Br^k״x*n)F6oҀҥ3]|9?ciiirn-L[MTʕ3^xm 7TZZ+*`﫯fs88eN89JLLԼy4dU\Y+V5sL]x1{sۄ(4g 0 ҥK5k}Qr2d͛D{Q,0I>^yTw{wh-wS1¥KZq[w@g޺ߪN6{$mھ=Zݚ5s.YR R=##35~9ӄTszeyzxS*pбc'*RRRzjjР]JH׮]?pR)rK3g̴\zLy~=zi=(5TjUo~I>&Foznv2}?R*Eyyy~u]zzbbb4~xnZС&L?P|w,ԪUӦeN9X6jMRRt:pI {x$ӐULsrj\vM;vw}*Uf͚5j.]{}iƌtcZe6M_u˥7KHHҥK5j(EDDroc]v͍i wxK'MRZZ$)--M_ΚeW,-﯎-[:z /k7=f\@i}Td,]o/OO͚Xo$b4Otg:w]4WNw=J1 IDAT>|cǏ׽ݺul_ׯבǕrS9TD*n]@k1Çp8e.e}>cƎb&$+XRRRwvj:yze˖iٲeyzRRRt1;vLk֬Q@`U^UjTU͎X `G+={~`^7^5ktkOVÞ=3hP߾*Q5[wҨxժ :bv[2%<4T&OV;3ռaC rxݠ#oJ%J|5J5RpPP^dž~Qmbv Se.9=ˑ\Ju_VҮ}\ᡰճ}LK~~ث&Oav%M;WgE4wR-]F61ΟS[֬_vZ[HUYx 9&@>w.ìbPu衑9ItW3䭤dTfGA7fӠ>}OLl"HPBoGD^D={c^wG+0v8 |>2 kwQ*ءGŔoQ׻>֞fGPH͖a{;x]P`-Ǚnwܑ|}|OjGeP;4OCBt[HHV4i`g[RŠ,ukq('"I.tW|/55U7mҋＣ={*S'8QK׬Q|B")!!Q+uiZմ~7$YQ)y9avH*)Mc{oҥu…kw<=s^D7OSNk6mR/W R5jP/l2zw 9n}܇j@Ϟ=4TdXqcJÆNfk^>2k&)9QW2;FU7j͊ٵKӉӧu5!AI)bTD PH=,LUNꊨZ/ t{ ޝ:ɱcf.XWG08qN=EZr^ . =f~٥P%K}й:viʙ@A 9Sg7mrcgX4gO S.]Һ8}ZgΟW1__/[VUfMY,#^{u͛k%i2'$ej֠AayZxN-(۲sךx.]uiiJvMWu9=t(k|UfM55__4pxqVZbQ֭s͞zutv端t5>>Gn͚ak~߯Yeb\.\qqJHL4;lHLHŋqx1N.Q,eKeKtKq0'|,v~=Z=>&FcbԩVժg=:tY5U\ C/G\۵MRIqɒB[,QjTV|BV[+Wjʕ:pPuF:VvMUx1#((4GKOO56nt:ֶmn m>luUmo|>wNZ/גիu1.@_";TվcK/kv$R@'Ђ_5;Kklq:kٸI;֕3(==]ńd ԃ{޽X[L?/_-;w@ 5Ţz jmdsu^Mp+"j|:䲴_ذNMF m ?/^TJ6M kVڵ:sEGJHH0;"P`sڣ";R@#((4,Jk6~*R$I׮)>1Q^?NҾÇeڿm'6:{Vs8|# VڵKEXzr7N3} qvjGl8qL,BB**vMժ z{eZ͎X ETRros)lYU=jT;v4W4]-@\}7m߮G4jUhpB%2R\͛u!:tHIIIn_Æ oۭ^zI7nt Wxxl$~nb)Qt1$u꧑v8Wo ?T J܁GTn`n͚Ng(JK.:9ڳgΟ?k4͛7wܹsȀ$)11QzrKbA RZ|DTKBy "gĉ.ˬ۬YT*I]K+h X{VBCCUVӚ1cx)S&.YD;wիWݘEYBBzŋxҥKЌ3t)mڴI]vUhh(pb)HK:iї_*<4T: 3Bm鰔|qQ˥e***JQQQJJJիh"-\P;w^Ѻ{?;(HNN}ݗRi͚5եKuYZk@@PL>_WRiKcXRâﵔ;yd[۷W5i$;wN+VҥKO?ԩS`uQϗQ$$$rL2֭z6mڨlٲy܏_RR4c|׀= L5(Q,5VXŊNg'^``'ȑ#Z|{9թSuׯ J"99Y˴TZV-9RK.ɓ'5uTEEEQ*P`qb)Q>J?pްA._v:CƅɆ+׽<=U4E[)+WhR۶mնm[M8Q.\Eiҥ{ Իwo͙3G#Ku=h޼yf%JPdd:wΝ;+,,̄w(9)UJ};w60M$%'+554p9``,$$D={1]vMk׮Q4ݶm$i…ݻ~Jԩ.]sjѢ<==MLyb)NgMחGU'JRz`OOOnZ[uy-_\K.?;j3;*򡄄K;w?HkNfG俟Gbw:kZI.>1̗Gz.]W0O@@oڼyZnmv4C6mҋ/͛# T8sn͚ɺeEoq9w^!P˖-͎|fSNK&뼽Β LIJr9w^KEBf)/UAIxbNg II&{PPP, I0k,>!$\?{PPP, .OooY䈼ˉ0X (|}|\/_bPsvfRr NSŹ:RPx`d7mjv Iҗ'Gyzeb(==KQ1(H{t8)U 38QuIAA&KJOO͎!Iҥu?kWRId 4jUT׮` Ĥ$o\ ɓW(?rD6nt8  ՝u o_;P񑏷1LP {t CnٯyÆ P-{jNgϘ',,KbL{PPQ,Pl6ݱcJbzu`v/<4TZKNUVd#}կ[77Z=cIΈ*UԾY3vi}LL)XN ??+5Ijlbz{"<Tt9N,uټ>_abБw劮]%JLR*_ԫwީVw)??C2s -[?OSdGT׮ԩ|=/// +23g3g%KE<SQ,Ph/͎B"8(H48*(toNᡡ:tXټe˴-6VuktGXŊs:yRSS*jcX(o//KIMՄO>q:r wG(PXV=?dpɚc(c;1m9~gdjWfp"cXopT*9^H[v48Qᑔ7s-^ib)^1,==]_yE*&Nǎ9׽QDơX @5gOd_u=D'41ct ,4rxtw0R GSV.38Q.&F68-SFǎ58y(PX,M{٭(^\ןyƤd_QݺjwU[>R eߗ&Ij{]ھpd_ڵe<=裲Z Sǖ-MN`,iѨ&%ժ$bvKo^| :bّ G1C3;hpTn 5; )?0pEdiNΞ?)~4 #P,oؾ=o n7$-7d^6~:yrwbȑjߧJ-Sȍrs͋kn)6UFIDAT}KR* T, ""n)[UR;7^6߼9RTo&F=="/J6n/Ιk֭UAAߩS).kнʶmE˖¥Kݍc֭QV^-ZD8c@ѪN떬_/ΙKJbMѮMܡC 80zvޠ>mK;|0.XPؖ-ZCS?> jY]g]vǽ?İ8w>w=`<3{vl߱1tF}[ѥֽ~h[yyzxٳ c׮]5͍F_pCvSg̈;?^~jݳg\4zt;hѼy.۶-㡙3ckYY=|xL81zYY屭46=={@C,hD z+z%…_m[Vcnju4h/tTY=GqFB4%Ķ:ͫ!T3 c*TQ+׬/}v<5ۼukC%+V#O>Y= |?3־NFwϞqGDHRy˖Xzu1o∨{e~R*OϞ'~Uyf1ؿSZV-Veqǟ~ɩSJRѷ8Gvb׮]QZVWyŦ-[b!T9w_*圜ܐ!qX~vdxbi㶖Ƙ/7<:wP֮U+׬'.R|/14.̸kQQq#FT;ouIIL936N3iR>zΆ qWdJ۴n?8ӣMin?lL㎸z:qGo-ZT"̟+ҕ+W^1/8G93x".xwݵw6l+~M7UC-,}k|TOqq즢sǎѡmۤ K~~4RF_Q'?1T+V[U_KJj] ={V9gȀ1k?iR8eJڳn9Ԍ1dHlq#RT)ՆJ?g~,^,0kڎ;w'Nlv4xGj#[v{Ǐ"Ow׶Sg̈^X^;C3gFiYY5lX|?iZht|?h I7r3 j:{;vվ}1Cjƭ'Ffj>=6m =ɔiӢ"vJj5ѣe˴3g8Qb=_?~|5޴ L=n]v%M{ٌڸsϭ݊NHmO'ꭶg8k8k=UAA ?д_Azl'X N>ح͜_Xn]B]{wƘ`AZ>}Oqq:- /ԻZ]R/N%TtIy;Ć?QoitM]%_zjԴÏ?z8⢳Ί!$a?ϸAuZysϯZu`Ҍڬ矏a_JYjUFMCv > Kɍ&Ē+ѧNmw>@1o߸3SOm6jlؐQ+>zu`׮T}w[m3YAn}Od=!4ϏG+~tѲE*] .яGƷ.^{2jmۯ^kF֭jX=lݺl!X ЈRT\;n\,~꩸ꢋc5V/-+/zw\8eJ+?3ܹs?6 ]}/&^~yL93&{ok㶕Ibҥ1jO۷Ϩi˖ZV]o6vwW}KԢy8oԨxiƌq 'T9i׿]rrrNUKvZfM U:Tq[+W}`_!X %>{aߦOze@~}iqX~U92jgUkƣO?&;VSN[N[fyu޼y1;22{۔믏fi%%q׾No,ZTK/~8^}hٲ?o)-o~^@n)xW//y{1oѥ0 Z[ƲߎΆ ksiѲrǝ&vܹ$:S\}{ظiS,Z,VYc*1yy#뽿ys\:qb4)w[bѲeGƎӟUCԿ8aB|gĴzĭSƭSFvѮMh[JKߏ7,:(۶?5 QRZz35A_y_IWɩyƤۨ]Gvq@늊k==bUWU>}g:5¥KcҥU{W7bD<2eJ|mܸZZl[yyXEN>x["77\zѼYzw/׸FEEE_$ PjԾ(ڵt{E*7t u<.$[}&ҺGߦO6[8k3Ox䮻:qN^*rKwql~^^|#7[vܠc}7~x䥪kQQ\r9_6~8nĈ?ÇZh_:꨸?SjĔ믏͛կ;6{ƖtM/6Zo,Z֮6m>Tnnn*:k9ѲE:cN9%ƜrJX:fϙl7mfѾm۫W =Z5{xqΜX]R}7 Z;w>1_ɩsڵdIINNNqqƉ'xu޼X|y߰!ˣyf_VQرc?]\\jD<2ޜ5+󟣴,><>7d^,7 4ݺEn>vvڠko6|1 fckUPG G `k-.=[tdR"BJD`)KR* T, "K$X @DPI,`)!X @%R"BJD`)KR* T, "K$X @DP)/j'Tn*6[O:syd K~Ú[#Gvqyܤ_rs68<};'BZ4om/6vs#8m4R k92ڵitˋ1yyIpydl<ڵkWMdɓ'GEEEz4cܸ~5rrr+hZKJnuh.k*628<[OUP1nܸq %J~^^-6<.&A,`)!X @%R"BJD`)KR* T, "K$X @DP)/>n>=KR* TKlłdh"#'''6pc)@[hr n$X  J&gIT,=zСCn%///bРAѫW%@[wIENDB`distributed-1.14.3/docs/source/images/network.svg000066400000000000000000001106441301211143400220250ustar00rootroot00000000000000 image/svg+xml = f( , ) = g( ) Worker: 192.168.0.1 = p( , ) = q( ) Worker: 192.168.0.2 192.168.0.1 192.168.0.2 192.168.0.3 192.168.0.4 Center: 192.168.0.0 work: store: work: store: scheduler clientlibrary distributed-1.14.3/docs/source/images/scheduler-detailed.png000066400000000000000000007104621301211143400240540ustar00rootroot00000000000000PNG  IHDR m}TbKGD IDATxwx? a! {d(Q=:Z[mm}{O CL#̀A"w/!~u}wBeee(]~}~~~YYYKIIi֬Ynڴi *++W^vں^ NիG۶mիW'[77ׯOܹsg]ovؑ\^^_UT2 [bsyyybmԳ,@$1kgg)aYPz@ Գ,@gB= YPz@ Գ,@gB= YPz@ Գ,@gB= YPz@ Գ,@gB= YPz@ Գ,@gB= YPz@ Գ,@gB= YPz@ Գ,@gB= YPz@ Գ,@gB= YPz@ Գ,@gB= YPz@ Գ,@gB= YPz@ Գ,@gB= YPz@ Գ,@gAmصk׊z ꅢ畕" ֭[7nP[@%JJJ-[aQFOlf]Fm۽{U+3/s23׬Y]QQ9'M6u]s=defff.ܶmM7|]Գ 6VX+/geef.^zވ:SRRrS3Z|Yiii]oTM6&yPG= bcڴwuG]oAk׮[oM]o0<ԓ/B]oAIz@ Գ,@$/;vXp%YK,ٰqÖl-/...))ILLlGmkӮ]uҵO}۷_˖-;B=Pee9yiӦ.X0⇒eeePhMjBBB߾F5jcЩs`M=k#=׭e˖.[' 8Ng :I?^=}1fA0dΝAz>8ϣ>\\\\ '.[_Gο]EEEp"^noٺutʔOk]vu7?jz>$3s 'ys]-x?yb9YݬP(4~]u6gV\q'^xq.I=+ܹ߾}{ݮѮ]N>nwڔ\ 믻v͚UT^ԓ;vty3 \l2i777?^͆[Գmee˖p5q8}g5̳&Ϝ9{vڵ竇vx>}1ĺ^_3gv4̛]nޒ<ǟxr-^zB /,9Y,3:wӛ4irW.\#~Əc;?/7[vXN^xf͚Frr_p<=+n͜9(b쩧mӦM-5cg3ƍ3fl-, 5IέM`ߤVZ1sQcja7%bÆ3M4ԹsS'v=eKZrUy۷BM6mڴIӦ[ޣG=ѻw_[7xkW\M~~~aaaF4iҬiN;ޣcNIIIu&D}ԳS~~~yyyL-kgZ+/sʧfSƊ^nG=j;|!mow\jUW^jO#??qiiiM4iӶmݺv떑砃nڴi[UCYYٔ)N?g̘`wq#F8juԘ[hǎ3`UVZr֭eeeiiiiiim۶ܹs=>1.vݺ=gj֬Ylݻu{ @sAA_~1w ,^hժؾ} vA(O?cƌuM7%|q;vǃ{cA wN<䈱ꫮ3a±oz5ߪW&[WO䓏OO9ﾧO5VU_}vTշGΝ/蒋/}۶mءm6mTgwޤ^|!LছnVcr4\XX8ciS5mԅ ޽;ÓݾI&1 ǦN*$$$DZzqqka@YǞ~u9*&V\Ĥ .]r% _]N>cce BoƈCҘ̌ʕ+8ѣ~wk B6nd5W}֘lH-hX977=I'?믿m7+ +Y)g̘1=M ?`ȴiS;bРmFx R^{gqi[R:d%%%w}縱G|TTT_F ;ovrqq>I煅4n\x}衇,Y)&}W ;wCF,^(#c'_4Xiy{?-ÇSYqW3}4믿eb+//o#_zZ8klCgBqs4+/4o999Nt \CG5꿸Q!5thb]zqIIIЎ;?/JII9#j᠘عsir劘Ooƍ=/|͝p̸͛c;veGyիb;%ns@ ;Yq#&hK.2}b'~rȦMY1m۶SO9)o+,,<2c83[l4m{tnnn}N^z \ ƧFLҋ)))>DJJJ-,V\}9;yQGѧgϞ-[jڴiBB®]vڹs5W/]t%_ՊlCyV\~ɴsW_}e>11Ïpq0o-[JKK+,,ܶ-wŊ?/\ i3fLz]| 6nǛ5kּy&M5F#{# Գ~wɧkф{k<]v zj?晧 '%%5 /ÏHHH@֭[n rJ=a)wcǎߓثW~g蝑ѫWN:kZZZrrrqq]6o޴.;믿1cܹs***N#FX=/v4ċ.oѣԼy͛s„cu^4cx=nݺWu5WUI{z9G3ӧo6mjqqqvE Μ9O>^fu ףv}(==Yf?kڴiӦM۷o?|~tz(y~;wF<[~OjmVVMrȐ?X?[_~I^^^pqq 7\+G3ګ>t/k~ֽ{4nܸ_?󬉡P/xy k'FNn펑#G~'%%5nܸE]v:tؿ~Νfɇ~D@] =;>|~~Map[UW]M%--}w[ΝoXG>t/oαcDžfW:t{.Loʋ/$bˮ3VNMթsɷzsٲU뮿6ر-^?5L7?t=썷r[-&^n/Vb~qǼ Sϊs_Ҫr :r+++ZJ|O>8bl?ZXNt'OsO9?C1_,zwy{Xzz~#ja:'jܸqXAAzG!--w?Ι?W]ѽ[k+Wd:SڴiS86h\iӦVcbK/O_u>3OJKK4wg7nȣ[qshM4y/䲚ڹs>0h`UVV|yODZjSnj>jj3q9͛7 1l*">pkTif9dd_̍~gӦ޽;|ēNСC3ip>pg틺vֻ ̜[n)w#f19!ׯķ:]J3?hr@zzGUQ:C-BlʔO#f~:}RwpI/z8`xf~wvK^|Oi-[ Rq_kV>L86%%J34lk֬@xy5j4f(@}Rڧ%&&5_|=N<܇[O9z =lѢ3zXvn>u˖deeό5:UI`eeekkX<|fԨM4v94ht;#|%KWxo:wӦM1b;oUgeeό> ;wo`AG?J#frrrYQQ>3x֢ @ٳ]w߳jU?x9iii5w晧sE"fTcN۷Wjʈ~͛G|uk4">3) 80Q4\qshВz?;w<ܹs=;nۼiӃ=׭_1ӯ_WŽ6 (,,~#f:w?1M7֯|yPu>Рyz?(==/1s̥ڶm[QO!R"!!K^)>4Ej[۷o&iӦ7M=6{ /|Qc7Gzoڴ1|UV)))[&nGgD?mȭUݷiԨQVbWg4,3͙i?:=1ʗ/ ر#|M?+nD,UTTD?f.ʶYć5o<P}ASϢʆ+/\z 'Veee]zݻ UpZ IDAT4k>РgQM}z_~CǎjE>p4ɈFEnJNNᴈ?:e/b=+K!qshԳ~KO9?~=P+^<˧4|I& u(n3 5o\w Qsrr^|NJX7nM,7RVVu>Рg 7وhEHͣE|8V|_^^ @\ ?K-"&g͚yӦ:Dgm~Dg>x_|];WIII"~#;v:g7M=IHHx :p۶mᓕӧ6sdڴm~HnnneeeBBB~[$'7_ZغeK< yXZRR qshмܐXj׮ݍ7Mr֬:{V(ڽ{wnF:Fo۶Akeee[neb+Q~6X>Рgc\rY#Ɩ, =-_ZD!ϼryCw .]R \jjjL^^^-lR}AS"Ҏ?ĈkzqHĎUҽ[{D,^6MF|0 sqshԳѣټiSyyy!C"3gv"h~g^ a"f"^~JMMtu2\g4,boA3a}7n~Hm[OII 1cz,S{gj*|f/[mڴ ^t4hY^֭g%%% H6n\`~6#Ԉպ%K֮]S;ā>7}gLlu1|1>pg{M4&,|`qQDtƌ1&54&s&1׫WEVVV2\g.,b XM}L!O??QEƎ1?ON9Ԉ7^m֭Ll.\P;sqshԳM6Ek֬YȑZl>e˖_~1͈ÏիW}OedׯLQQџt_C"/ Mi-lR}R"2#fڶm>rgFuEa5i(;u7WTT>51F@g{S>$b&#O4=6{L#&En5ef.~䑇ja5j>SRRr٥7jNt>`>5>_qshԳMF3mQ gӢHDƍ֭{ͿqŊS}m۶gE͜9X qGGMj.%9Srd~g(,b쮻YDcƌr71SQQ13-[LHJJ_Gr~m-TmMc}Z'VN<񤈙7x}ڴL ,ʪx4DYg打'sD%&&;.ʙ?:hN0~ѢQ%λcNcW:v[Jӷo .(ϮzX7􈱋.:o ~iݺuիmV#>gŭgѯo?}]-s93M6QMHH&yG+/G9ҨQ{~ի.\paΝJvmwDd Bꗗ\|a-7I&&1aO:7"˟ٚOG=+nM-[rrrn;_q_~Epǽ cǎh—\rYs̄I4ɂI>S׬Y]#sLi/^{ԑ]˟]v=W_uEGo/[gyjA}4+JHH79sZXz1~SS>gŭ^x\PPO:aC[.;-\䓎?Ir‰'U?}g^xϊi^SPP]v]tTZZZ Gk֬Y4|oW_u_U7o3O~;Es=SN2v횓O:G[W_P_Fo~gM<;dvqc<眳:譪aȐ3rb q}aIOs3V!F;wڿC =㎙pԤIje˖ɓ?x?lZ>OƏ?'Κ5ǖUS]u?QG2[[^|˗/[tɒ}{2oAG< tѣ*->SZVڿguFU?աcǣsA 2Gbioْf͚UV~WsYb2k֬ܥK5K^^ᇍjN<:={rOeew}y+WdefΟ?/++<9ѿ^;h`*u{r>}v-===)){m6oZ6{Ιr{ V{BЈC/^1֢EK/‹.իwjN~ԩSc^oJHHPϊO=EM211Wރ ѧCǎ;tlߡC5jԸQFJKwos6o^vɒy_|j,s{ɧ=xⱫBM6m֬YZZĂ]v)/yĈ㲞 nޚLHJJjݺu㴴F***vE+z9fBMN쵇:oVZ5m,qZeeeyyyqIqQaΝ;+**1P(?[~S ))){JKKPYUc]v=ЁqGYz@\P+~ax+Wh۷ ۶mބ,+_~糪= ¹_Ԇ_Y> yQUg}/ohۼiӴiSzԹLnڴi pÍ1Y)JYKjzV'G/]n .]pfJLL|׺vV׋H'|J5>8pj|*n3 zVzt㏧t=&ӮM(iTBoZ>wIq}1C=,cP>ࣶm"5r=kԨQU?5`7O=+|o62r/Ùg5q}zZ8+5uz=k ,ٱS)vM7'&/}͘9;#O]/R}UxT @='vŕWՓ%%%]sZc>|Р_}[n 8w^byG4|EY7ts=vhΝ6nؤ;9*DϞf̜}i?EK/GjR }>SϊC\rٲe~a5F=s֜{}5YFv,x9III VXqw 3=ٳWqIq@G8鳞y~ti׮]z[~7^|;vrD 1;|YZn]>P?gŧs~2/͞套^ުUsN;1o>7w?r۬5:b[gꡄO?t?WT/3V!w1cGW^)m۶=/CtDD%%%?_9czNNN5&2>#3v?W]&0z>YX|ATZV>PCӧMٲeK++y\˖->Ï8rq#FhSlݺxΝ_Q ]v:lСÆ0|ā0˧NK/0mVi=z4xC6lAAu֬<Իﾽ}暟/| M:e?&&?'$$gsr6o1c̙3ϟ&Ӛ4i2bāGz'9N*5?d͚sf^|ٚk֮Y7%%%5jԨyzz.]3wFzHNNn9eg[~;w'%%6jԨE_6m{=tذ={<3s_]b벳}$)))999%%q͚5oѢE-ڶkױC;vڭO߾{g4iҤog.{]v7'&&&'''''6i4yzzzzVڵۿ};tޭ{={ .3g%KVZm[]JKK]tI} PY5kVg.^6{ 6lܰqÆ⢢==MԦ͚mӶ]v߾g^37`@e&YY5޽3zΨE գW,@gB= YPz@ Գ,@gB= YPz@ Գ,@gB= YPz@ ԳQVVlٲKZ\ @ðvڬ̬˗'&&ڵ[Y}~o5jT'c*ϰྡ(⾗ijy},e43Դr)Ӭ\2+ AQ\asa o߾o߾~~~bXMJJMLLLLLLJJxYAAD"ݽM6]v޽R^^^^^.[XXXX褤GYZZ6lݽcǎ 6lXFt rJJJ"""Ο?Ք KKK{{{-Zm۶sݺuܹ3`j yW^rJFFFnnnnncÆ ]]]۷oѩSvW2(v9iii)))yyyEEE 4pvv~^xOOOoo:[{GøHuӧO˾H}sc\e0a<**J~߿`tt(*++[xq@@YUTjH^i,09666>>>¸YN%^ӳTPǏo>Տ%tooK.iӉt[ZTQQ1k, A~wcǎ4(00P222FiȷQY'OOty-R2?~8 ..NõiӦ}Laܞ;rNyXHR?Ga\ ʳ jur9aD_GFF T7W-v_ɓ'cƌվ+yzz갷7xѣ5kΝ[c +ԕ?iҤ -QnCHTPP0z'NZ^b_hэ79V"1}Laܞ?nܸmޭ[7mz#0.҆蜿'|"LMMMKKkѢ*GDD8U5a ǪTTT̜9S{9eʔΝ;YYY߻wرc!!!sssLrWWWj׮w߾}thggW^^ٳǏ'''EEEpe˖{ｷtL}lݺUvoåK߿_0==}ǎS=sY¸ѣE"QIIfGv 6s/Taʕ111¸˗,Ybnn^tuuuuuׯ߇~v5kH$O>z9c<:uʕ+;vXc?ކMLz0<ß/ :::nڴiAKKK[[[mV-/ 8PswSoٲ 7jjժU3f03S4\S$%%%OU/u.]f Va\%7Sdff6ba\aB!!!mPW]]]{Y] 7{JUruuݽ{ofii)zԩ{6tUWsNg"hĉ}*=\~}ƍ¸sHH/DGGϱttHYo^jM͚5 ݷo*Y]t𧟟q}AVO>~m}%.}La~˗+4k֬7oΜ9SM\VrDU瑼BGy#G gϞUyu߃Dyyy.]Rs .k|a/{!kʔ)vR+JKKko Æ ۳g*oH$ZlK666'O̙3?#a<&&F1&?Vw!<<\e:kz~=o޼)U5|pR@m¸}ʕ[o?؃1b\=ʳD1BXJs\Jd>}ZTX7V)>>}||֯_+M:߿{n1Çk1 8x3K]rE᛿z^zɲe<<<]ڵkG۴ic_F9 ]vUphwSA6lؠ !H^!Eڣ< LTӦM~Zʪ7nddd(AY̔|Iz۶mRT.haaeu| omۦV?və3g===9ҠAٲe0رcǷ~[dya\ain͛7ן,Qr`QQG@]¸}ǎ™zmmmon,Fa\=ʳt3FT<}.^3ܽ{799Y.ոqcWTTo+Wmmm?a<..͛fӦM.ҭ[Ǐ;::OiiC%KjƌEaRSSW?S߿ X]XP{c vTga|ƌڵS71Wq(ӥ<+44T܆`ٗr9”wr)apر-ϙ3G3uT{{{aȑ#uW'O...{xx8qYΞ='6mY5lذgϞK~S_z~=܄Խ{{\Td)/__,m0Wq()SxNNNllVᲑ?^n7.!Sp * nzРAT L8Q ѬC?~|AA\M6AAA͚5S VЫF}4Ν;' :;;O<Q=u={T, `wS+Fv4AckĸH{g gRv e#ÇMyyVEE𱍻N._, 0@Uu.""b„ "[ly-Zۡ/0@\\\AᲕ&Hmiiid_:} 'N?pa)Я_? #yU0.Y`g).J0h ;;;[Ҫ$&&&''G.8n8%ǽy0驤I6˻w6PxxOt\]]O:AotI~"QÆ Gi֛! 7|&Uϯ#/]H$͛7{xx|GeB¸ƍ:A]UǸHKgI+Dϟ˫ɱcd_vfr{Vɉ'A%Y¸ٶTQ]s)*++0ap%&Mo^>3223fXiAnTУG'cDz0lr喗W_uiԨQ+..4b)%ɃΝ;kZɫq(fkk;rH`YYYu$$$F^LɈ#v>~xuVn9;;2?~\QQ!;;;WD666666ÇVW$H$Z~ϖ#׃O޼yk֬QV"={vΜ9K,u/¸eeexFA-ŸHg0n_^H^c4`+)S:-9}tYYldĉ/'MFl" ;ѣmlldhkk0*>}* j~;w_hZ+W,,,\z}VNN2Egݺuk]u'սD!ì\#2ޞ~V֬Y|oݺʕ+JvH$V߽{3&4)γTH^KBy&L*-- :t?lD(Id_dffVEzU9yp_~YyM6533~:;;[yC劋ΖM5s̮]N81##CuڵVVV+VP꾀{yyyiemӤIX,I+J |~z z积͜9s̙u(Yc?puuꫯ ! ;& 0"ǒ4uwܹ~z'NЭ[^xA6bff6~x~2<|#GT0~- y¸6\^zpիkQ*99Yj!++&M㉉ΤaÆg st_0`ݻx ++vۼysLL!a}Lanccpea$< jWU?=yDvY>i8887Xƽ髢u|͍7"J?3g۴¸]-^ySF򖖖 tqQu(aܸqvvvrW_mU7aÆ9::Fn޼Tɓ'L6M + FEEҶ: tĉC*o={V~~wԩSVu]xPXTO{=W]6m۷[XX={`S>0nWxp69hΌ4h leJL %sΥKD"D"[ݽgϞ~ƌ#@?JC ݻJsC =514hpSYYU꼏0#X{ 8P1M"]v`vv>^5gΜJ/-- 3|>@)ⓓ5A{uc$<(PdQƬY >|(饗G8VeUVAApe3f(ҳ0R r@= Gz?f9D w50.ByÇwss ۷LX}TgYf͚)`ƍ̤[nѣG st_%e\wS 0nׄ҅ J$rЭZ=o߾p7nB*Qٳ%Kȭ/+gr;w;wN)ѭ[' ~:t/[h1w\R2Ν;ڵL׭[wa%mg̘ѢE a|߾}6lY&oƌ͚5ׯ_o[YYfi;V,//4P]ƽk 99Yl޼3}La>mڴ6mQQQImmm[l)>B*QI.]3dddEWr "7}Qj̙fzj+++a믿OT#Gf͚y̪UMɈƎg)ܴ`{UO?Uiٲe/.--Mfҥ¸T*}UȔK/]~]4|}}>;nXjM{6mWP eq(ZfwիWǎcffVc ׄ 6m^r"Qǎ?c֮];d%%2YYY{>>|y?pڴixNNάY˫k8g___;OO۷CVVݻǍ鈉{7:w,͝;^z扉nΝOropӖ-[Fqx=5j/b&Lstt8p3C}La>}=ڹs7Ojju4]UZ:}7o~WWWUN`mm)ۢ!C 7}ᇟyu Yڵ]iigϲSSS.^xڵʩT6Dz{5[RRpň#tҴiSss󬬬̸۷o+ʕ+]v1=zT7X,ٳ#:vظqcGGGX, SRRnܸ&";;޾DF?~jAkkCѣ}...U{޽{7&&&00000P "sپ}* 1]$=z;-->`:ٕ={ӧo߾uVddd z;vRF򅅅ZRxv&M999I͛7Ϝ9sET:`U"$K3,)gѢE?MwqssS>}TmWXh ''ƍ5V%WPP 777 ŋnj(՟TyH$ڵkWiF,HB?jQ>IdATN&MneMEEE ;rppkٲƩ };ntՑ#Gfggkyt//HH^$[_f~ڔT< YpauUDfͪnӂ 4D5 ٳ=Tiܸq``xLM6l WTT̛7OɊ- `Iuҏ6l׮]kDF_|YI;={V3HH_g3={ R1IHHвjH~۷@%%%.]0.YPtmȐ! 7͞=[~f̘aaa!͛7OiӦΝ>}6+**_~Z&c ^{ &+44h9,HhѢ}>Ӓ>;֭333#^?O?Yƺ¸gϞ.\8*Ͽ6=T֍䭬ٱΟ?/q(ZiѢE ĉUuȑ̙35S,;;]v|CAVlӦM69:: [lz-Z޶m[͵IѣG`\SL>|m}||tMŋ8q]4f͛7\$/88x̙Z?x1q۩Smq'n&%jH{͡uEٗGyJƍkݺ\^Qs oƉ M4)!!a=zPe͛GIII˖-a&F_-JJڊ^{-11q۶mTk?Xܻw˗ߺuYfjmJڴis'N2K/t̙P0lذׯoذ]v4޽ܹsl|`Fرcرjfee5zGDDx{{Pcq{-oݻZ {_hBq$ߥKؕ+W6iDcoÆ ;v쐍ʬħO}-y6L/Э[%I \\\<<<<== cj/޾};55ɓ'7رcNtݸqcc{N OHHw^^^^YYcF\]]Pwll\'W\ڵ9Ą^tΝ;b¢prrrqqqssk۶mtҹsgkkkO[c]􋊊Ο?STTTVVfnnnooߠA'':tڵ{U1)ܹzſ;55577y:tСCnۨQ#e[={6444..***Ŏnnn^^^vssSҏ)\&+$K3}رcǎۨQ#ç (@-ygeeʳkM+W-) O*]n,96իsss%dff8p־FԩBgNH$5NSljlll \+@vΝ{mҨ67g/xYa b>6X,޼yڵkkܹ̙3gΜ6m:xcǎ3BY#v#J_yݻpW+33s{լss=7nB*ee.=ٳ@mVuV^-TЪU9yϞ=nj3`333P:ԩS/_6vݹs5k p8+@ٳCݻ744Bjt ##0$d˞=w+-+ӸH4ϯ|Ï?0=rrdf sse 6zggC۰"66666vݺu 4С Ǝ榇4(BbRRR޾}{ P[:ujϞ=Ǐ/**Ҿ×ƌy|*' <{60$$) IoP ..7433իߐ!C,,(%=SZ~Qx&N*?~޽ד'Otm~XF,C۟}&}ӦIJ޿ }OAF٨[tKU%JGܽ{={ѿۯ޲>:j7`\yVԕ+g]yPfY{<}ɩƆd_ n犊|O? kw߾=~n[Zن{M,JkSV]_dofIn~+VyxK??tO>Y0cFMDdgMnTQQq#1QImVZ7 rPk )Rk̘1͙gN#g8?믿_cw||l$*6ӧP8Oի +Wd_zvpς™K OYg"#M~t۶5*P̬gNj5SQQ1"cbdܹvmu˛OO=o^bJG)+/_rìSÞxD"QIi>8pZU_W~AEEEkA_uK";#MՔZFٳ' F,TEEEHHȾ}>õuw6Vzxݺ!ض;/H^y]aKcƼ5{޽~=|Ӂo[mʕ\7V=I} "u넵YƎӧK"Ya;w.]z:2s>> ;EWQ>Ӻ$` ƍwv!.X[[= BDy _ڵkBB͛7oݺ@&bmmЉɖgD+iT0ܥkJJK4<|QN;}w;pď.VnY̘?kzR;kBVr/i>JQB :ylD" R ~+**"66m߫Woݺsld66k r)ѽ qqmhaZgjZ4{v+U(0(s1\rÆ kذ;cx}|lKJ"gΟ_8sOWlme5gʔ;'D" paȑV.F &MX6s$N̞UGUװaC??CuYA1ʳ`4ɳgϾ|Nz7nd!&;[[VE]XQQann.LM>}& dۙHY_Xk">*ui^^U$ T*6=;}bi,}M*cC/u6}YPX4/IA8R'?ڨGWf*,*r%٣gMKSĔI צMg%;[_S<^Lrr 43fL,o7cǎ~a~~:Ξ='[}ڀ޽vK?1%___//;[¢HZN??QF$Ezw)6 +ٳG뻊BɰAAY֯"MWDkR^~Kyn߹#UkR'?ZMvJϟ 9Z sBINN>|xʵn#F1bвOo6kџ}zaBi ϝ}0p߸}[XpUzK;[['\$=3SN۱cY"hʕ/zi6\BӘ܇kkcPZvhWKLZ}7:yr3.LHN`Aӧݽ{wԨQf/Xwݷo2.ʳ`P^^^aڵVõiS<_PUxHHn..;uDsLfJi hj~C0–AP QAU Vlk*kk[mkVjmqU[8QGd! HH/II)$WCs*2#@$p|GQ~ɓ,7<c75)C4*ʽ4f[5>ff c~'N|Ǎk＃ O<}(͛pu?%cccǎYXX(O%)e2 ^^<ڝ;>'?>Xy֙+LLVՕ&s}Uxy킈6>_ȓ2PP-Zj14%bٲ+G<8G!!}y7;kŠj'CzzzJ7nܚ5kRRRjkkϟ?raÆ>C,tt IOO_$i#G/;;cg}'9w;6ohim؜=cF;;0wʕk߬ohzvYeMMf}CڠBE0̺ՃS]q'N˫CB.Pg{03+~cҲnrM_#'Uf $On״!HF7bDIJe/o޼p+kj?p7Uh=yǧ GGGiӦaB͂@X, 6`t۴i| [3g o޺{j/\ֱk“ n( IDAT<]Vk\nǦߔ)%0_?Ͷ=)($RQc~O?WXKuݼ+ry GhB5 xmmү(f[5,ݼƍǗ.~vv~WU;5 tss۵kW^^^AAΝ;gΜ,,PT77xٚS}řׯ_LL L;Mz_x:>*) ᙊ7tuq8E#̘ʆA:T$QBCSS5:ΎFDQ"-E&* hnff*>?qfg+Þn9[o׋/cVw9r{GԝYx\.wݺu%%%ᑙLDiii]]^##SN* Wղ&';dri"m̘9w AMM\>Ld.cNJDξΥT ~YÎY\`Z]]E"^UJ~*4q8r2ЬwV |ABEƌCR՝(ʳ@Lז-[dhN&ׯ_1QDVV1ѣ@]͞-R 4ZAL03i8? @x 8wJSssG4:O8QxD \zB?.9}͕!xΜ/ x?2&t"S.U=TC{=_v+}W"/뿲 D,[&<>{E"qq"q> z޲ewU=A+s{lv}##\bEMmAӦ 8P8|r"1QY*!C#`ΝCzhk Y .n:__ߒz W g:u^\z /wAGZ܎yKȡCG *A.X e?񅽭paE_nl9JJr93])*IU_~UT,3>/r΁-[R%O>aX!d5s8^]vg=Dڿy U\+g~fVVg ymm#~Q|[5MC:d2CBBekN&####""tu;ݪ ]vʘ(@C"f-LԠӻh 5ݢ)h4gbbblli4Z\\N, >Ev"s7l.2A (.2fΌܺMhMv g262>/w?ڵ"9pZZ._N|!slsOJtWׯW_~ Y 5Kzx?'/^\9rf vMT?Ϟmv~wwf[5ͅ X, 6`6 OOϟAt6VVstfƤIe/a\oiw~{7o%PeV-Y=qpڵ# #HQ_}%"B_OOeR//߷y O*;?_b\-BytOjj[||lͩT}̤9z2}xHd`_{ CL]!/w΅Ο#G|7+Wp.U"HmD}y:\&sJ<~EhhƩSStDuu-2fds<:~A` DUnaag4ݢ)h",nr6lضmjRh7Y|y>}d;].w9#CNaؠABSx)lE031nʤIOps;x#=^Ҭxc>}{+$ }g9>o@ZYXXuk-B-Z"mmm՛hK›:BH$̞Ζ4K4h4Z\\NWlVp$ޡɥz3SS3SSug`6}=a{hk =ʳ+,+,,,>>^ Ftt(N֜JFEE+6.A"*;5h4,>QYQYߦcr6lضm[[_zxxh4 $I AH*K `< ӿ;qL&3$$$==]d29222""BWWտF888<,xILLLxx8͖9FJzt:R] ՝8q,d2YI=wl2DIs*)D7V mffKOOoUUU,HB@݉VAyhggg+;}*AJRSSCCCe^JFEE+6+ِH$[[[[[[u'ʳ@Kx{{fdd(&(bOaff`0TV~\.wÆ ۶mkkkXм4ʳ@K2 >|XZZڱL>]gQ%c2!!!5'ɑ @>%^111l6[4-..N+6+,ވb633S`VZYNjjjhhhIIlͩTjTTTppb>:NT[W,LfHgL&3$$$==]d29222""BW1Hzp6-[sGvCycXaaa2`0@o,mZRR"[s*جz u'Jr׭[+smGff&jdٳ IOO9LIimfekNtb腰!Ac#IsS6,`\`0P= TEʫW.|]]]?ngWyi'OZZ8N{t +\%%%5RQQQ 7Cyh'Ojy^g \bSju10V mff@ʳgWusI3UH"N˵A77u#?%=#}}} K s'1::dug EE6ug+55544DT*5***88XYg@R[^xN=,Pge"LNy(pCIQ3ӼC(#ug bV4; D\.wÆ ۶mkkV Xм@u'?Oʨ"b-tWĺoGrԒa2^^^[l6L&_>99YٳG~U]1qSF4pU>>GHyZsspC4{ޛJ~ >6nsgOT3ﭕ# ՝ gٲ5hqqqt:]Y@PY 6Wc#WB176|_, Z[Z_O9uX™ A'렼[,,6~Vw"b633S`V=@ y H腨T,P3NsӢfUKܫ?^/g } eto78ݿCqZjjjhhhIIlͩTjTTTppbɸ\k*+@̉':8`Pg}Śܜ,cŻ-[FԐK.zm#Ar6lضmS@yxxh4ݹsY w@SSSuZBG @r9`w w|KڬFrvzL&k˖-f''',S\\,@uggx%@x,S!bbblli4Z\\NWlV^ oC x<# թ7&(; OW/~ZT\4YwJ8y⤩zzzZVG4D"Sin~qnVaaAQaAeEyccCgǛ=v}RЬ=8ީ,/\TTPS]r. ]]>C qvldLUY=Jz+*//! 55544DT*5***88XYPqeqAC_rG=g}ŋڂ/$_(<Siif=zpAiwnݹKɞ}uk<}R{ී8횛rd?:tp? 5Uwnȸ}7#჻lv}ֲ '''BB[^Sgtv͆E}fݗՈ!,s )KL~q)޶׉v79\o^EO ֲ$''ff Y,Sk&/j<} i+5=tpoڝ[?nkka}}ݍWn\m1?[ol9 M̸SR\,}+WQ^^Q^~U ;:>v2U?0Vebb8{lGGGugD\.wÆ ۶m֟<<ns*)/V.nqegଟﱴ#6JʥnUIqqIqsg<&M:Gi0VX'V^zj'''oo "&9LfHHHzzldrdddDD :::llԝ,즦]M= ?|F5(vM񸭝 ӎܴzжuN7h4wYC~{!J YG%t`<|pwάoqO2LhNenXɃ[UGGGY_A\8w&7'i@qߺq*ۖ+n^:s;W0Vjammѣ###R+&&&<<͖qniG3FY@ϒI&ʳ@Pjpĉg>ޓ{Zj8~ս )厢V?bf9zF] @4ArpccmSH>j,AO7].b @ HO\Ɇ.&HT sksEOWO m6ֿ~PlJ֔鷃f;΍D")(Jz+NHH1cF^^^{)99999>;v;LVobe`DGGwm IDAT)0+P g\O,1>*Τ]]]s|sFnj7u"b՞ (M֣srui㴬q[D~O/ YBSDU`9W9x@ǔZWvrr P(*N\ m&ixxxh4?( +++.--{{{{ u`@6Uq uttTΨ^$q[5zp}wUWU|Ś7,/=+[Jzz?G.'H斃}r%'ߕ8AS(o/^AاYY82{΂̌;˖?=ѻ)ۙy]o*ˣw7.W Dc^rP>YM d^S:57Vڹuoſ-.~p9 dKCHf=zVMM'LxExrҹ5u(_fƝG=eܘ df/nrZ֗>xQgB5e]0V=*)**QM]c2!!!5'ɑ} @SUUuݼ<&ohh>,??cfȐ!Ç5jTKcr*dJe3!mww}:d26v;4kn!E)/}o.Rd/:wgT^ɸ: +֖}e]jck]|9Y{T^oHy"s\ѻⓓ}X, O%}ݻ\dWFG]F*5v%⹳D>tH<]_Fקڝ7o=>#noЬ^Sg(\ʃ glΰXG=z 0VLLLxx8͖9F q7oJ\`N__c'~ۤIcRRTb|a*΄ 7omi/_}/M3g,[,~/|}B=_(љ{z[[iWj4xϿڰ~x'w3&xLV@]r8+oܼqU|7"N$ZV׎: y@ h_zOhKA?neq<]ϝDyn^pH;aMQɇ.474w'( ?`^ag:222 ={Jbe`DGG)0+P65!C;v,<|Yo cUVZ"1޷+$=!w0$&/++9AΒgroߐfɕLL>Asr񲶆vQaZv\A|M-ΩRTpvqc" VF MNo ًO]W+azU "[󦦦իW2믿oQT777kT}PqPQ[[[rr_pyaaݻwQ^.n&X2I[H;ۓ#^ y/ݝ[hHGGx=2'}K*zx_NLL̦N?yR({fddLA===]VYn$/ÇTm7l՜@2yPs>|DelF}=lӈA>O}XIc)W99R|~FFƦMLbee]\\$ r֭-)|+yxxdff+0+P,nNAA#GRhv}Su:ArZ\V9YMg3?}wSN5kIR(Issĸ*hkk;qO6|Dw{Zŧ'^nN v[E,;\)`i"|%ױ9tpHee%We2}kEԒ<%bxz^S]Vg&Çt2lˆiotT/:x[:Ptc%=fyCڧjU)(((((h E>LfHHHzzldrdddDD)iz@ti>/>}O>`~]|6pn^lnh.-(ͿϼrEkw*77wҥ666zjnn7{.s%g,\,[ϞODN+cҒeh`IRٻK?/""%8-0x0`^I.X UUVrQ =@Ḻ/cI(i-wYѿx;}Uf0V}3f)ۢ;wܹƌ2&&&<w^ @C)d<zwjժ䖖[X,&smHKKCm@y ۷oHD"o(d&ZKg[mnn޳gOjj TPbo] SʿA"3;[SQ%TV4Pll$\ڜN:S*x ^Ȉ*p8DNt+A֯zFDž#TsoLAT@W_o,CK:y* @r SjDGG/YT}@;UTW'߼̬W `qC~555"qxۿ|/v}ϭ‘n.Ýԕ0уaH2kz?XIcTYա}J;wx "Ln\]]WZ5{lww oocj=f<޹,d A555޽[ވbH|G2:XY!ㆈϞ= cԷoRP˪x <P$WYU!O#:V'$I_R@{=efI_Mu3Qj+qcWx?Wf lZvӗPoe kݸ~UŐB5Y4K\.kJc%=P*577\ERRR֯_YAH~)$$֭͛FZfMRRӽ4ʇ^K ߿ٳ&021<6*H@n!Y>>3g677Ww^mXSqqӧ#CsUYokg" To_>$_  EbIYs$o/tY<׽@A{ym ^qGxsp}CUϴJ +a@-h4ٳg{gmVU,Y"qoEEšC.\HΝ믿>yD :=3BBMϟOW;5i{UVV40`Yv܉5 f->,\A":ڪ+%uee{=x`J e)/&iCCCqxX9~zVk,tN/͒;Ksr;[XZXXQ[[8MM%OٕZI6|F;qNJˉ\1 hc@YZ2r_ܶKi+.XP';I邝S?>.|U& ԫhΝ%%%NJ+>#i۱cG``ԩSlْ)8h@p?;{=&0Vߨ^tRxaa]fϞmgg Pچ?^8b`衮|:Mkpʕ+jJ 0VV>'r*H@WW^<-O9Yަ ĸ}ΚP&ƆE.Gwj է.`gEHgue" ڈ›y<;Jz+!W^ٳ={+**:t!2_560VX@p233333wڥ\4@[[[VVVVVVLLL>}F9bĈ..N +8jBJ۷oH"DD6Dڹs?ʃ[ZZ.]tҥkZ[[O2%((}h "w;hLṋu:'1q݇ G^*T Ԏhx*H`xuxIdϩu;qܤ]@Hm-Ztar%16P(ă#GO),Zx@)zcc?VxZVM+Jz+mUSSsʕ_~O> IIE%%@eZ7m:&0+/RRP7n\vmTWW?~|ɒ%֭qeP%̞Z&\:~җ.w_KWSSSXX,@0VvV kڌ.#y] Gb'xL3%.K~Քu)^ӻhտS6^$~9⛌E ccx<}67KN>g$\'w/]_r"Hz*cc=^-gǼ)Hkjjb]*0VZ-(8s겷ĉ3<=M9'ŭ\"htm^'QummiU6d_ۚܿ/arԩ"eI}r-[< W-ʳ@{>~X82vXu%ss aaG֭[*.9XAruGb[8x rk>?u[?PH<J9{L&wvxwijEN),N8; "H"?~$C›#G644R)x(.`z֗nУ(~BR7[NQ0Vffu1t//_OOO77>f# ʫN^x5--'?oin>F8n\i%s |(898PQ8>477sSbW/$V-#{qs݈{$w_K88L8XPmg$9ruuuuuuY3EFv@G鍄/Ns ,XqSIIiִNrlq\gDOyz%N$O23Gl+QQbOP&#]p~| R'dή_GށxD?]uYq!+  {ܹcmVGif׻e^S>,Rk߾}^^^Ç_lّ#G^ < DEEn2TFfF$#3v|vj d_O+)u$@p䯎Sl6m׽;---=NX"kVaw㚚c#;rG>~vk7FU+>\>ϸΚw{VߺwNU뎏jii8SRR z[ ?SZ񙿰 iBgXX__~9cv'mu}];zIG?#`n|]t͜vT9/r%> LVW dډvvT*@Ѹ;9XMǚǏ9~/1ct:ݓ=tHȄK>ޖ<*))MhwnMHI!Ξ1طv ^ՌŋvMxr?'IP(f-[vЛܺ䓯wˀAkk/~[:R>twwo=zr^↠ ˉfJ2]}hlnL,--ک+A9~9@N?wT0b#54UUx.[[[]Q^^X,_=t:MK,'M?hk[#c:QV 捫wouU.fb2cɁb_j:{)FFJʵU/^d'߹ujc#%VwN{;ޞ:e XԦP ]___Z ?73#oO}fChTAAu:UVfȔ`m"PU]u+`~;9o;nو ʊĻ}{:~a3Fn7oԮ*#JldzOg99|ĜUUT8E*r')IdÇ^UZQSP@ؾM3wժ'OJlp:m]_RVb'}+3#c>x@siii8󗁡qϻ^:qԉý:uZ3TT~Ws;'Ҟ>J{CBUy'{v+/\|?~ܸіO/^8WPq39um撌@ `Y$_XL udE">vMbwu._ȕ+AeyܝP ҥ2c䨻on-*zUV6ԨS|~|z+Q3:s_w zE6F1Z[[KKFG 54,߼9%mM`k=k֔ql-, fvSS~qqVN݇YYOw?#֣F]9rDt>|u<׿IKK_ݯ43N5?ȁNBl߾Y4ew||滻;Mlvݤ1Ǐo%$\sա[9z*3g鍳{ѣWFEEyܹ^^^d#P 9`YTQSr뵺h*ۿ+Y^G Xs{ުz|?a Nyzφ`/}Ϩ_`7 nV4մP:Eć\(Kp0*EYYyaVxoٲO۵xD|#L55q}}şG~wq$1!]^6 &=kȞÌw_G``IYpz#c$P(qqNt:ݻF`2';yصkjtt:N(_R^SEBR'Mdڨqq7J+* 4bOOOOOYfԾyL IZ"W F[܊Jܺ}ܻn3;Bb츉';H2 (++}!} ާD9 ȉߵ;*N"t9sɺ~ Q/,फg@3]i&1q؜F&FёRkJ|9rώ;~f0';(@,P-Ch*rfwt PU%WwΞr8N4- s<|SUzԼW:YL-[#D<<]zٍJrm;vGWX/<}`g7.jLʷq*%bf8}=O6M#q6Ϩ_`hzzm4x9Qu$o` +!W 455O>~xmmmX~mtttg`2SLQ>A6fl\\Z*(2>wMQٹđa 叫W ^"23;muc߃O=K/3. ∎ֵcǬ-,9յđ/6m>iR`gUTTU];ˑ/}=>}.wB2,hɒ%K޾{rTT˗dޞby{{;::hrPQ DP":"O(_|Me SMrrz۶¨wFGG&$pe1tOKKkΜ9,kΜ9#\Q< Hf߮[K'nJrrDW`׼A9ϳsKlJ kddb2t( K+k1cƏ4n1[gw${"3V&Ϝ6IrJ4`$͟H> oݍKNWRR\]]U[SMUR46fii5a,9CU5 Amx(%!.&ar^W \Jh4UU5uu m}}C#c#GeSoݎI{ ueEۛVKK[SKdIS;wX{Q7ӟde彬kpTTTLS70066616jj6|ĈQf#|C>x& q i3e*Ae\juz˕xٮc4ώ\5o<Ñڬ󄚕!)YG&$tUu')FkNjFFum.lߔLdyy23##FxvV)mM͕  7nx=1g_WVnf5UK9N32^Nl|5UUqN:fԨ̜7fd&RP\|#Q,'>O(_|MeHKoGF <5s]VW+Gs='ZYحyϫ`3f8'L,2 w;˳2Do=_X`5J\]VU^Pj8ܘܨ:U[\Nda1Z WCDQ>qF1kF1k,uu ߱TUT8EB"]y[qqŸUU'ϏGǣZ[[c'sَ*$߻ n(JTB)YGE-۴Gb8wEF<]{Q?ƆX(3瓾zK ݢ ٹ3=;;&);1Ç =t:FFFyHSnn__uT|frwwРs(!Ri8Tj[r) Qs^d GdfMVr˭m՘R\̍DxUV죭={lٳgcO~-&F8r'1['Ν{d[dBBY>%c:Md^ѣujԎ *ګy$t6یє\ggqO鐇)S$nktutğa t|q{" eݢ<ce5j–ZWn/.$ٹke45 &!uĮkxNqpJ"}\޴i'|a oذڬQF5ruue2edIsw'g'=y2}$ s,j]۾,ݽ{(by?cǎ\]]ΝiiiIVlj/‘ow,Ϻzqs* C?^ee9(u.[Qi)q{M EF^Ko{{?CqYYFD @USUnJ eݢ073 Z$hɒȄ7*+?YN[q8(J~~8Y#FtssÒвY I'4W#W ӜDG GƦ9)+_5Wץ?&8pҏJnxWN-._iRBJprTԞ[Ev-p BA(;:Zm$ IDATolyUВxi6z(kxhk/^-ef^sFllǭ ٳgw̙3Νecc#ؠGJ####b/r^ ijj*S#W RMܸ?o;ji~;Y WCF[mVAAAϻ/pXҥKgϞ-** @m "%Srr^_<16)I9zz:;w#pҾu&˩<Iӊ* Ȉȴҗ9].h%1S"OJUEE ݢR>].))ǏoxCv35#uٲҊ )#={imۣ#"">#f t`0F#).1Y#+++8mF|\p$r>>,KII髯Ȑd߿ѱY&M"; 0aB7+_i_7ʊ6gL¹s3g>rѢͺ;-*]mJf]CWRKՄ"6zڽABÇǝ?~׮S/KY{k g]p~~}o kt?z@ I|[5lT1c|n]UMMݻܹ_VY(3s'SÇ{zzzzzy@q77TVaˈYѣGϐ+k UU/﷈#FNԙ֖VB5np[|2J|\BmFcX_sssCCC.\/III...ľ̛GܼZQU3Ͽ+|h̙&Cs$˺--1lqynnȈYgK (7|yxEUg``AqqBl =rJPBZ[_SnQt0Y}-LHȊ ݽ{ǪU%d&%SN ~Ł͛,,PM@~?xk={82uT)ǀ\`c;8*85_ `|Ǎ8adr%> @N̛7oj|}}<󈈈۷p 6Hxs@  p8mOO$|>?֭RR|g&8ΎX+s3._7o3 )8kk^=5:Ç3z5ItݺJATqiՕ8Ê`w>㋛*YGz(k m̂,x͓'k:cSTIIIgϞ 9r$@ߡ< J6mq$%*dII|VQQ~'g J]|v r?+i|7r_ ,yWatr 򜝝qNbk;DnHHHjjggvvvgΜH()014tiG74eeo77c}\P(ƈn" DLpZt4~NiZtoY.`./M8Qd$ΝOEFe|[5::dG!%&L ; (2m4eee&лk=nWmooOJ;z 䝺r[ ,1ɿwhxVr ƊUA*$ur%> @&&&AAAgϞ-,,QV#""$ (00911Q©@6nU^ ѢP(nӧ9Я}ׯS(k11MMFFnoD\3B\?.sw]DRR쳮֡H(<2Nw(Ϻt$ݢ)(gB2dӉ#ki +GgײJJJD\e1,b߇.4ֳ:k.!HII忆 3r%> @^KmQVnnnhh$sXP}}}IP(III...NKn˗㒓 ;k8aC11lvXx8q .e-?8mnObundBqЇ, ޲e_m,2xʕOwH[၊O* fϘAip>O(_|M@< ܹsnrcI*%*82eAT6_\OvP0XOv]XdEey 2s;]WBR~ٍJr JJJ͛,mmE:t(''oGӃ6l@$J s8Hd)P('/\8wpAΝ@*Gr={3.ϻ=-(H߽յBdpU*ʧoI|7\)2өS";^-2r$YO- Cy(}}i dL{R# n{FyuvӃ WfZ|vr腈_/U˫߇.654Ǖ._Կ_ȕ+Y윝ݫtQVYYٹsLLL(H *כl6;88̙3_ҭ#nf枺xQ⢣"{^뛫*&XZxvˑ#n_n]U[_?/((}u{KxF9L;8-ݴ>nn֣FG|?xF9Y&'Ν;mdJ=tjni裷{azzWZ[$G|M@IP@61onѫ7/I )AVjt*qD__ng i+@@R+Stt9VWnKʭħq6cFKW٩Ϟ%p[:>3} ]+!WF5 T <<<<<۷CZ>鰴_W tuO}ǪUĢ斖CC=ʚ1@Oի%'ٸr<K3J0c^Xv+>?sz'ŦM;_7606i6SMnl|UVr)ҮH1?۷ok;w"114tupfdK/R32(5k !(_|M@` ̜93;;‘?s4=v4F ɩі7n?~ȩQl,Dj4?N1̬G 'nKgy/ne+2-@ȕ+c߬Cz{{X,777]]]i&:hѢ={Všm͟dv^}?DIӴ#FPg/WW)5kJJ?VT\ysex!aaʕC%c3?wq*_|M@~< ֒%Krrr kH\9re~|iVg_?y].z4f̘m۶544ũ"W 5׼AkW/5wܡJj 1EWt kii4TT4T HK[gIST@=\]jk( ر#""Blvppw7䑡"2+|}?pΌccE6ކׯvS]z{:iupcGŋ"oT*/1X-ףtY-ٹ~a'iyE-O)}?477 sr/tw Kփ맮ݸ0~x)]L~~~  GFFzyy!W TTMyq-]|/i_ 4ul/w+Ać\c,o#kk𨨨[fddH2UQQQ``_l\J ,:cggGv\b%''yy}ۿ:E_-,,tkxy݊RZ^̺::cml&N>yrӧ|׊3fDܾ}/%ԙLKKݝO-"5EDvn"n_U6[ 8qBvF\0s$RWTVz"[ʼn,,';7Q;!W$/AAJ.[``^UCCDUU՞={p@ (.--.)ᶴД454 G1Y3ں⒒ʪ*vcc3GSRRUQQSSfl!W++_jp| SMMo#C!ڽm w|M?5X,$n+T*ʳ`Pxѱc:~?kDM"F[E;y5jݺu2Ro$vܹo> &ݻȕ\CyHJć\6ٹ/*+*N ,,,,,,F3 +ȕ;W(ҕ*++4deeر#""BLMMw?8(@yʳP@P(YYYn-t首i~x9N0!00NKrҲvӧOϙ3@ʳ@Pr$> @ʊF6mP(*jjjLaJ|+gH_TT֭[322$aO%_P},˳h$֟|ѣG D**,*423fg3FMCW3(ʸ<9Qee ̞=[FnQjllz;sر JJJd"+!W 9|]vUVVJ2URRkll_ʾvT*eBUV65[k@ơ< m۶]t)&&]P,B0C33s!Ch _WU/:UIEEorr-[K,BSܦŅeUUUUUNv{22677:J N Zhў={<aaa.]ڲeUGJ BRP(sZZZ Y0h~ܹswmGmT*UEMBW)S(nkinDt:cΜ9H_NN˗/T*۸qcs@DmmuJ'EJۭX\TT\TԶIRM7e%rEWW7$$߱cGDD$SǏ޽_W0PUU555-,,$;WJJJdGz-6l{yyyL&SAJJJDDDIIICѦMᡧ׿3KѣG ,(--%CXc[@ 8{,qg*`2iI df}v^^^WWJKK\pȨL&aٺz֭[K,ihh jjj?b98J1< $$݉UZٺzXSg(LL ʳdGKKɓ'wUYY)TT*u{566d@ (//;m008hhhxҥ)Sanr#ϕ@yH0?+5 rB:+Px(5UUU{9x Ǔp*uu-[l߾W$,PX\.722RV9lsssʪo,sss###SSSkkaÆQn$QHHȧ~*RDeaaq5sss 'Z***+E,賖kW/ݍGSWkpcm!* UUJPf.uqEšnNg?aѢ%*jjLrʳdSVV֎;"""$tP +==ŋm?h9sjeCq477{{{ G===ec*߶mہD'NxhX񚚚ڊ FSSS\)g@TV9}G tڍl#چYY%H IDATbUDL gȲ[fddH>% `< b/_nZ[[{mBa0ٯ7jiiYfo&2>wsihh I `219 'gNp8YĚjj5ӪkkL5y¬e%ɹi9"6_-uptcR\`X'OܵkWee$S%%%,_|޽q/`Dv}3:2***/է0ϟ߱6kɒ%f銋 oGF @@.%ދ?ypz#C3#{oFDF#L<`+YZn4ҤϥkֆwX=UTDqjx\l IMMp*6lggw7(.YGGT gg甔⠒+k2@o-a>H___Gv, ⢯^$2?`]JOV~@_ W 㬭#""$(00911_bP򧤤Zikk# C77\ N?vƍɊ dٳDG؎ݸdZ][c^0xw@O7 /TIII...,3O<n~:*++#jii]redEuٰ"8 eJcAde2sGK!!W _tzPPPZZچ h4$S 0[[`_ *(9s'@**yfpٷ~f oܸNVT ˚;H2}T?#8pv"xS )]]ݐT/// bvvvgΜ,'ϟ?nZXXhjj,^zuss3q2..ndE2捿KKJ#mFxk)L;Md徚K'%+k\TT윘/ (yc2jjj666C.>e˖`{ dܫWEqwnGt<H2È#RS22rb%''K8URRK@@@iiiPrq)++8βe8 2iddDJT ._SXNPht]/ ĐJT /UuUx<Y!A@a 6h4Iaaa"PT(雖FFF&&&$C \xQd|ҥ.\ %* 2s^dGzM7fHVWB1 W0ꆄzyyI8;skAY Ӛ\322222"1=zϏfFVT _?ImiinkkIb<]143205n )IRA:<<<""N©%6,iO88z蘘cǒȝgAbKUL!nf\ +X,VrrrhhS%%%Klr Y ʊVVVL&xoڴ)88XibccGIR\ A G&kUUTXH rt:=(((--mÆ 4MAXXmpp0OVrrr$ @< dTKKKjjpSGGghٽ*6y_~Ed3""BOO@Njhhn2TÍI{::M>Ӿ^j@!W EWW7$$$55K©lvppݙ3gDFѡC]&aByȨt.ĉT]Pf ~":)QH  ` ]_Z*/7_b(*, n 4#n*.j~\Y[[GDDI8UQQQ``sbbG,[,::Z³HA9 뚚T*NWSU2Dwa#mmlǎ;^UM5g_<aNN/s|>m--mS#Fc;f$ff#Yf'^rܸӜf:͘E%| E 椧?}S򪸼`3 Fec;f3]54$?u*+bNJܗ/jj2ƍ0m&&( )eM}S#"SfYYY7;/ @JNN>y]*++%*))e{56W+[paxx$ ʳ@洶>zHammMbO%.]e˖۷vc||˗{u>G iʳ@<{g*:i$% g||_uu'N%+*PT|>?offef ;AP {wqGtNyЋ" ;͢P(׮ߺqr _?:vioOGF\q񓂿8}$ K@p;WOS]_T[ "~kkkvj +!!!;"""$f?~|f/^|ƍ &3KKKll, @<|T+ADEUUUNNp\WWx鯿־|2j3k}@ ^fQ('uHQ}n^7'ݷoOu2)Ά=]A>q9\&C]Qe79RAgmmagg'TEEEΉ]~k4gJKKg椤$f@xNg =sdJN^>|ߟmw-pС111f")(PXϙ5A}qݕgYJ8ٰ_&$}v/r2$Hl|PL%G "ASR~iEj8X,VrrrhhS%%%vԩS7rWVV?| 9NAAQ`_xAv8!Ȑzq㔕IGj>} ZYY]zuĈ ,~w56/O!ss ?r+kSAz޳.]R71ki!n*+e2M""WbAAA-ڳgy<^aaa.]ڲeUUt2VVVvsspBfMMo޼9f̘f N( UIIOBmm-"+ 0Hdff 7̌IG:x<ڵkO:%2>}wM/a%Ce" .նV::ۺ-**ʖldI~kz$a'9W>O>."$=o9 ;Ĥ_fj{0n0M--MMm:]SVև䤄woD6Aj?vY?e9|}}}UUŧŒo߄zyH3ݳA- ַE}MTUUrrAxtla8gWivddf` 5e4)ҒDqX 2#=uw*((/ h4Z?sAC455326QWWWVQ)..ea/~yُsb0duu쫯sƐP++6QWW߿ʕ+7nÎLUYY}3gl۶m4!>~x^yAO<:O\.Uu d6<2 B(ςNFGGf0mTUU͟?LJ/~%Jn~r"e{zk\UV2/edddUUM9HD-L3O]3YEEm55 _zC{fL{Y{-"?n9g5 .6f:t `0"S83 %a{'6|&''Kpݻk_io;&UKU&T[SK.d}(/n8W`nnud?.Yرc>|8A&L&??ܼ#) HNN.)) mmmej,...YK.y&j@3&:RSߋfQŅ&OLmVdd}'V0ܺq=ގ ^| Oor,Aoz?|Eh%/W~pMU ]Ip9<<ȑ#*,,lы/522255 //oҤI?~ ݳzeee}'$333!!/O?m޼@JJ[SݿlZ ̚yKgϟs\q4cbvE^3 >mc\n.v=kS>o}$LM]###7,+31.5j*Eq:`,[l;v8vX}}}4^tΝ;|SRR6HOO=zt@@@޽[3T/mvRyUUIYY@7,É-*##cccCmJ=y䜜rPJJj_|UY@wڬ??8(OK}obWGWRRpءi-&%ܿ=yLq9d|@gu3ZZ:aIA1URIAY:Mm)r8W+WnܸÇrMZ!-ݿJHODy770q 0a_mիWQbu |21oH&yţ/0 Μ:*Ϝ+L'h5.)̥*Vo˳pDÇVVV233>xʰܜl%0vgs>|A 89 ?}*,T &sskۿd%gFF}$vh+rvv?rHS:ǏGtN(ʈcYCS%j|+^Xͳ{n///EM__w![D(`PFVv'g!(hۥ˅/nD[GW0 ĨkhWɮggQO*K+yCah[bGǹ9lٲwޭ]VZZZHNNvqq!߀Z#%%dee=[]]݅ """\(Orr2y߻/SRRʣF:y򤊊ʼy СD:L08|Hu -'٨ KKKcie=d|!O,ҷffy˷wݠ#^ " SO3C&B+qPWW߿ʕ+7nCϟc (++KH'AG&|ׯE^%*:::䈮)S (--=}tUUE@.F08tg + kjj$G`Tk]O`B2GOżk%G5jfiտ- @|ͽ>|hee%߾}F9rrѼe ;8*--y gaaѫ_Z+))cii &ܿ CCCgggrgmfff m |2p23ҒSS*++de{cbn__߀FQ)AĐ쏕 M444>.HXk&Y]]0dU,[S>F*p9<<ܹs[l),, ,Xb,@JJJQQo8pf;8CSLMME2{F)X =\ѧB^E,䓑O|| ~P]BcB M-#$<+B?|0|UU0 A_xƋ}o%]NaV^?jЧslٲӧررc"[w 6ɉjNvCyHZee%yYC---CCCEFFNJӭ)''?sg Q[pPbWUU)4&Lĭ _{Ν=j҄ >5VRS~9 $PŐaO>bԫˋʒ";US9䈽?% @2߿rʍ7>|PTӦ$RtMwȔp&buƕ֯n]VD;)VUIѥ1eOa``d@^wpokSBX]~9bdlleU @ͽ۸8{֮ŗa@N^ph, nM-+-p&ێ 5 IDAT֬Zukh Vk;+yXSU9U()(&GƎs* @✝Ï9! cbbwA#GH>>sՠ+NrZ(009998z6{ʕ8I7mMN.;dzw< ,n5$+gADR߇ '' K8!ƌ266)({.-d^}!1 {7^s3\#\20< Eܹs/]$ڬ'/]]ΧҠBR?Y v8ڔz[ؘJ' o++J>HH khI> \njJrTD}" >$fzV]Y-4kGDѧ ?&5MAP4K544Aj¢% %%~!+++88xΝ'NTVVџ~u+˕gbYCj99 \{$ž Ξ@[GOtPWAK+!m:/=}ars͊r QSYsVA?kx#sTTI ֐c_TVV\HII͙HUU]  92| q6@ԩЈ{=z455DPUYYI^P[[HAƽMn\y%|D%7'[0h=FtP]]_OKfߺqOJLhx\1)Q11f%l!o,$pLBzK}!--MRrhYX$O>˖-;sLJJJbbӧ-[fll,Ýu-[C ?yum'MGLuR陜laaahhx7oPt˛ˍ`0 6%k[[d0EE*3頠eec$ to9f{ǽ㋛[ 67dU0 ={đ2r2́]ǎxNeA FTd8ׯ&w|Y?>&}g]sl‹?WAhIS"t}ӧA999ϟ? % q_ۿyz($qpȺ/}hyIF۷o_MMɓ'/]t%&9zhWWW777333ByQzz:yYÁbYCx-Xrݟ 驟MxMU5$NJ!Jqq3.mq-ma$tc)sgydf7o9¹,r1AQ#wPRa2)?Wf{ 2\$Gf̘1c RϟGEEuա n$L@4JCC}CB|CBݶ~V$1 hÇqI }}}}}}777WW1cS)4Y .qqqp J _ >6'MA5YUs飇(//gm)LH,ct B6}6N|>]pO䠴?͞ Uk֟;wII1y'>uoL;|`׳bG||VAAaג~斃 WSS_l>7^qqќO]r޾Ug8W"j={ZΝ黾^@pܘG!!CB^DF֓/ 뙵Y *h4ډ'#G9rDNNѱV~b= , JKKcYC}p;7 >qyZY{N>zx##c5uʒ⬏aa_>,$01+|p?픖.M~+֊.3._2d2>yiG6]|Np҂Wyj:_?; >ZY{Įgg漏JJHM&,QUUoNpyS*zصϗ%U'9W=T+???<]|ܻ=뜆  ffo,|jkkz۽Հl24x5ՂOlQo=ե a-_yK# }=} Xu b^ҥ:FZF:*ZjZrL4Q__ǪbTTM˩#`0]O"m< 3#=-d"7'ۍ?\N'9WΝ;srrO>VΰAeEů-g@rzFY|MٱcGiiǛ&??͛7oޔutttqqqssX ʳ@cbbxC---ccc ]=ϚZX dͬ.gȱ**wT}[yYw$"^0v䩳VnhSݵCBoTVVټe8ӄd{]vo?!hQN޹uhs٩Y٩ =[ɪ)f Y &w}͞^ݽ_w*NrGOO+77Ν;IIIqqYw i/$*@0('00իHjxh4ڡCTUUw,oƍZZZGpwwWW @y^tt4yYAQYX0xiITQ2rr:v. z÷}5tD@Mߜ>y/ջ:z쫢pʴ qm]y}YLpqcbfZں.4 9~䑜G]e2EUus:tPa%%jk6rof]lںħf5eǎABNqvvvww1bDg@yXvvv^^ohii)//Oa> ȸc\KK(&S^OWjJrkp?O_< CYY9VrG|eg<5rS}&稭Y>|#GpCYY511[jb"48MDU4EEI+'J +dZd?bqV ysnfkCsLj 4CVF\VFf!!aa!_G{G^Sǽ{5UUΘAa]ǜ/ffֶK(2Ax89?uJ UP\1egthx-|ʩ;ӧmݑfGFFFFFٳGAAo߾+Vkn8g(X7o[F]]O|2?o۽pʿuw12=kт{OhbI%Lϝ=j~Zm(`pP奋f> O:2mMOs'M훸ط))Ie~b$'גn;8h0߳JYZ`i5ߴ'##s^G'_Ov{}}})>Ԟ+h %%cǎ%yDijׯ^}[[Wrf)SN+ihH}" |_|!%%5h ggg''1cHs Bmmmi}[ѻoco!1ёii)y55 CUM]MMwG9omSJQc/sw׌ r8:_~=Չ@53τXo݌` JD^n· >i`hm`h׫h-[ w qS^c׼# DQ^ё z6;&!!4"ӧ223&N:;y8(<<>99+/ZVFF]U_vv 0ݓs܄wIIiiIUUuJ z} kۿss:ζ*(*J i~B_OI)(*bZjjf{Opppqt4o_J? ~lyFe+Qtttt_{Xt+W%5R]]cWK~9io )#% AN߮/, x *3g~{ڼlFFO_/#.>-__vUUsr^qs_7lputlM rhP3{8tyd$_<'//'//&!֣G4mĉׯ712je2T[뒷7ܽAmiiqi'Ϟ8wNfZ1]ӞU[k6ʂNӧO7o޼y&ѣC< Df%NljjJa>ͷo)-R&tZ²M'ܽpDp_-b0k1fv.E#޷+23))v?EjS9ݎB^.W@`0u|gJf̚dgXKJ?v9I) ih4b_y֋֔gEjHTR_/߼!71_9#X#mbeƏ6?t4.di{A5,֌/͚v)-/_eˍZ^l޼b$p<ѣknf%%5Sյ-⻦=MfVQQJЍ m>s, KIIٵkf ZoNaud7ş$ǦUT3LyyU MM--]]]=]=}C#ccC/ma;v yHx؋Ғb^/3~C;nBeV}ۏ͇X:',J_sLUV"7m_S yZkղk 'Ǐ#/"#ϙ⎂~¢'/CUnYYU5O8'_h9s@K淬cZm#%%ecaѦ]Ͽ64"?rݻ?Cv%KZ[l$]q͇۴׸+ֻE״'`a11AA!!Qqqꌀ2Z...nnn666w|2gW311QQQ0 dtt:jjgΝ>ʻ[dF:8p"}ACJLl y2gGZ{^f?nIanMqaxʡE^DFX[PI8!t cbJQ^FhYFN^ףg |>mկOygƫ@vt ?<]Jmj**tt܂l v>lݯt7$,'U[;iٲ^5FSVT#ǻǻ^.7$[ϟn}I9={ٳ-[꺸,_|T< :YQQѲg߻-o=Msۏ 11纏qJR7#>w}ϙCmDKNVvϟ"i?f4y|u y)XmϦY3l6;+7'0-**[~玊RYXZ;dKSS--y99VmmeuuFVVbJfVkM{~171w򤢼9μuxf0Ygn{nkn5tFI bӞ=YtgРATTU%GG? } t.nv'.Fikh 07 ~ ߽...'N4oB@< :.--77>vȖ;io+ |VsuuZ:w/_29sş(YU&E.""8,lΤI&>5\VmL3;5.Q9xf7>./^+hScmKWw53]]], <[XOACB]t:mLL򃭭[[?SI]B5Nh4V*.- -(:5رcNA tHUUU%8y_S ڥlǡCCZi: 3ǎx%)pm[$gO fZյ-RHMEe~|,ȶm5_! ݉39|AfʳCbbbx  5@ڿFc+<I@Θ=s\)Ѷ]̡g_<2"lSVJ~tuW6xLhYYIii`eA}|2Ców7v׌3f3/ĉ IDATwgrDUY榦uڵRr篿aklZ)(*zƔ実 r¾}.ZnyM;>fϾ}X^X3g6,_nRtQvvv6l ͽrʲeˌNkCy_NNN d:7'IG`rRBz5gϛ_oQv*YYMrxoگ6>v>FѾ_󭯝:*]kafv6jKVu]O\OEܲENVyv|:9r?q=x`M~QNde'|ZQ˗/w9rH]pIm'555(ZT\)#=5#=uK ttiniĴR]-411.M'1M66a|=]llٓrsA)),_%UYQnيz֬ZȪrsfLpM{!Hر]#GBCzqPXya5t]}}㨸R5;>n\eajАoo3cc7aU\B()-6mݸbE+g{ AL9;=|РwKШIrr9h/]] ^$*.-YKfV7osgvOص-vr -͞]U]"*' CNN{B`gg>|pc,SBBjx,%%eccCm>Т7Qüܜ7O~||V);|윇4,<QGl=è>O}摓_,֚ >.^g2Fx)/ͦ?!Čdٛ:5%y}htr"gČ,Ç$R C3U _Ro-'i +INOZY%w $u*+,w٣[,:~~I*҄Jk AÇw$rKySS"vwd®n״K#Js@VJHIqYPZ]HJJZ]리AAȑ22cHefgKLk170󭱬}|J^hD̙Μ1m]6#$Evp®n״hVVVV?^]U]"*' ிfv6ߖQqqc}r|]0!!%q7vvvÇn׿P\h?򆖖\L=Fu F3knלDV->E ϙ&5_|Kw=]DC?wvrMkwWǫ7cJUnГM0_%Gt)oʪ)&4<05645OIiixĉc0Bs`ɕUTcD7m;i֮&>r=s̬KxC%uw$%S#.|/Mֻ[^BtvppvpQqqoz:yGgϪReddxzzkdeeG2qD sVByALL mxܧOjRHOo6lhdho7msiS<]=`m30h^^*+y&s¨Q4Gs8O.>aXVQ@ݛA]E%7,p8RRRmz!Ť"**mLB][e۷|?xܴTTMzjX׬iylr.ۑֻ[^ӞFo׿W?((gyo-zxFOJOO?~|ff&AFFFnnnnnnƍê] ~ Ak}𡸸78p`j/:7S/|w6f.ZTHj?!h`##Ed ^ܨ8Nؙ-B>fϾ}sJuWaaaW\YlY޽NY*III.qMӭGѣ ==+WIv5nlܺ`޴ rPEE?&zL*ncP[w~r{ z&=mC#WoxpחǍw 55>%ӧtakx= p8)I9|kإdfZ﷜ < K~`g!/]-)QWU43: QTT: i."%%7433cRT ii9s̙3D};mr䠾k-ʪ1z8tgqqoyA.}ۦ_v@|O+0<=m"b܈j**|;Θ8gyrwiUu5/3޾;980kjx{~~E%%ꪪ͛|O'V+&^SVUWݺ Xiӹoϳ(I#&#ʳ? Wo~|.E"._]-)tu(ςp8wz0lI7_{N~#5J dde` S\ ( z3fIiio ]Gnj5:VF[ Xxt޽g6p^^˖df+is_[[#usXpcqML3端n7Kqrut׏r֬A&ZnyMCy4'==1Fn{ [aA̩!|V;yIFVz:k[vc#=\SS) z1Æ#uu`Y07AKgjelެCUT8͟_WVmp<ԉ_UVTlA%)'wa]--r0+7wwq8Ð&+f՜|d55lHn{h3vu~+"ccڱ~w >Յ-@W'&SFFFm$ϝ呙߼e皯6PQ Z:߭_U__ ffO8ܥCSt{4mgmڳG)eEE'f:]~MRW?w…䢥ںG;uyH+SS- Ϟ Nέ<սvx//rړg~=zt5|7ѭGtmmM UUTʪʪ켼wII))5,As.կ6|ćAA KGGC]!-]^Y_X}T\\YEA._֨ztk ]ʳIIII-,,{; z{3o!UYX.PWXl~uu/XTi'N_v?}:hi[A3m>s޽lҚ N / va߾7Pb~ԩN ezFP>NZWBjjSOuwKaqCӧOYYYu+s۽}ߞ\.u4 7{rh^ݽh}: !,Hz##[KQMMEI=rdGsf>t]_ׯHHKKXZ996Fk屒 }ۑg|̣ϓ#O055U&O$8EDDn~~33Z[X 4hoߊ#yQY5fȇ/""JJLA^lؠANͧ-|qM;:hOUp89YYy&SCMMG[[MEMM.nD ̆:::Ԧ]H`y(EiJ= j߿?tu55_\s/nxMTt6ʪ\j;>sgO:{񆒒 UHF33hTLu")ٺܻ[^S<4ryC K͙,X5y̫6s=z5C&Pdff60F6汪3X0ץ'Mkede%&cF+(*p~iظ(lQmM͜ iqȟ\ 'Nsן혶[+NHJԞF#:T=4:p(:Y***R@R'9ubQpaBk UQQ=wo:>gAC$U]]-yϕ 222=jpWZyy9?8gv@礬L9_]qI&j? t%4N[E!:è167 ly,Od{wT6pX= KguAbiQX(v u#Gݽkkq Bq1!a4VdVcP@"*$혱wwV[?fߓ_4hN;z܆ 'v"h>h]|Yf+nnn (QπCZncyQ rD"֪u6wЁ_ e}Z@P(G/**ba`;ww0cўD"taǎl\Me2YXѣ0Tj:8c0Ԉ Jժw;ϱi"&E*^ IDATZ:ڳ H$/_c777PhTcbgq^JV@@CNtiiia.] Ydddw'R)j:@3A{VKW]]-\:+::͍-/=cR١Cq ڳZ4^/\:SNr\|-33tP(X: p ڳZ.nFEEОr]zU՚:v(n>@3C{V e4:( ڳZrӱ\.g,hjRSS"&4KgD%%%cTکS'q%ڳZ4qppY-NIII~~yY@#=4nZ\E]vn۶-Y͍F)//7d`qZ,ڳΆ BdP3Z8ڳVVMR4$$D|f%??|RDLhhjVrss-v鬬۷/]TDV{)F#v".M0< 1'[lYRRRRR҅ JKK%y>}:))IIIԩSݻw;t)))O æMN hg5EEEztT*Ǚ^xSh&LrIk׮M0!%%D"1% YI^^888X* L.\; db'՞%b&$g5t,JYč7Ǿ BdHhj6 l4g5@3`B{VsPQQQYYi:@B{Vp 󱟟\.1&g5,v6ڳBq֭EY.OՖe2Y@@0=}}}r0= ۳EY.|/b&hrmFQVgM AJKKkjjL*J|8Fh*++JG֭RIj=˵tVMM͙3gO:Vj\.mժUHHHN:wܵk׎;:);;{޽LII(-- СC7hРhf۷oÇO<&R"(vgС&M k*55u޽ǎKMM˫T(ZҥGH+]~}tҕ+WJKKu:pp>l0ƘpLMM=ydJJJJJJjjjNNNYYYyy`0ٽ{={EDD8jv@g6a!W^fMNN-6lر=P```}3?+o4kZ^^^^^lO2eJ~lE׏3K:.=====}ƍ_Ǝk {ٳg7oldffm־@=˵ W:u:W[nݺ522r_7o^RRRO4n%555SƝ;wܹ'XxqFu^{\[t7o޼y>t~ʕgyfϞ=6j~Ȑ! p>|ѣGT"HiWSShUXX8lذ~^YBqqq\]]=wܱcַ7,>>޾ /9r YU\\L>70ĉv?,Ç_v ?fI$}6rxhra}<== ;viΆj,w3T*5j}ٳg'LPZZjǵYYYÆ ۺuݳu]Ǐ˗'$$P_>AО@E{ v{ 4̈́ <ؐzԐ;˹s͛W߫rssG5_^x8H~vׯs&&vpyzgܹ~RLL#<2bĈh???^hʮ\rܹ3g۷/;;[bs{h>}z{nz̘1#G iӦF)..NIIp322$vl(Juֿ^zu)*** __v۶m;r׮]O1骪&MtFDD?~|vڴiSTTt;v_˖̘15N>ު7K.w޽M6nnnO޺ukJJ4o>AK;v4hPcbb:w O>}ȑzMY.L؞ժU+/?ˠ'|# #(((**ʼ`bbW_}5tP[Z|O?dvڽ[ 2UƏo:HMM/C)/jժ}C yWm6{lSY-o֞={l7LLL{xxϗ`HHHHHȀ^y>w5Si{-[{ｷzԩUUUV_0a’%K""",_JHHxSRR>Zs5c">AK/^+ʀΝ;rwwc g0憶gi4-G|͘1clClllll-gfeeV_zG?SJuۛDGGe:!Lf7w߽{x˭~K.u79wܒ%K,۶mc9ww7x#**'?iKs˭(+V}:u|x~W_,X`ٚRvѧO^>eʔw}V<11ѣZVpB˸T*]zuжm 6ƞ;wK @Djuqߓ^ʲ-+**]˾v4Jmvֳ6WLv ў媄Kgyyyꔛk޽cX~e|ذaM7d„  k۶g:ujݖwyǖ |w)))~m=ׯϷ⋶f)^ g4/^8w௩)F9}Xvfx=;zv=;:DF 2Omp je;j=U ۳|||lPV[+++VZUSSS+rʦ֛e`ٞu^|r`.]ygwww3gs=W+e˖Eqڵk-AAA ,}N'p&p2دͭq-ѝxόwq6^gje;j ;ة|\֭[[=ꀜFն;:v.G뮻,u\6md?~}T*U`fffN-Ηw!jժ^X>A3ee]7|=4v(*.YnZ%g2qqBCC-֭2e2H$ɉ'<䓎ᢢBBB,7LNNnӦͭڷoUDTN< jժwޖrǏ^/X:硇-O۶n:|WpsZ>įoJT)2D"UU K Jnddf[IAA>;vhӘY|e{(**={v``g`ǎrFҥK,ˈÇ-#F#X[5X_~۷cI'p‚_ξv꫾mDvmw;{{ aN]tBaU6O&yѩ3]Odje;jlD{K2aڳ&MzZ񊊊'>_', hmXePPǏ[l_AAA[Y훽e ۅ߬RZԮKqΘz=2ؼ+98^k=7 p҃ 8ԝZَZў咴Zy R)u믿aˮ#lٲ5k7ұ J['NZَZaΆfӧOw8`0۷oԩϿp7/,,;QR([ƍ:^ua7PaaanL7e!4C~ kKc{h̺ ydmkؿcƘ!Y.gK~m[/_ާO{׮]MO׷Y:W}5*[nu=X%4IX+-j&f/_}K\MrqF>v ؇,TQQa>o,ٳfͲjT-{0aѣ;VjF'&L:a^o5=<<3fE'h E7 Y`>~J )bX+de]qBVVng$e'''K[1b;SSBnnnV®VԱnոFitleuŦF'p8NſW#NK&wށ paDׯ&ǴZ 8pGgkw}76l駟:u [ouůZ&[VjruxjVKoUbGM7}۵ܜa}׈GL%_0GJJ7Ǟ%Zhڳ\p@J۪TSN:ȑ#~ƍؔ YhKV/i"mCt5쁁2rƞx xk}Ǻ~ځ_ #!v7pH}==9}*o쀘bdBlG@멮6w丹5ƾi믓yRyӖ-[hWT .X]JڋBBB,ϟwBZ 99 ;y®e7[c SʤWyLOuu; D{jc9J?O8 F՗-u袬dcߝ3e0++iG'pIi)Hm‚ǬIGZ G{lبY&}ʕ+ܬlo>$2!'Nwccc-{MMs0ڞ&8hܹc0&9QY}yyyN}ĈŸ~ u2XTT"4Oi)n^q=C2XXxp|JJO:!J&v 8YG؞R6Yڵkgζ Vd_:<1q=? qZ/[ EDDX]@%֠ ᛚC;m߭íNE>7m{Q2VV!hr=%HRȑ#- ;YŋMDC%%%=R|-tZ=z nٲř9Ԣl9M'hʋ #}FYWt}G Wdݰ%QQ+Q+(gg IDATVk>vf{D"iӦYu:ݬY ԩS-jzܹNg9sfNN00`eNKKsJ2XZZj>A@9{dMMcsQBBBXXeoYxqc޷o{2^TTt}6v&Vs/f rO@oZ(7MAś3wje;j,#=~Ba*##cݖ[=޲Һu&O| ;hJw/-X`ܹ6gV? gΜ1bDRRRn2|px?{v___6ol?A݌FcVj+Zdn+M/] VEmlv 8YGכC\\ܸq+Zm%/^,T|}}zy䑱cZ}i˖-ݺu[dIEEmgؘM:uV_rJ[nUPP_O0j3Uݺu_j̸ђd?՗/_>nܸs5rk׮oYYYA'[nua߹R j"b>uk<4 闝 dOs%lϲc}=s#G=zt^:ua>SV_r%11q֭[n_W4hPvvK%%%o=f̘GGGxzzVWW\t… 2m?#}!J׬Y3p|WSRRfϞꫯ4hݻwteeeEEEG={hH$wy GٱcKz~ŊV8pqtҦMNWZZuҥ'O^p"<˖-Ȱ|i߾}ݻw٥K֭[JRDR^^h_qy#4dsrrfϞ=q=zX]aD'ϵahMv:N%bz8gjje;j,ճ;f>ڹsΝ;Bajڏ%s-̂7mt]wY=lӦM6mm/^9" ۸qw߭hPZZk׮]vxzuJ׏7N|jjj>|aY_+W;v`|h4>}Ӎ4C=j*xQQoo{xx u|b} beb6%GXQ2//ZَZbsC#욲=ĉujZ춽Ydҥ=w;w G$''7m[nmӦCVv4oo;vj7I1b| ԣFӧO'TVVjժv(ȿ ůX? wV6Vhr18%Jry}pՕEP|>;2Fj`РAǎ6lXoeGs͛ѷ(sFZJ;D'/]q|oPV;mjje;j,#\ʎ,D w)S]vX"$$]b{Xm۶ Oyyd_=11сhIz?w۷GDD8*k:{G'*mpRT7uk++65$;OuuؾN>O?OnJe||O<1i$;H$Rtڴi _~_|q)/ӧɓ b7xVľ޵CZjĈN2eݺuk֬9|h>}L0aʔ)ڤ[n?ɓ'/_y J 0dȐ;#:: 5ܹsVZ|yzzz}/wss֭[lll\\m{9qĂ 6o,E"jժ]v6G'Q6RLlteiX]]m0$D{cڳ|}}GINN,(((..^^^~~~ݻwٳg||O߂J5k֬Y._Geffꊊ \Ӷm=z߿{ZTgСCuȭ$R6mڴirrr8pK.eff޸q\yxxx{{{{{nݺK.]v4hP֭@~V^]UUu#G>}:###77J&) oooΝ;w޽W^ *{˗/_~]hZTf_PPPhhhTTTNbbbun߼|M^^ޖ-[Ϝ9hlٰ=ANW&qrr+wh$ZizBlGO $о,3'eӧӄNf̘1cƈ2D"u3f̘1cC&APJ$桾Z"iutKRI+3Q+Q+@lxb;IR3JuSӌG_JӾQ+Q+@gڳjm[U^)V&6VܔiSS+Q+@gڳZ ybeb"0/iSS+Q+@ga{@ צMpXRh J[pv 8Y.ճ@ 6beb3 vbD{asCH073GӋMS wpv 8Y.,m|}Mnfԭ8\]n*v#6;D{= @S.aJbXVaDdBpfv 8 Y.,ݺ gӚ>tnjVg6je;j,#ljL{0*ju4W/fCLַ_sVVQhr12`0 @SP(zFNF_#V>rba1Z:9je;j,C{@-0)*MI(b>g^F .J&v 8Y.F.kjܯ8_xx=#?VY^)V>Ta}DDLL[ߨpڳ\ gXp~Xdj9kbIA02rXPVNC{8x9L;1+D}憞:W|$Ԫ>4јn$v*JDЬОbh Ο2G'("[*a;L&?U]]o剝p='O2dHxx؉`sCC{UcƎF**ܤDɧF_-japmۊ9je;j.رcfڳ\\.7Ӟ t;;w&\[߰;Ua$,< Uv \Vz&vh>ŰzHrݸQhf]Ӳo?us{vMmzb,BnZَZ)--5^tːd-V>Y.M*RFT*;)B9'W|gLk h˵;ܜzMd ?ƞ^~hTV p%FB_e87?t:4AEL+D")5.5.]ϿC#J<ѵ[F>v ؇,#ܻZL'<.&UUl;/:kk8?Vh*qTzZَZ;Q* gXէoϯZ)܍h43'ԯTM3mU]PR+.z2K*)鬮W~rke4////7keW!‚4-R@3G{Q*cV.]cf͞տW e%en;pqc)J\8vjkzyy靻tNfՇS'Õ˃ mx*Qp"DkoV\+R [,@Ӓy,42ڳ\(< /.^8or7uA m߱OtF_5'TJjbBk  T*w٤Q~|_Q i{y H4{lӣ_@DZI$O<^SSc>umXX# ؎,C{<<=̡ض0o";vǷ:hR$<}Hm==< J[VQRPRRP|#;?7#ZwKIii P(FǏ=fL&];{<4 RxBke2fI$ׯ~ޝ6=77t"fR{oӥw/Hj5/g_mm.&KKOWWK$L8顀z'왓K'vvԪQJΚ#Y3n}{:+` ~WϢ= FFz%44 So(ee$V3-ޣ+ts戻J~Ös^ڭyg=G^X6[jucr>Wngqee/Vf0Wdv-88400gٟx>YBVdW:w!ӓO͞k l,q??g)(8,YD ԁ ]J2Ӟ`#Fvja^=f܅s)Roǧsn1{8S훯d^rXҍr[}byN7Vbz WH$=m?m9[u=z֯Lczzz=l?$I^nNV֕E7ZmR_]-wsPy|mў咄VTT KcEuvȝ}}'?0r'y}[C'# ^~^.v~& Y.I؞YuKWg$a{Vee4}8ޣxQnaex;s:%Fz3oXH$ɏw_RѤ5Xԍ43ek7m'ΝpCرbhrI,VFvja3JecS7_ɼ5DPU8sF^u߾y+cGлI=%T*Ba:4_eYF|8…]ƌ3oj'sG;/PEWJL;'ٲeg}v2hrU4MgLz~ՊO?MH)i3nYҒF-~ink(w #y9{E \> '~r嵜Sg.+yGU96lH7A}M.+vRN2a„Ԯ]knڴi7n,)߇]Y ck׮2l3suǟx<,+ӬFvgN'~}aW~Mg[C!-۰v֟ʧ>W9x FuѢT~~/_)VzҹW:q̛7'NyREf̘nݺ{wٲeiiibgz=Uz@>#0QuF__ɏ<&^\Xnk䓕Mj[Cޯ᫯i"[RYY_]^oҌFcs|qvD5v#FN…;V}kWyF O*._|ƌaeeΝ;_xᅘ]Λ7o&ۢ=Uy{ "f%8zad9Nd|Ǿv֟t¼upWڶB!o0D8s.`0?~ʕR32,ޜ7oL3.bEܹ_|Z5YDPtzV˟~ĉCBB瞏?8%%E q[g*___qii`1G=z:|i{ϖ-v¼uNx nq 2u|W^ .|رƍڵӒUTQQ4)_Fаxi:OH%飥\(4m nqk+>W\|ݻQ-[Vެ@o?CjIvn &͚պ_&mɒCfI̙҅NH%He˖˶\UUgϞ_~W^aaa%- uR3< d>hok ͯK ݮr/$߱cv@@7Sl=qBC@//Qન>|!-FV׵'MQaa䑬%w{Z ?'. ۶iS7.snΡP(oOիidyVYSOGNjkoL6nNw )zfd eAD.#t%%2J:BԔUYYYmmmׯ_qƽ{ +**t>j(GGQF)XɷoϯjiiQWWӳwww:uÔ>yCCɧ_=C~=QZZx7ߖu-97 U, Z|WYCWQYYo>|877WSｇ,_ww%=ΣGͤ/kkM {x]!J,+槟~!--1cL8ʪ Ʈ[oO?ݼysoc,`0{ZZZw}'?~ Zf[׿wEŷ aɇNc=&OS,ACZ8x7ggh5XQɗFdT$%%=z… "|/NFIo}zx|w/yCBz(X{}|zxѣ ܼp$>9$/,&l*,ܹO%GiuH5U,^v''f;"/_&$$$$$9;;aFZVWW1{Y\]]fuۻw^h]NN4*sa{{+VtD~iooЪ*D(U*}S\| G\K)w20IV{kvttF444bU?lаS-+kkһ"+)aX/_~ zf 3}Z̡\}onqUtn.̖րŋɶP?S &# ˿T.ܩeU/^ 120p1yhc޵}V? m"#͢P(zz ˔n釯p>!!ٓ!rUdd[bPiTyMM 窪Ç7`}_d2%foիWnbbsnܸ0qD&*}t=voAXX ~s|95n;{~"ݍ?kkH&w9(+cǎ(Tw0! HaiiyU snf|33{\,?OOg~ysw #+(h:,21%e_~>8y@OǨxG 2e(sSS-'7'[Og::Bg|RyǍHYe偄5w0&fIf&&)' [ Mhg̘1a F#ɳg]JO?nkL&O631qpͻw0}t{,%`ff˗,knn ="J$((]cTTT?~ҤI L6T| H䑈./\eyN& ݽsҴ5$ yg ɗZŸGg9[} #FX`ѣ?~ѣR%HSSX@"MLN"!fu3nnnm}=`mL9)GWGsΝqY~~{_rV F:xaaKCC{ڵk{YrEP"{Rr E߈`e#>oߐo) :X7VP;D>y__II>$وΛ`dt_w$%/3?I<|d4/i~,>BsnQT __}5c3DAϝ;w;v1crĀ,e`NzZVV&Hf?ݜcggrʝ;w?~˗/_NJJJHHطo_|pB`6gΜ"ZYYEGG_v7neMMMdd40>b~An`dL]]#G~P'٫(}[C2nCHuU%{W2c``{mݺ5))h֭:] I'}=b#B:p+p).++,$fgbbqy9yd!F=]7e߾g'yP_?i~{nf?ӛ*_Z>n\rvpXx1yHJIer姝?M>do? vJb-JÆ \}܁}O Ə駟TVV;v,22JA)6g)K7_QQĝ_nݚգs}Gnݺtٳg{{{{{{Ϛ5k6l8pݻwo%87<~/tss{mڴJKΜ9#%CŷB?wTe+uuƆ#*֐,~A䑄?&=)K f͚>.8w\D )AϟwurSssWU$\ϵ45YO>156&IN.WRדEut7qN>40`ŢE\EJB\}[x+_Fv}U_ڼ*ݢiy͚Ρ ʺy͛===G\ =KvttTz*'O|WB277OMM=v옽l֌8%%e͚5]kjjnڴ_7H8PP@W|8-0xp[y#a;wޝF}Rm ɤP+HKK4i҉'Ň~(=a9M۷ɍm"y͑66Æ ;wss#VVA²2[4gyf'\ t;hr m-:vl<+)!Κ 8pdAwrrDYn޽K>CnQ״+axV((%JP(O>4H!Ie[f jii?~lO_p§On޼o<++֭[ T:T| Tv۷:rϲU K?XupϼwJK%«Rm Z|wZܰn}qW2d2cbbbcc%!DOOoʕZ} K~Zmm7n,[Pɗx?kjh,~뭽kԛ7 bX7oGfN*x`ZٳE 3%?[lrrUPWsNo^ N#Gl'^(7b-J* nI-?؍޾}ԛ7ۄ}/Æ޻l)(n~077#Kf*uTtzzzSL5jDCᐞ$KϪf2 I߿/fݹsҥK_(I\֭?Ə?T|Е?o'7ݳ?4tu`̤S{~{ 5$ ]xD1sN޽=$H++((̔j e۶mJ_U{Rj*o[,Kh{K f%l.4=+Z371~H{xP_7! 'mWZE.3F//Qg <c{`G>ljn764}i|q{"ݢ"=b#5sN+g\)*+"y-x@SS)ɥoNRzNSPPo5~x??`WWW$cHT:::ܟY,Vee|㑸={߹sfk.A{{+Vʕ+Džf+*}$L~K?W0=ZڹRf̌], ֐L-U8IfgHj5K>kh}iKJ yӼ&M\ "Թ##7a9ϟsp!t|S^"}qڵ+Wsh4Z֭e Uع A]]5E9a/;'Ϟ=/ *ݢBԊݸ1ʕ'/Sjjh8qAAŕ55R R릯?wݻw߼ys͞*9B?,aii? d_㮮!!!G.O<)8z敇 t~yEEIo]$ o̞'x>yI䑽{~˂?|m ɸ-#}YS+a0!!!QQQ {_~V|\<-Ua[_qy'O'MNr\k=hk~'ff|#ս]/~?{EĮE_J!TW[KˠsE>.|Mư!C",矋_?mۢ9sEh\ͥK2+** %7jҥ'N(**:vXdd " !NyXYYs8 jkk_|ibb"$#--sV^MRtW\'И?~osvvuxFFo!~ҤPL1[ȋC#acu0Y̮4D1xdgO=:Z߻kV򈣣4w3/9o?}Ԕ شĸWQ .---""DI_fSE)_Scd`0/8x^p0ù{ ׮ݺwb =NN%K8&:ASSsӧO9r,c!=Ky 8WPiҳv~9sdܸqCp[f&LOzqOR/_T~=*9s5^pӆ5]gEC[#bfқ,k&vtor'*je=8Wʋϳf{{odx[dfT @LfLLLlllWGWW?422R2'0ּko=U^^ҢMjAwssSWW}0򒑑!8.jfC (((o5P ., :::;^W/6FCC7*+*~?ru_>ERTרT*[7 (rp22)5_ʋϳθoL ._|_W _BEWP(8::-:3mښo!NNxmV~s46VVllr_q\)L>}|44M[Kw(^# eXGl>Lr\`QYoX؅F>^Kߢђ%} fоLM5Hb-Jmm??6ls5ZO={<5fΜIB,,2`}}}n8STTvUg IDAT⭦'h;1Ko+/d'9:9PqQNb/U]]ڪ!}Q4NpT*ux+aܸEɀc'>^=v#F{z $[|_W Gqqqk׮m -kkѣG'&& >zٳg-:2h{x#g:g%eYhkOloc F"=mm/WƝ7Up(J_TE)_SUCPƏ=~>`&߸qՋׯWxq777("&sb 4uT]]]y,e3lذ{+,,h :::x%ƌ#`䥫'!44TWjPG.USWikkJ_O̳96ԗjhh/iFAܿc9d/j37qXcU﫾P `DEE8qBBBBv1sL7o܇`̙b]=4mdr߼N"̕+ ׮N~wWX K+b {Lf)ȐAzBijh}?Ha0}ѣCfQ%VRUV bgb-J2cCyd’7SR/>oI'77wܹ&MZnu[Bz|So{{gTxBhy(/_ }$ϥMDŷ45e,1 :SZ$UZ&ryr+/=>ZZ:^*W"R d ---""D̚t:}aaaA<~)7AkfO]`AYYJE:8X \TV=ܬCĉgΞ6ΆA8ۓzW 7yCuE_A?`ч11w**+#"Ҏ!3rdيdo9="*((ٲexY4mӦMɼ&>=;3fL[23˫8ߗ.M;}: HJMm&}}T7nAHę˗0!odӥ!l7t $ |K'C󍜼t^nQҳ-ﰠ@uUK6);=M355YosqqouJJʦM:pX"$$OQӹM lӧ22*+y:gbq:;4wx.5ȩSs:ڡfGۿڱCokgaf6CYA, 9߄oafƗںe^i'f&&䑖/~QnQ*aÆ3L :::>}:zhy%&.7448dw'᭷ՕU)*}`0N8! !!!;v0cľ@@љ=m9PBBaq1PC]}v@D `ƌ-q[~9vbZٳ{ `Mdr˹}+0ť> w@PƢ? '?Do* -Jy<#>~^^ TVFDeD|htTE)_SPtHRNjjj#F-((KF! 0BVib02DSSSpI611:'L4I*}QZZZDDDIIxt„> }x\rzV^aRaЉfҳRQg瑤N;]~{4&6$լ42 L jv{iQQÇfGGȪU'O02;yԩÇ?y7fCVo(bI r1a^o[ cAź[5EJښOp>uؙ33{h7"^Tڵk)yr5O\T*-jʚ467/X2d*͒z>I/ wUjg2^ҥٯk|`gfFC t(k ճژ1c222/_,..Vm#F1/¾ ߙeggw-Ar%giS@W 333śN֯_VeeeA޽{dB;-[ҧgv37l}'t3D 8~ŋIKLݻW?OOSׯWTW .bѢ~~"^Qfx|N0rڥ׿ٳg|'Optbժ?7~45u7fCm򪪇yymmA 06:N.O6o?w깫W-|\\545Ux靜F >y}Jcu(k YlРAUUUÇ8P:99wL` Oϗnj#/&>vچ[[[ǻuڣGXl… Ż@hΜ+Eܠ }r 9s֭,ROCYtutD~Xg|1%^xJgn:/'Knc~z5_!ov3ёOGש9yb}JÊEھ]ϞfnN~ ߠb-JBsC%7zh*_e&J`VVJ7q2 ^Tyxxfeeo0(sBBB222z""77:(@ٳhaf6M'h>q^pp¾߽u˖wj4Zo'3kgq#Q(_|%:ZC]] =ꯩ</wbl* t(k YJNOOΎwX\\\ZZ*x3e "8~qYaee%Şr%WIp8{ >M4ĉM700G;@?bbCL8ljsLuss }2(:IIs>to#GZZkBٻy3s9WEDd: kko3J) =s󂃅~ڕϞub-J"BsCgggWZZ=w|ssɓ'o۶mҥzzz2ƍ|gΜ -,,<==_7}+VJ;BA0̘Xj"rww?t萵S/_>`.踒EacR/7gэ H==r̝:i 33USSSs5*wK;"}r?#N =da!-{cK׮UY]Ɔ#G>aB^R L΢ )++?v옳,T|*...nڵ ª:>>MQQ(K .K}}}}yG!afI"K|MSIǏ1O9{wQ)++7oq Qzyy -|_}hݺuVjrc>b0!!!QQQbfddd 7 TgcǦRX޽kdd$ߨD믹|L&3""޶meWv޽VcR+VOڵkWnn-[^G/<gϞUVikkTMMUVM6MTSZZZDDDIIxt$U7TRy Ғ٥@Qh4;F7HR!ZZZl6 ?nϩ:t˫MSN%&&N6ԔFTWW߽{711ɓ' cҥ;w,,,|ʕ+'Ntvv6mMMM 充>|xWP(񮮮Ղ-_<::uц:::퍍 ?u9A~~~'T ɌeX~!G__Bp&(BCCW!> 9HR-cƌ{.d޾}{ LDcǎݵkWddG;::Ν;w9Ơo߾ns]3, NHHt}}ŋ/^(999_]ŷ: 333śN֯_&O,--.@Rmll(bS9VVVvvvׯ:-doo^&&&Jd5~S8sSRR6m$,.777sss\ɓ (TREeee7nxzz*J ?pРAaʕgڵBHɀ۷ï_ǥOR+yF;M |C[G11'NBHHȎ; $UoVWW3r MMkii;P*HRQcǎmjjݺuݝF70[Ǐ|rozxx1UV3fҥEEE}\J<NNN>x_~Y^^.:MMMUUUo_Aڍeώ_<ŝJE%E HKK- N߾}{XXdB177W?G*JMMÃ\˗ɔcTbmmtܹQK͛wT3330uƊtVMM)""b׮]t:+P(%K/HPƍiӦ:U+&EJ/{-gN0d黡q!@nL rww'Yg.uuuOOt^ /_(#/~~~~~~/^LKK{qqqq}}=4000228p?ìԌgϞ744R(555mmm:nhhhfffaa1|p[[[QFijj= %K,Y"--֭[OF 3EI-d27lPRR" aaa @)!= 5|cǒK4663g{o"fhqKjeqU |||l"^ iӦdrSҳCF:::H(]3>8jX۷g޾5'#sqqouJJʦM$Cz$2|#@zzP(&~=_0zJj+ #$$$**ABBB222$×@55 &l ɩttt422o q3gE%3>.`F\EDD7No߾=,,LQTς. ƵknܸQ__/@JP(6}^[5eP'{L&sÆ bfggg#7 @l(dxG:::x555W^6l#z{B-mmSQQ~G~OEI-(((o:F[~}tt4 > =8pÇxl6C Օc Ar()S0h!k6447:>>MQ gH&N8|G1 8zyQQmll $H3xgN'9@..(1u W ٱcPYHς^066򪮮~Q}}=oᔖaÆYYYh49 }' -.P2-#(dG+/<~@R"""JJJěNӷo&٨Tҳ PRRL~O<2d͛7qA#GDI-s8~C@j IDAT)Uۯjc2111,K:dmm-ѸTҳ@T*ʪɓ'G󵵵---啧UZZjii)5Y,Ǐ%&q ڑGHU q#߰i_`{$w-'Lrug-%~ <<<33S4mjj@K6XQQQXXXSShKK 7O\___f577͝;wfffYիWƍRrUyku)RS.m+臨טJTmjϳ$(..nڵ MwsslTEw ,,,<<Fg *൴۷I6*r|H%eP(gUTTz<- Ԍ^366p8aaa'oDCCc֬YM Cz@K8ot @!= b2111bpww?t萵DPQH9kii|((jCJ}}Wnn.߸sdddhhK!= @@!= *((o:F[~}ttdPYHςbUWWWUU4778KSSNtmm^ݽF3gҳJjYM\\ڵkěnmm&٨Tҳ?jnn~eMMMMMMkkk檩tnNlhgwu]DDĒ%K 9HPZn^ |0'NBHHȎ; $Upn *N[ZZjii;P0HςիW `gJzMWW.ٮ]V^:Bz2Z@z@?QRR"t:}0F՟1k׮UUU;P}QS}r5566#llG7\HUEEu UQRR"t:}0FbCz(9Ft?K]]?`Hy uwdfg.).}VGGGeEEeEEA :tq'I-Rs jjj>>>zzzoСC1̘X% h\'HܼtǏ1B:;;XXXIt{TK^_Www2CB%l{wUq|GD45TDgUL[4W{oW\-{K#+2{Z)n". 0. "0 _s=gAΛϗ:77Zj:=====>t-ϟ]noog͞=[U/={4UpߠT*|O?S3ۦ<``4:Y2]_R\K֖ yx^a ig9]]NNg͉߽ 3K_+Zϛ4G-x- gΜiе7Lh5r<44Juu{]SSӶƏwOu}~R JKk_ZYZN5ZYC:J XXllj_K.,t6{54:b BY0Aɷo~Gߦ8ijNG$yOQhJ?7 XXY<"5|hk;koQ(HH;sDuvH$͜5taVijݺuKs:dee͙3ѣmz7E"Hi,rijdz&%02e@S'kZ]4fa=lT1HGSvJvJroB{ 蘚)7o޼ycZvoq޽]ޯ_ 6xyy*xD( +۶j 9g?8!Φ',/-Wտ򳕥T7 zk׮'dgg]vڵǏ=c.{?Įsnl;yCV t)٫W^zu=\\\Z5ڵkx{~mذ˫m иlVޝF{/Ws#Z_?2|,hcGSΟ;9{'8oTKh& Ms'3|1 V@']rΝ?>… *++pƌ/rY3fHKK#>4(,H$󑿳] GOft61](~e戽}0s sc$ELcEyJZZ.ЪSR+$$W^!!!V~zӒƌO?.֭۰aniOHNoϜtij ͢Q=v( ͑IZvWRXb0 Zeee -rwwlU]]o6jǧϞ=[w({쩨0v!:+ׯ̙W߉CK]-‡z ͢QݻwvHscgcգ#9r8i{ƪL!ZuZjX 󘛛oKHHׯkXFcXx09=Zk:Q2ߗ***֮][O```pp] ZxtuJA&vsmSRR9X4oXf eR[;m,Z0+WL4RYLLP(o:GPԵ4hPHHHPPĉ X)0{ޕ+)fQ 3"PXmtڵ|Ν;w\p|||lmm])C< 6233{tW`P(t1SV]{u5b=pٱPRee]HZ0իW^kSNӧ2ڄի{rfffff槟~*Ǝ*G) P*hjs;v'%iͪU _˗cbbbbbe2YmKAN4xt]mfru  MO#;_m]\xAPwcU 8y byƳX+BHHHHHHXhQWkUk@ hIBRLNNNNN~]]]ryXXX@@C{V ;O~+,*jOz/6bŊ_ؽ{ݻ׿H_&wnR(YEjOCOP?`?1&XXXwm][z'jF/D"8qj_?͛7^knn>xLJjZjѨ|p\>y>C< ziZ,e)s'_Щ*i9Ggȑ@ ((0T<SWSSs1;bv`۶=zRKܺ=1/_VU]yAR=רws~?07EnN8woXFQZZyhXS7C3.%Ν7ݼYdn>vP??HD(4:FY۞9 &$e64JP?~_6331b\.k ]R"泜WgύQ C NsaUEESgk]&9==9=mc@T:fe0[yy[w>vo+˭,-NN5jÆyrZ}>+tf˗ -U(Jʪnvvn..|p2dN~ak״GH$ܥKq{>z\VV~aaMMMg J<@J2"),*ږp %ŋWo޼WZ* lm{89k/bq '\{]7ݻ#1㊲2cSTu-z1i$\NK-"]WC>vNQQK.-E &͜yWf7bD3W=|xZn߾uڹS(>ma1ZZ}il\\eU֗_zuϡC}׿?M`;׻ IDATiTYUuQ6BrΝ-[lٲEL&ԑw@@nhnIݒj5 ?^+[FFKYgeAֵkwz89#ǏkMj䚚>_74t…)&||yK vLs„uT͛!-KRr/9 */Y%>UW6\v}]AMMMɓؕqJ2vE0Z"+(((((h[x`R,u+UIU׋YZ*Z@dok#?^ݿOc.{(k+ CM95[z}YY׮i楝ݞ=q%ϾZ֭ݺݷ:=='OazR_qBVVrzz3;[+VG]kok*^kyjW?rd]{l믵6w[{njqѣE"Vh?:{h=AiYم˗NsލN؉-Fy$pD=yx:x555ƮT* hFLsOûywUI/aZɳ9d GkuLSSgL%ϝ+*юYVsA7u뜻wo7n n?'OufoWw^yɠ[o׍ܽrڏxu"II^kP:f˖ 0@kvС}q֬;EE=Nع-FyD(HFJ$wyCr]_~AAA&M3v9 O=w?EQiUE빯 EEٽCH$6̭Y+0U ]dɪO?}%<|҄ .. ]]T5̮s>p*9= f4n9ʚ7>XkpZpUdnnOrCcG͗dk7Sݻ̓nnM]RTR|j hu s><3eedl߷o۾}Wo0v]пQFɓ'7N$"Y,"`j k9rt=;k133 Ç'ߦATΟ?m٬:`Ls{k7o^re`~(+kKÇ.l.{ds:-]jmeԴ/ZӮ]u#^u衖\cso/v_Hpys ΍^NSS;H'Z#cTelml;Gy'L@kt| VkցT͍4]8{V9޿Os.]r9͑=XQݺ} Je׮^w555u#6[r܈_xu'9euKnup3ΞmqڍG0@ ;׻$iWR5knƯ_`Μ>Mo'hԨQ/NLLݴiSdd$٬yxP(ﻷ u5nU;qNIݡ{T{cQlERna6_~AAA&Mk{{[[YUhl79Y=yORRk+KyӦ}@*j#O5f##a Ok 5uj˾y≕[4%>[5^0W,K W{ki>Uwx9Jew7h6us[Lv-dfOMy#GRkP|~l(/~`a!_W^>yRvv)Snuu7i$???OOOL.e֍\8zcF.CA6`xb8$$$,,L&R7oޏ?93r66~ؿn@JJMMM= GO Jc˶7:zVc0ӴFGj?vrp(*#;~iZ- [57W5ی7}iTڒˏ5X)8uy(++*);9|޷Xk'|ѐA eedl߷kefgO}}ο/YF<޾]o6YYY=XY 3vf3z%Sh]Wڌ`0y@ PT ,h*5`FY>>>g20??xV{'O9R+ׯgfqT*3ƦvdFo-M{7:rݫh+WFFJ$-4/IURZzSv<Ӯ],ss |2DZ'\Ǎӥ{pȨ!Cts[Lvj-_OHNޞ:8oZHVV7ZxC@@\.`B# :g+mS]U}*ؐ9?F64= _+Ph;mZ p^QÆ k@***-[sΕ+WNR֭ 8pjU߂STu#k~a V>co͛/\u Zkt_\C['V~+7yyP_F>''On 7nkE{;ah=\kkt[L򙚌}D>L3-.{w%%.(hYY?|իW[z衇}}}#  Kkk/oĽu#f 8ܶ[Gg{wK:9=Z ¿mVտIsڲ3gl@r势fIMM}.]ڣGcc*꥗^Z~}ked27מ=Ǐyر_/Zuڎĺn..2YFlDx֍3m6@ P Ww^uOh#֣TWXŰVɨf]v؅܇R/UQQar JTΞ={֭fڴiSNNٳgfЊLʺTҲu޿7 J5OӌpVucnr*JC6jei55>v'.hhcf2)k++'\|&L(2͗^ڿiӝcׯ_0gNhxfٳ>3gRl֨Q/^x^{lVD,4u5PxE.khݪX+4tlVB~e˖͚5K("C8uTxxu묭] `O4G~IHX?&>\YUUwx@@ tw?U{zd~afmC||,hicm]RZZwض>F3Mۆyo lKH8~7f޵~`o_TRRwp]JrWB*չIUz3"lmlroo]Rk'OֵԺpc=o`Wj4Z(+000$$0V@< 0Y!!:undO-ݭW7vX# 0Xbΐ,77P\nffٳ̖f͚+W7N_EvX#G4v yÇcmYjlml&N;|"0_ԾVTϝ6@J捞<Ď KJKU*Yv^ѱU3Ƚoߤ͛_Yt?k˓ϙoFwgxͺÊʷ_yшSj]bϴmUU(!9y]PpܹGWJhi۷o`___;;h3`,C&?9RQV1X%)+5'Ml݉V@flH$#GcbbOO8;;8jjL& T=y[FF~aakJ/Ldhj`慚vvLST GH$YII?ޭyԀv6<ݪo,~Fǽo_[VM/oVAܽ̅4,9=]VÇ[n1gZ|晟׬wE M6EFF>C.mG< 0eBpƳsoubv8uVmڕ}栳sgc2Zѕ+WRiffly戈{h!!!:]PDEEI$7F07٘1#?U7(27uԀ;5OjХe|mF}HLTTiH͘?]ֵkOJeUUSWy5@g@ޣFihnbZNN >|L]ggkO9_ sJ iprH+#uYhkt.-22rӦMׯ_oQ}yxxK$ɉJ))):Ncx]yZm.^?ߝ;˫y{k([XTݲEk$ϯ嗷S~tcǞ뭦.jۻ7g@ij٣Kn1g L1Gk×rK=E{]Q\K̏ZfS}Z^YcbbOޣG]iii111κ#RSSe2YxxxnnShT_~IJKS57|x;߿Byάk>]jkc3s͑ʪeV佇k;%9ﯽ5i۶n..Z9_}^K&hWTyn1g QN})r~\1'jڹ+ϽrKsP(NYdXWݺu+,,,GGiӦ}YYYmk< ӧO/X@$2Z􌊊WVsB@ݖ-޶bj`` B3u_nڴ;)I9S޷FFj[SS{ﾫ5y,-{KS#F_0wȪ ֳZ#k~A=:׻$)ΎxUxO4yZ(JlzkIa~ow7p˟K5g<;o8oNX+#rT*lUun߾;<<ͭb8:::###7P($ƍj^`tO̙.T:z͏?nNn1g ө.NgNNc7+)؞SI yx^웙ϧnU'D :i( z!ҾK2UjjL&{g?CWWW}UBaObE/9ۇ5sSF/|~M)o䓠y4CKUĬ w=z+-vCn5s-vuݼzٚٵ=}ɣ}WYRk;ڹӵgO#qrtQ)n޾}:3\VVEe@ pg43/z}psq;w^=b BWPpҥgKJKs]n1g S30/[߯y^KUU'dWd?$VWV߸tW\WXi3fε@Z:z&O,}}}b!kk iӦ-_|͚5JSح[.\pz!}/W`zpU3 {Oj445s;[ۖ3v+W/YUCEe}sN3$#Z>˹uk?4'7?ܥK:Vn_ŋ=\;׻$)΋xu{?M;c{PRP\= ݻ9t`ae!һ%EEwfQcBa{}?퉵:f6  7n\Gh>^ ETT-[6k,~nKӂefOn..&}ܘ1uђ:Dឞ~~ǎ [x >g\iCmgΓO~a_[⋏9xcrY4t#F=SQil ر¢N} wmL3<^+Y0[nCp’%Ku\l٬YZ)i ڈxڪxfgmG* ?LQQ^^QY.lmllm{trdavS&#...!!aѢEgϞe5k֬\rܸq@333Y,v6v!k@wr<--[tiAA.Sdg}?tuuW_ P(<V9ǟ@gt ӦM[|5k5yj:66v֭ .\xu*988BZ-B~WZ㵃!茌R(QQQdƍL>h*@'fffn*`:gtPqqqDǩrrr"""RiJJ^jȼze*@$&NdB`:Ciii}ҥK t*55U&=~*h,,, B0/VVV{"[B`gttӦM[|5kJeRձ[n]pŋMGPثW/zhs!!!:NP($ƍ/YG\\\||D"qTh!򴴴gggJMMdṹz @YEddӧ,X tJVzzzFEEUTTBij:1XT "**J"lܸQV<#yxxK$ɉJ)))z ʈg\T2,<<<77W/],aaay D"]RձQQQRgXT "**J"lܸQV< `<<<%SDDDHҔtijL\.OKKqvvqTLG< YXXDFF>}z"HjullgTTTEE*L,.A,GGGgdd8BH$7nTz)0IijxDT999R4%%E/x@#bbbu*55U&6,,,"##O>`HTj:663**B_&x@%322BBBtJPDEEI$7jtvij:xDT999R4%%E/,\.OKKqvvqTLNx~gaay D"]RձQQQm!++Kx!!!:NP($ƍju/wءcq@#<<<%SDDDHҔ_%fΜ#"&򴴴gggJMMdṹ-9GT>e8LEddiӖ/_fRjull֭[.\xbkkfN֭۸q:4u_~E*Fyyyyyy555mt5>`E< '_x%K2BZ~ef͚% :СC ⩧ڵk[2uuo߾KkJOO~]L<<<%SDDDHҔ }QXXr̜J6 R^/Mw-RI$իW aA233>߿_~=ztЇx4qT+>>رc%j}O֋?T+ţV/FF=zѿϝU* lf44 -D?oݺf͚5kdݻkjԨijPnTZUJHH8} D#Zlɦa|#G;^6k֔cSX٤Ih.]4 裏T5pTxʥ*Uhq…6my+|rƙL VGFpAYFY:g<`^Ϙ۷o߾]*zyyijҤيDgܫ^QFk׮ׯ߸q#՘5s"[FT)1qqBC~h#V U `HBKѣG-[jۥK+++>=3Sjg纶m۫W-^\V_|۷oܺӴ\'UԫUϷi޼e%9oݻٺY37o ?s&2:QbRpwt˫vƕd788+QQw>LMKH$jnڠA֭{t\bg̺ᮠ!!'ΟOȰt90JnѳgO*]C*u{mr%Vm&QdY"s'Zgg QD"1̉;|? JHK| իJM!Av%qϹ={Z4n\\Rvf:u"nao?Vҷ)ww*E~ײqfeH {=~쓤$CF{^%?.YĤ$_sG9iԨo>\ϬV?wND2l/M_abrѼoݲ{wVvֽ:w +o=gSVv3gJw,M-wn(LHHi%D"a,d` ]ij0$q6%B{O<\]xyqʪg΅ V*vEfwAt`?ޞ,0d^~YV9RM"+kĉ:٬z¦$7g_Zs]~3{ꖺ Te6,Z6Ͱ͍J7dbg;8xH(cKee"Uohԅ L<!nmٹC'N\~!inÇ_,rdRYܴU&M5ER=}zٳޞ,H$N端͝gR;"bފ]mԻ 'lz'NؠAooŋ?m}ڶL*$ zN!GxwRLر)l)G{{+L96>{(]JEz0iyOlעV|T>G߸};Wn..-5تUB6ZzT[߄~~(lD"qqrw`Yx[3-G9mZͮ]{ ڵg/_VT. eBnnc>;֭[ɓ.|pC0!Tٮ]ڞ>kRJS{FM1bŏ?~lx;7goSiǿK{^.޾eˑCT~7n\aV6mVؔտhso|{YZٓ& ׯߛok?}|Ælx& ̖Zx'N,\&r0,0g Z:%>XM|Zի:̻f ծ3;##= _P@.NNc^~YܣR8 GJR#v_ߩuk/\z}qψ2!YZF\jp&t/&tjmϴ˿VEFYA LΧkWT?ȑ>tn+|@*ȉ/RӿG/ݲIhm|dJ)m6J`z[*3-/4[jivպrƑSBC8m܂U*U*:q¿yU(cNo$xk7oEG:sZ|| ӵm[~ÿ=sRJjZl8Sw>o3fcmozy2lк81W''q3=##)%L!-r.Xߖ Lˣ5ohҨQGD;|xOpbfnܾ=wSʕMT\axL&stt4饍/j۶oN8̸`S/viFg؝oܾm^.;wJnn DݹӦY3xJhǏM  (fY[[li}|||é.|`uJwpaBC;R;=|x_NA\i/:;P~իTyPEJ7q6K 5|-)95t4Õd.yw*emϴ¨W\ޱc[f/W5k/Md ˾wߐlVݺu'Lcǎwn۶ϏlesV^Y{4h?WRe& ]<_4F#&.ܕ+@!]4䟕n:]%&>>#}(Ԍ]iysf+_oK|˫-w؉;OڵmyՈzu$޽ۻww6@*,YŋQQQV4h9DagDEG{x\ZFFةSf]5{yڛ7\⦭!_Ϟ:lXtJʤXΏI%\|-V`m柾ޑm۞={`ɣGשQ#ȈW{,#]U`6m۶3f yNڤIWK'/W_{[fpxxVvR> mڠAdtyGݵ#='q@v RҴM1]ֈu{GF;xys \1)%Etrp:~|IJjPNI VMR!3^.էk}qj`XX@h詋Jf[^|twulwAʼnNJݺu۷c ijLjרѱUS.h{gRVrynݴ͗\VYR8rd+h)ii'O/oa5?~mT*+❼Tpq) %ԠN1iΜw&$}_7h7{j&M*}(kTZ-HJ`9z[*3}h쪕|x@hc?>ܸ?,%ܹӻw{ PN߿W^ZX!0 7xR´_MА}eehw/5kJ2AySvAGnâEN7^Mͽ/JSD T~cR!U_ߍ/jVo؄Kf&:tXfMTT͛7WZ5h Y,0aQT|?{VSѐ޽[7k& vX[3wH>ϙ #GAH8px]:iwzT*kÇo\H {^4)+;Y[9j̨k۶:=2Pޖ LյiÆLZjh*P gY|%q3jW.n..: n #G2235Vչ{w*>3QAAII:= n ng'̚UؔBB<)̨Kx֮CJPz[*3P:`Rs阸8myXZSyȑ-wnj|E}t#degr?.^@KdzA0bSu:ݻ_ _J[BMUYT)T(Xam@@< ^|L ?m޽ڦ퐾}OH$m*ovQCYG~~:u[:Uħɓ|X]s-:y|'ӳug3e8u[䌿Tg #6vPq?ܩmٹ:[["W6;jդa"ݝKR{.\3+95u_{NW ÇyEڵh!>?x`^TS 4m˫EFZIo܂m@yG< ̭Gǎ dh?7%֢q㖍8K^}=zժ➔4#mjz`JСփxQ\&kj+;c~Jleeҥ66δקLXBq 4͙3۞d:;Te䌜6 ]R\ؙ3.YrTg )zTI$7_{mҥrvtg!C>^;گ0^رRvN5ko٬Aii>8Ej3/\ 4@hh*UztXjUww[Ǐ#oތz5%-Mo]M<|-(׈gyy+Vl%eR lRtƈ /Gǎ,_>~LYYw飇 Y'_Μ:n=4Zz@:G 0Ʈۺ5:q:xP"o,a1ɩ_|]b~3jTTu:R!) xxү_QCXJ>95o&WgNw7.^lcm]܉fot*U*ϜIL"|7wO?5BWjgI/6G]*-(ge?vNOڵo_qCܹQC/?nP++]lb4#eN n'v-:zu`ܸw޽zmd&6jȐ+H u֭¾*_oK|#7@ŗ{YKWIÆD[xur\\SS===K:S{ϜI-eccӪi:v̖H VT;wuF=K~A5W/pw}qCaa=gewWMtlݺKv.֨Q=<=>ٳII soҰa֭WxH223-]*>ɡCm:׽z6k9S۶m(D~D#](jy9d92'..4T*!-͢Z}cceXJj׶+z~811=##;7J&rwժ9:8X@c$<~0>IbbBRdR\VJ7bm%÷B>S&[[A.|ZeA9$nZj*H= DRzZl.LqqvvqvtJJU*U*R!)Ϡ`Y8DH$LfZ@y"V9Q%C< D"QՂ `gg@}vL~8d2YZ,](Ǭ4h`*Pq@ҹsUZ P.t҅pCT( r`TZfMLfBP@E#HVZ87 &A< Lx,0 Y`ij$gI &A< Lx,0 Y`ij$gI &A< Lx,0 Y`ij$gI &A< Lx,0 Y`ij$gI &A< Lx,0 Y`ij$gI &A< Lx,0 Y`ij$gI Dbbb,]@E< @%''o߾}?իW-]@ecj:"""88x߾}'OT*rӧK(g'999((?(((>>^/˷o޻wo PP[nfdd|kgguVY6Y"U溻yT\}.={ةD"1u)([7U*FZ{5qha#Pӆ_|r+<}{k.K|Vۿ6|T o޴N'ծ}'5n䔒r;q xZѿ@%E d'W'zd+&3FU;CիG߼~N]|/SVij ~.9}m6ojZMi+CGkge ,ҥKǏ7nܖ-[fqMzwl -;#mOf~^`hܛ }uiϛ/l>|_T*]|fiXYYMow*ҺE:oN6j\u?2=j֬?%–_6j?W\,-[;%4hHq[ s|\ٹW׆3nn /DO2v3e˖;w=yϝ;g@Y~Rsf`>D*^.ZYY;9; hckkkc  ..Df'-(Ξ>qڐ@7nvoj?K$Of3x'R(4=2mfa!szcԸm/`ʇ疜M366{k)oȧtoد?mؼag;S&M.ɊTK^R`;oOgNsIY"|rqLt#iӦM66mZzzzHHH@@@@@{,]0Jee ^Df N4Q0qD4NͶafQϔFhڞ={ջ|񼞉tx;9ų[my,nTOG-M#f>;P!44$,=(g 4h A޽{СCX4`nJ%s4Q0`A Ii` \\ʈ׆Ѥ[<4壒Z9Uv+V($\n_KixWx7Jf<1שTq=y2\[ڸ%%;}Z.DKC@@< Чnݺ~~~~~~'Oܿpps,]0ijɂ 9&Rț+D~镇1sp)Efigg[䔚5kY[[kׯ_-a ܻw[Lx4jj2|GKN.H_3qVmZ%ꜝ]j,nK$F7ԕzt s…lڦ{A4i|&J2-5Eܜ4ARSSTJevNVfF IjZT(Z,BFfFAlmmApttwU"hB`U{{{;; 6NN 899Y[[;ӤX'˧NT~\kJjO&˗.hի׬נA ,[wѻwnioQR l٪]vv=s*U;t($bR75d󍚜u捒ijJ@yA< (6aeeUn=Azw`)V(2YY A233 A EBRGBiff)I~YY,"--577~fhNyӞVZlݬY S\U7n\3djr%yZm~qh 4=7_~extrrfNN֝0}ߨͨG_|;))2GUzzdߟ@Z呯^b}MNxdmQ㦣[J2#ﰄx8Y@`K-Pfi` 4BܙOLVd,1yY...͚p?g ߳!*2e,iiI L|C7RYEݸ䱶)J_yzdvߟ5_|&ֵ/:qo\ ۖL{"w6KJs!=l#28 9&Rț+1i999Vfg 1s>m.?^-[)lRLOO6 Y3\vEӬ oM4b)S4o?>''߻c깓)Kh0mOmqf*LZ^&etqSj'5dTgXFzx`Bl`j9&2;++S!BrrVkB`j:%%Ǐ̬,R"BZZ27''7'=-]R!BJJZdjurrRjJT&3!6mX}-M3%%yԈ˯ QcDiB.5w߾ڦ뷫6X۔g]D@O_lݦ~LnO?/ \~U_Y6K^lqΰhN<+;'$53,Cp)pK})*Љ<Tj' Vꋯn;K}Y2ٲ׾1EyYYY7ݼaL&wssSթlTr#.?nZm,_vF,eq1tzZn۽g6m;֫&9)KڳORѴ{/"\rqgΞRFmaTZIJ8Kx̭n[o^ʰ?AYVm&QKH$/<5 7wƦ$LsV^}͒'n{w n3 uxTգW^^{c1Iy{27ӏi<~!Lnn}:ʵ۵do,;;yeY` rhZp'֬Y3tPgggKճqI&T*K3k߱ˎU{oX ]͚U=3?;2Fդis=۵߈{v?fN,fݿvL6so}iEs5:CY9CE" -m2cƌmAb$5j^g|vr#qNmsϚ]Q RqJ|қ! 88IV2K#,0efK-ͮZw=tPppCR ۸q r89ƱQ =p2VYM̴ao435sk 9PTTبl8λ8}>xݐ ^R4jԠQwg|X҄˗.(۴r6 }>=9V6SaTjjfyɐ#Bˇwj޲Ҝg~+gϾ}nedd4ͷZm[RRI [~_?ez" ;(%?fUw&-Q~T]jE-(zɃaZ6f#V\ykڌ s~ksJ]=KmnΒ-BÆM5_#ۡ_4T>{>yaۋ-DyP Zzkׯ_?f]T+d2Ybf0 ^YGq7,]eW1m)ϞF49G%E67Ѽgt)>IJ|Vj&Ju17? f㈇Æ KIY@5#o%H<8sLww8h͋/Pdggbd7J]\RT/ͽA0)} *jذaÆ A"fddk׮\~}%ܨxƆ:R$$<2t @WgdˇޝhզUK~|ض*yT>߹KwK &h,ȸ wyUT7+CSSӵCEdX<5]I>p/==]|A|O*;t4AB^]۩tN>˓4lxR9?)1A>|\Yaadee鹕F+mc7_tC>lӦ}ݺ.Jsjtu/^OJL4ʗJ[>455mӮR_"c'ȇ6CO;|ؾC' 4[ YOD"ٲe˘1cJ_`|Zݴmh,tXM ܼVjmNkү۵5f~gJȈ5hʛzviu=7رLLLÍs/ڙ>>Vܥ껠K4cGfM.^8:-ѣ/>_)FK5P>Kiɓ'O>ӧO:͛7~#%dai3w|s)3m;os KC'Tin^zFzd䣇anݺql6Hms_}WRwKMN,B /#(eK>n׾c'].++ L iSƮa]I"}fOGZi7djY8'==MKמ;uU\ @5By6lذaAHLL;~_zzU?SFFu$*_\\L؃{asyYZYwY[W\VPPy!ÿXWo῏[fp'Oo  ǿ1Im1žgNHYY|˳i.];9<rg_FssX통j^|gzrw[عkR% ~F`aaoLi[i2Ds>ZxGÜKjMMM˭خOo Tgϝ;=T}6~XА Affgc#c͵Y{.Xww^/Q J+1"!!^ z_|3s6lԸ=ǿ19.6_#29X?x^"3s vNړDy'lfˆ뻪o TFN@bii9lذ9<<<::z˖-cƌQO?͚5K/T)˗۹}׊kjp!7ow2f Z^}W###NRrW%\ȣA;v\zzs|u5Id o76Q/կ^[i>5<@@,[j8q?((*k͂ p!n=H%IG<<U~bX,6RիY۶km*gaa)W%H$f7)//82}ϦRկ}cGٽ-?Jw Pv7'ʹsȱ cJHPuٹ#ߚ6ɹ}tx\\u[n-66Z5O296666"o T}"_0(zN>+5U[bb}]* L{nH߯E"шQ+;'r9T*I2 W\FFZTdD-HW֧M<@iVuqqNK IDATԶaIMIɓԔg"fӦ^͚oŒȈ qϟ?762ؼEkzZKgOɊj;4lڪU[mruˆؘϦ^g6`RdGssD's9?šE"ݳjKM6M> -+((H"$QHITD-LK?.cť ppٻ_ͣ>;5mմW ʳĤ[nݺu3t"-==UYjC2ʤ/'ʳ@JeXII y~Vfݳx9QU<["*.DJ$訊*fgg â :Y^9C+!V1kk y7pspspskTWlf&"+(`Xgxd($B)DEH$ђȴ :إ{W7wwWnnnvv5*DP-PyaRCxi^^hccѠrC,gX\A',@QQQlXz< @Ւleggb9׭gjjZA'WY 9ϳo''?5t"eVfVژ[X:P7211RXP <{ߤ$:;%Th#++##X qRNQb: ***2t Y  xbC'Uշ6KgnvЉTqѕ¢IfeX899׭ "#KCgY{.Z(!!A1إKCUYVVSWVVSr23##WfC,W7^BK+kC$m (< tTz1-,,jpDN b*!e&M%t,67 k̸ 5j6@Ow2t,x4ⅳ^ s+2ReX,Re6;w(gg|s̢̬Bkkj7hبM{/Y"6646&YTX qH$R6*[[: P!(ʾ^*mllC?122*VfL9t`bd݆?GXMvn߼`,`N^r‚ѯzICsT8*#?[6O߁_ݱ|n,k?--GRR= ;spq?qҴwg|hoP.WANN ۶d,JQPPgU~C,KKF=b9UЉUY@Y^}T˳233BCy(V\ uY7_S O ^|LvY'M[ՏVگUuO?|aڏ-\DK"ʰ=-((㌌w!@1~ujfnqjʳbcnS|sU5أW;޹}k8<7[YYV.;|_Myt/aRRbeeݰQbh@yP&67رkųJׯM޽;RT,kIZZJJA3s]D!'~9kk2-LIy6a{wos:sm;o}D&DGU潄4Y@g]PM._ѱUW/d2`ǎ]eJF!!w.u[_:v!?=%%C:xUv%yÇa֣A#X.ڗT@Y@ _~JHsvvr˗ΫsY׮ ٫D"QfmvjުaFn ljHq|OP{ :wY3ߝ6y֯>qfu83] 8WPP:?##}ѧժER ] QYNNNnny%6nAZjX39R勣NfĄjvlm3rjw߁ڮALMMoyʦ^=׎Y{bTgYa?m5ΫٺM~\߲U[`wg=}٧ ث0!!g]iY+5JLUЉv i`g%z[>|]TTT<ݽy-[vssÇIII999666nnn]tӧOB&=~8888$$qqq陙R4??_,[YYժUA͛7ҥK^lmm^z5<<<66633Ʀvڍ5СC߾};w,ysjȨWޭuy֥s%=ϵ˅ƥnq`:N[&9} z⣇a{j!JC̙UxNn>Q̘蹳g޷sIZ<^QQQl2buOC,*,qqq'N xLg#GG֭A999۶m߃NxS/Xsez.ŞJ̌[;xmrB`˷dSGFk_7uhՇ;3˿^\mfY$0ѲdޛܿYوhzTdgg>~P!VBT*W}oC,LMM+D,_=z yyڵ)ߑd;w\tibb6J#G9rd̘1k׮SNW^++>m۶?CMѣs΍r~LL̗_~v%K̝;Ĥ߻*!-----MmuN }D2L1-Dy\s! ~ڔg?oP 4pޝo\pNsyVx؃;vŪOcyJ񸸘n4]B9*,,!Gy/}wAh۶msRSSyǏ\:\OAAA}>}?`aa>ϟ?5kxlٲ{߾}ej\XUvwnT_87e{>z9[i9gӿԅ=vjyV{WoJm |v7{[FyVeH0433itEq镛[+r0?08ʳrSjOLL>|  :uꔷΛcӦMׯ_?z˟={6t۷oCPPPΝ9R/Yxe 8TMyֹ˳4lX֍YVH"+۶QQBmg)Ok]RXXxpqo6mּ YX[s+ݽB7y!M4SЩܼO@AyPnڵkľ}J$=O9r7J7wݻٳgO.55uxFC=}׼UA@U|||QQb3u)Z_r߀69jpa]Ԯ$I"]՗g]v955E5ĩ01˖|?}YV"&XK< (EÆ ;wܡC//&MYZZdee={Ƿo߾zj@@KyyyF*672dHkժ}ԩ]vEDDu!{===ׯ_fMKKK333##$Dr̙ EDD1ܹs666ZXXX8i$ Y[=ztf͜bq|||ttcΝ;WPP:?==}׮]ӭWrys[휜^0--5έ6mKhU*^ԩ[6_p_syօsjZp VFۢLmSR 4@i4ܽ{~R;88;wnYJ{._|joꫯtV?ӝ;w7W_ }'NhÇKv.]Лi{155U}̙3;w*W-;k u ^8ʥ ܩb0,4$!!Mܾ8xx)+Gǚ̒>lZQ[[϶(w5oFiպYЀ,-YDSڵ+w&M57Xݼy300|R,YFΟ?߫W/~'n|<9s뒪=z۶mjg~g/=*U p+YY%-9sꅶv85j)67ڭL%{R58hHgت?W= 11^5Kv_͚_CyVޠaj* (*VPPj|ժUmڴr%K4iD5o>ӎ6lЃlȰ拕TN;يo#k˻Sךj?{DIHwvm?SRGOjMZ&ړkRTYeZmK1(]y{7pshبʡ< @aaaA///vQ[ۤ$vӍYjӻj]vvyFFFtt>;־OOS J"BBzy,T̫bhvn߬<}byUNoϢBԩ㜐}һ@} .WILM6((,LMI.,,sV[ђN11q-7Wnnnzâ< ( ')O>-JժU+ղx͛7'ST&(g pkW/$+>z ^Tuȁ -;sBiɐaIrx6R-Ϻs>{*5^*fee]>B{v5t! *H"""%QQ1Q2LS $QɩRFXZrQ2*Y@EIN.ή#ըQCm<))I<ٳgj_}r077777U+11Qmj99]⯏9dȰUֽ]FF<~ 6TvvҒ%hӶsJ\gꖷlHm3;ZhVҼx$B)DEH$C@jK{nnnn ==ԫ< (999pJ^^^%"gcccllZt_? `gggX\,, yЁAITDЭmy<¥5oZuXV ?v`bAKi~Z[>ϡuk_+cc^ԥ<>qj{:솊#63supux!--5--UV[{ZXZytFyPQT;UJ8EN$Y[[+>ْJǬ%(GߘT%¡{۶$酺ƕQʳkپ*7;ĤzSvviJ=ԭ<Aɯ,j5yMr׎?'L/իi;𥳐g/^8a25+Z7jKs=SQUij+IR6)fjͼZ ㇕׮|+hω~^jeLHiefn^R&M[n(*ar IDATj)ǏoyVXXxݺuU&&&qqqJ񐐐t]rv/dJ1wBV[aa!a!q1zK-PTƍ*CCC L Q :88ԪUK|///7oí[T֕|ϣ AT}ߚTE6{jh7~G sKU֨KjCϚ+/qWKkSl_{w}W[;1T& bGD<43+cY@jժjy֙3gVXatʁeKE tW)xU}rPm۶v)wVVVJN|#G%M3v6?Isy֔i˜\0g/>tF`۴k{6lj*NJ9}9|{:;-8^B&&&_X\<<< @]vߕ7oLJJSARG~~U]v-iI^k`ttKwCهR[<+++3*ը1olۺQƣǖra1/^^-CB}ͽA}tҠ7n5~Bp;ohkkuǁZ)AË -(P_Vu$$<2t H J$QiH-j< @}D2L1(6nܸ|rCe3gΤ[Ғ=zۧ)N?ԭ<jFnzV J$|y򬞽ծ>&Lj:mH$1*ήO3;zn}a2.!qBcCgeUVfMzl{,??,U(*sn.]̙cooot#V\SB+SSoܨ\ñk׮y5k֬L9ZJ5ޢE֭[ir L̫En\7hϘqWLr[bX >~=:oҢesuu/2ӾVT 5eՖ71Ss# PD1c'v˿STqSNU-JOO? no0ajyV~~ٳoT!̙3ߤ5jH;ӧOݻעE T3fgԪԧ ?ֶ:հroۆyeZkii3o@^[X[:Lb ׮]3gjxٚ׶h_?|R< `k֬2C}qiӦiIW^LLR|׮]_}AR*sV 9Nln^}&9M<>'/56THZNTڭ 6{vm!K\wĩ?fYϪڶp?%%Љѱf EPɭ5\XZm9:lӶ7) D٧OJM0(X133sۭ[Θ1CCfT-Gj7ԩ[l/^0aBr_[6m*uKRՇ-ZrJ###;9r7T͛L)& zsN{n۷=<<\g2L)\WN$5sBC.^GⲲ2,-j֪ݰQmuǻSR1VGϳnݐf^ʼnMmy[XV`UPŵRʶZwQQ 68ya3[6x6>@(+Lv ?g0W)H$83^u).DUߎ/Sn߾= @_gnSNݻg:=nܸ>ۻiiii_~/ߠ٭^Z<==.]b Շ֬Ys bO>]x;>ڿmj*ΨQT˳rrr t!//J$m۶TpϦ^Mޛ%KʺkކUV[R41!NVdģ =+S-zopY@D۶mԩӓ'OT ?Ν;wԩyR4+++%%%***44400088R_~g PTTw޽{֫WoС:uSNڵSRR/\pvXn]:u迉ΦOf͚Qӧz=ptt<|pvJZ|||G6//ݻw/xEVlHoՠa#kk[=Y@ܹ?3eʔK.鹕ʳWXxbݖתU… ={tΡM6swwy%ۯ_4_r,@AyP\\\n_OvvvRRR:utXkbb2|Cua˖-7lár۶m=zybcc˴j755'rײeϏ;VZPUEZmb'gV[ R=UY@eDoIvܹm۶+Wd2׶if&Mҭ6Kcc}ݿKOO/u9sƏoddۡJF9tЭ[nܸ188u֝:u9sj֬Y. ;//[n㏿˳gJ/xI&UBz@Rj$Qz'JU[m9;mڴ;*,_Ri5-oo-J-,,===6mչsgGGrIy֭^pոT###&Mter"X}vgeX%K.]`ٳgϟ?OD";;;;;;ggvy{{ٹS:woкM{xڲg9BG+++C/ʳsvv?~ ry>}cӋyyyyyy͙3ǀ9#X+5M,+E233 *L&۽{b$WwH$1jbe+Im(ջ{קvЎ?7ZEK>ʴϖ͏4ٹoٛ~e׎-i5mb'+vQluKAׯ_|+**2>6&##=77|mZʝ9VMX{W‚ dMߟ4g܄>[]ulغeClLgS3f&@ Lv ?g0WiH$83^u)խH${TRRuۺݳv*/M~Rjyֽs>x;!!^ޙ:nG ?h~*,,\8ǏhoVOTʳ88tpٶ<"uتҏ߯9o"6##])~"4 ?~Z=XIvIʒTvnz*3͛N26+Kj+((xrVgK*r?x{:hbjωUS,~/}w;sR6k3foT,ͽw79!ݭtdfggeRN&###$$}J^Y~89\%RuE$kD&Hs>b Vz3?ۿ dee]<? ׵tU O8ovH}ޝ1e&*E|bUT@+Z @U{j>lbdٛ׹KwSX3s:KTSSSieeŗɇ} {{nݶ}^}f9v:`G/-۴ZDﾩT%63wOVeś7#B& {m}{lD>ڭ瑓Mx6Kkk롯ʪL[ܾYuZ~&KK5aL5߯0\C+ ʞ ¾=ے]z}ǿ1ðm[77ۡBTQ <{ߤh*;wyKЂaݼ~m_V'+,,۳CH$Z .YqC}4wH$R;Ͽzs6m;hNyKkg~~~0!!.&&~}2<)&ڬ3?fe oٕ4Yln6}O}$\sy֎m[_DKW=g{32ҋ'\+SpnT,Ai,J  gN)-\˺[jJJX}]Nn|tzImmmKFmy+F$Ɨ4Y)zp_Pǵ[L,A"")wy~=qq1:*;AђH{ͼZ4v)a~~Gu;WA>PP eaJkٴE6=qaJwo+0uй%ݺR,xE R 08C'WN}צ..΢1ak9j{zn8MK]Rظxg DGG*&_b2_#[_{*]8?|h )[ná2BEK)H$jueeeFGEy A*(@esrjض CgQʳPXXX|<Ilemc8H/uIZ)z$);򡳳GÆzY/Q$QSvUS=[n+RiԉóPt_kծ]M2ƞʹYոIzn=z^YrC%'UY2qVjk23/{m=7G85 |9ed=O?rEw76YHN/~JR׹06VRR->PVtWKE"r"#RYyyyRX[vnߢﾯV^}M<= x訌 ss5kթӭ[hs7 }po҄~X[Ϧ]^p|ijٔu{.Oʳbggռ{E...ؑ#Fռ--.;K^$spt,g򡙙وݦ'wnsђW\pVwwo@}{v~2i^^ǟ٢s=\Z Qppf 79v6˧tʙ0-/޽a~aaavwdV*$ɖ.(,<ռջ?a&/_+QxIYm8t`7ͯdS/X\$).߳>;olټܜwc-dg+fڬ23{a,Ь}*cÛ?i6{ e AED2EElP)8زe 72doڲgݦ3IG~_Mi˫~θCUzg@;*VzY$g \z(BBB̟M=~5MWWsOP]leJJNNN$/o#]8oֺ5+4MXÏ+W0b*3|`N)ty IIIlR*BJrRVvVFFFj,BZZRVe'do_\oԤνVj9]VVV+T|冁U в%.A;2q^h3KOKnVQKOF!^4r`Mdۖ >emmkF߷K'NS_ Vj&6lY~C#0<+C$W?`ʳdArB.BzzB!OOOW*\!k*r5At\.+M\! IDAT \4P&L;ʕ++?5R||U jҢ}77mX_4*vղю3ilECQѰ9*N֬UTvV:t{nڰZ6kĨ #F4=< ܧ[7Ν=C]'\>!X2C %%j7LYUB?`ʳJJJJJlegeY6*UTxyuS yӆsgЎ|#{DφRSS4͉Sf꙰zZ.xPoMdgee?*VnDS[L+*hѼO,K+AXTζt WJiv|!.?;eh큽|8eڷϯh899k7cc]^-ݔ%&v׵ԯR9[jӬysgOQUTeʔHtuzvr|R5DIT(򔝝)+rAd r<]LRJEZZ I̬ԔTARR333333SRAHKKJNN`Z666N 8::YbX]d#V7lmD" {N"G[uz{;*.&vaJRy[3oYfdTD|"QRiDDsM^`C2MMy `ʳ)Z8,vx;l(BeeemJ*K+A(b)h;|hiG;9Qy_fr`+3}~EvWʳ\:?h'h? PZP֯0$,c>^}οU~թd2!ǎn6j,G& pժ/?CFФI3WAhGL\;T6kG:tbĢͧ@yUvvitssb)[J5+?rISw;"aeeee޾VV]''cΎ2UwzZXT(֮ń_ﵫ4˔)WJOwSǣ"#}[e(;o4mll7l-Wd- VIIU;qjܷ;,(b``= )Og)W|A\\]E"+[[[GA]N΂ 8::ml]AwrqqN$YtO@rűc~Z4qoo6RyZnn ]Ȕإ[Bi^[<33S\v;ZW$ɦ~9F;2`\ҥNJ?V*|zV;$*ۙS""k"MU0 d[7(rus%Bn>GϜ>I;2:- 1Jʳ}ڵCovF7KR%=/<ʕK'OIMMѧ\5뷕l qqwn4[`_&MKKK:M8e*?8tfFFƠ{aI=yh􈡏,hWano:瀁C׭[t\vݪOݴ'h;ÁC5eeei3fӳ:x/'55o$y{1pݢT@ ByF[6TPyy[3g-tuu5dΩT*Ms?F0bxHLt :ywO[#tϞ))?(zf[MKKsG_Db?kOֹN3uڬ so֪]I^>VVqW^r|VV}>lаq F#U7ӦLhܻ4ggo!׍?PPڬ*UOֶ}'猊|Q^!\Ea!FhanN"YzSٻπϟX&Rnݼv5=Z8}x^D|զtcbOu4vv+_BS?PXY:H`.o^mS!!>T|ekG0*P-qJ۴+ixxzX,1z_V}EWز}Zu/n={1|]*d,(lmmmm휝]|˖PF4lָ9*D"ѷ} B5LJʗ]ܺer9(o?x9{ӫC.kW/۱mTReG5lrwܲiݺ5+?㩻G>Hggg=?PP%XD1S7B6q7+ټEGu\.nҠa`+=3!>捫 "ӫz5jDqĨ#FaH鋴4k+kWwOZ-[ΐyxEbC v'ؘlUGՠu6.-O ge 8Yãu3R`@`+^f5Ϳ.VN^Og@< YP$("Ay QjjrrR\nckkoo),YbcN:qׯq MVuCTԐ..Ml}uK233,E>N%YheeeU*ՉO?ܷ_T}-^ۺ_~=kАO :\TܟU*JJ9ȡwREK.iiKg/tJ*=U`Ygbj˦uׯ]ٲmVh))_Nu[YY8*>>޺yu7llXG,&f7}+%%@*VRulZ7vͪPkS(,''KזȕS8= kwܜ2鋟1|HJ [Щs|{N7"έ¬ՠՠaS'Y:#yzzol,zTj6C,Dy(r"ZP լUr*=<<\\ݕJyBBBċ=s:|ԴEkXYkk54 nRZ+UtsswpHAHNJ|6o\у\1:t_'ۻ3GAkէiUTuvqLIMNIIy7oߺ~ P:n6wp=fҵJKR`6 6;X:\\Zh,@< (B={=J B,899-[600Fk׮Qo8;~ ݻnccԬYvڹ[:͢Rʷ>~>rtt.WB#GO:~+r7՛۴oFOoOF~٘,mVVV>O>޿oＯguk]\\ آU;u[7ihN[tY=V];~իW,Y{nLVnݺ5jڴif_pɓs'L0k֬e,qq9wrr*T oذ_qo6mZUJMBRmٲE;":F$xsBie#bkgE=nQRGgݬR5aGXR刿fw5*55f2J"G8AR3HO$3Jz +˯E"JݻlذY .,]>Z;w""R6n4r|AP*{iݺu߾}̐a*̬R`y h4?Ч@ٿOli޹[ՠacjQ蓝={ÇjN=/z9dȐȂݱcG&M.]TG)>`oVƹ>w_ESLII~~^CszZ& @Eyw}wb={ֲe=T*ܹŋMٔԳW.Wc*E}hyZ^Cr;R1==ݔi((rڥKZ:Kl߾{ 9Orr;#JMٔվCg`\\l|\L$n0RWOKMLl(;;{Νӧjժ]tV J2===>>߿zjhhhPCPO3tCef|T yuͽnݪժS|6ߔ((r`=4ތ3iӦW\1ij3cƌ˗/M6fkkkMϯI&&M7oެYs PT[3bZA,1jswHH8sO/vs9 @@N|M|}}OP|8; vzV|\Cs4=˕PA(SiO%ZlۧzV")J..7^lٲ\/.%Y ,YD74rH'5jnllmtYY*=}f se) pW_<0O[vv'>t`_}9&<((^?t!!!}1>T*O1c zW߾}%:<~X*_+k+kcKA+Mz677wVm}zu=(^:sL&ӍؤĉIII9{.TӍ_x^)>zݴ?hqYյ;g4S\c}\`X ȩStg[ndlϟ ѣ1R}5vkս}gy V/߼qM||ݽKqb+׿h>=-ƍkqq1N|֭/wt"(y(^tniӦlf͚7nf|@Iw͙'b{נar+{;88)Ϟܸ~?ܹY4ʖ-?Y&N۹5޼qUOstowVzZ [Vl8v英@AQ 1&Yq9;; JnWrn|Ys~Թ#ëFڝ:wkledl%>7skV-ݷgGfff^wY?dK%6KkW/7mnJʳdffJRxݺu͟kUbbV|@I3Ok}!7ָIMMz_~jRkU+~z͜٤X:JIIt (y^;tҍ{zz?K%VjjjQ/aR};znPbo](Ws:ﭮεJ?gP8= Ozzzq77Slz P(̰J|odRBoݼ~t:{>>** 7o\7xe.0D2hȧ>ZFL;ݏ=CJy-у3O%Μ>Ezvdٝ76o\;d`޿;[׷ޱ+P9mۊD"J#m۶qƙ-#G|Ezu_JZriӊzuo@a[>}(11aU7R7m$vSJ]^ZP8~Ú ryzVV]&L^s۲eKEV\ծ][7w^3,)b?sbbJ~+ 9JP$%4KNN26KW.yom XL9#rPQb9s3&Mw}3>p@ݠL&=zVJFm7odhe4_}\@IGy},XuւΖa\ݺu fffN8Ѹ o߾e˖Սoٲe…fHo@!hnߡ3[1Za 9C&;ջIP܈-PH$ &?>G\R}3d/^|7o4.VZUX93f̢E@ӧO6lɓ'?ydE@)>Pҍ8-GZlmŝ:uj߾}͚5cbb]pXP(֭ѣ\Dzu)((U$ ѣ[n>}:;;;XCAxEHHHttt^\\\6mRV-777777R㰰߸qCR 0w1c./<ʳCg%YKy_%K̃nnZ.?WWwӫ~F(.]WhÏ /gX?hѢcǎ{D"޽'|ҺuMy#=}$/{x͝J'77f-ZY: -KLt C:t8rș3gž>}פIVZj H$#F1'OxÇ#""rH$NNNnnn>>>UTYf5 ,JϜ9sǏť*JuNNNAAAիWYfӦM===>R(&P.FEJw .2e\\LL,ʔ)3hРAY0` & ޽{mKT*bqff .H)w0gȁ, On} ]7jש'F$"###dDYbBbbB4")IfT*U\lL\lL==Lٲήf< ȟMJ*HOOoEGF-WT4*J %Dgee-mAHxe񖟿x=Pn2YBT.ފFFW%KLFJ_(J䬦ˣ"QR=lrNN.D< 0+WWwWWw}4qEGKk⊎TT q?77w Y@9+zͼ(x]IĨ(iTTݔՅ\q}|qQ7յ\eٙ'gCyP"{ .$Ybd4*J*Ӫ2OLĕxK .o_+++ ry\|\VVyqY....F< xiG)'$&K##"###dĸؘL3& zY[[[m) qqYYق 88K$8z#ܹ)* {{?25km;03\~S'ݺy-&:*--%bzjӮapY|EE'2Rzо3z~B|\fVw&ڶ$H$*u"#7رV˔)׺MUar>}3/?\jeeUjPppH4jeQvqEEGJs+R)HJ'g aU}3UӇoxS=D;|^{{_;tz SSSWXy,Q;$KJx?-YXF_hޢU5N!_Ri٫w[v񩓿w[C|G+@WWح[7۽Oc-ϺuქBo Q>5$heeeM{w۷nٰͭ+=|`r!o޸7,\֩sB YL]zd Ribbx+:J_,1!2R)}+:*J (>qdIGV!ߺu}ǎtر}Kz넩k,+(C7tvvNJJsr85{;g^rUW'W}%tP?H\ ӱcgwOOARRRNulܿbُ!Z4k^ ػmoڑV}uPRhnaʳrYӴ#?xڌ5Zu͚cuk|oʕ^W៏_ A߁DQ2e˔)WX`޷W=wf- .gEt Gϟef+J Z)nꃸQRuT,12R%%&⊊'g5R>KSýsfMSL`׎11fOkAC>w7TsRBB+GkNx|ʵՈbcl^ ;rTYzE9̝=ce_?៏} 4զVk^*,  ޻wWs+?|X10 lۺAs޴Y\y> ߰n|ُsbܢj*WW rɈ7S9yY^gܤ,JA\B<(uVttd4B}bbbBtTdddLY$+[Q|իupj)۶n|-&O3ݥ=n5fb^D}5~oHNc*5Ϟ=-_BôHrf d7URi\]]l+:lėGi";cty k4_DISfXYYs =)InطkƭfȻY7_-̊PLPD篿"==*:R[ Hip*WvΜF߬Y{Ynk T[owuŋgW\l$...f ^aj"ё)2+z331n_9/QM_ŋgƭ{էOi-[Qv^]\\zpՊfFF{h҆Yh/66F((@+_ŐS@`go_!R}Lu⊊FEIw)Dgee6ӳU]4m;N]@Jn\30W7o^>_QiѲq֍O'%'I$^^޾-Zmն!39ÚBo?T]4aY-Ƈh!<_ezdef޽vM>wG~1EPkZuon/۳;ʷ~@ |4I 9$нq]mwҶc9MUM.=]oϡXnxxڬ}4z #Pd((RfdT}oŲryQfSP$ Cs}BXjU${wwwJǁI^>F,pެukVhbW`TfyH [ؠaJmbYΝYYYni sःC>Elϟ?MLӧ9^>F%E"Y-ϝxMOL,(j׭f?$ӎRi9|ϋ\vdƷ Zn[y&*&ZǮúyz{jӮUvz>#,Q+>|L^gef~9=nDjԬӮsWΩ:tp߅~Y6Qq{qڙR̫z&1C؃̴H/?{ϟjj:Ǎ=,3#C$P"PJU+Y.UjTr>oϞ5I@V\~^y?&}, IDATݼE뀊..b٥rj]vzgѹ(J9]짅s#L@Q(Mkq>OwܵfTvV=7mX.MsĨ oG|*<}CRSS A_JNJ2d005'Sg`mmNwzycX?m.E<Ӧt z R%J}W>ݬ7mX=w '>y.h73J;;Ctq M+'>6v5֭ۏ=^w*++}TAޏS"k~]: ڈ nnظCFžr_1ѡwnj>ի4d=;fvALbv3]wY <{DφRSS4͉Sf꙰zZ.ӎl߶1;;;LrXYWOm1ѯt x"Ҳu۠/f{GͰ.ʳWJiv|wqMO! ABQP 2=Zֶvj]ZoֶڪZ@P 2ApMX!B~7=0HHu<'[Ó`Ln['O|d귶mW~affNs5A Ps_zy05gqUo ŨQ c;umq}.@eAgD (<+9d>gV׼YX,DEǽgOcF/+*g`@ѨRtdE}2fBs;^~YV27?Quqd"_] 0`–skkMfh;w~li}zu5}|>|M^IM"_5)] _:8:*RP_,~*232D~9ma?vRUA@CP*lꋏjH?]L8aM555I$N{ϝoȆo9c)ήGg??M:KyVU} =(ɗve2H$KS3efQ O^|0ehXT/ի.ΎZ0< @ u7 C"_^RMnա[EDmmMnN?'PۉP"nk(A@P_UYQQQ hߥ@P/6qd&3m|Y|Ռgş|KCCïFj)̪JNJhhheJ…˙Y %trp`*3Śeaa)okogaa`Tן#aشRȑ ' ,u{ >9G}knnv ܨ@WH$*_  ~e%_ ohm"k4x{|)7sGk=:eCD"*La}"}첱ao7MϞ:^xrLc?M>s.n}(]2JmhTCC=⪪_QOU_YD)U[gEm ]?"0]J鎏?5t骽?v=gnxf [|y쉵ݮ=_~mz~9h`t;X*nRyl?%oѳϽr> =d KvΝңq}'Z*oɃff6vv.BC_19lc}q(:ihmgч҇EԦ_Y5 [ɎG+5i{XP\|vI[Z]~Oe__0׷, [wS2fOݲ5~E=_~M]횏=v寴++mbWɃ~-b<9˛T9د~bD"Yӓ|,N(?uO`phdT ?ggHH}/ɗ.F9?nEqqkjkkO-B 1c [(d{Vuttajj '</33SYP"l̙3 O(%+e|~A\vԠU}}n][ZZ1VVVVVLQ\ZYZg~zfm!sÕ_lyu!*m5d#Wn1 {Y>_#RF^^fNJqn{𥳐ϫ(mlds^nRrߎvrC5 jkkk]S˯5aGmupppu1@y 6DRS]OUUEEwXZrpvw[>}O>9۷5_>ݺښ><ҫ/(?Kz*"~')USLQY+6k(㶽Srs{@],g:555 J~E_XxԅN3F3NVֲ,&IJVemmCt:‘4s}|} )qTh4|Ug_1izw铋o}уdOwp|'~E:CKCãG똓Ө踅yti~ܢѣ:z{}ii QkkG^,..7.%$_ɻ"?&R!s})W1 Ћ*mC[eeS Yښj_ *+`l+}}};{ţNNNʦN.۟ #c|8UlzM59g._J8'f{ԓ~"/˻SUUU_WCll&Mv +M9ikaabsP,nZZDVC:ڶMrUc@Vn,myEӭmlXVV֎L'GG3{(!2%"~Zv;A=Q:>j7? v"'Q"޼w=%KJWWUZ[BYY//olN@ 쬮U\UUUTeG *fݕJV}5l Cy۾JܢnJKcG#uҕ*-o?m~Jg Ltt,^ƽr1b~wO8MsFlkJy%^8/'44M$jnLpztS/ށٿ}~Ǔ'(9vsf |Sdⶶr>⪪\vԠR;,-dV5(+vY#K`򬦦Ƣ‚ɞSUZFdgZu53C1tDr%-ERϯ\015S~̙/?ꬆw~cSϫ:]q–n%Hj*~/;jR- ml(G ZXZ1F98SWh4bŋ~t8+SI,1b0+?f^QIG'&qM|ײ?r`HhlWU A}EuW-L'`92Pw(Y IMH_X3ʯ: 7b:"Aݽ)A#Q>~+ȿrY3,5}=^SS~1u`0uvv_]Yu`UV++)ٖWt##&s$;jʚtrptQY#NP\TI\e-^IJ š3*%oyo~V~燝Aᑱ_ 4ApM$c Acllbcckkgoggomckcckckkccgg`cckmckkkkmcLF(g8)|~h%ImqY+ԄUr F1giƍwukcccai-JΟE :G)qAO0qҘ1ncML  ݽvo:'41 taK oJ=5hai4сt~#0xM󱵵į,^R*=Ued[l&i)'CCìkw>djjbs^^z3[6)OPrGSS#8$llmlg?«v~'% eLIӧ^p{ewvc"􃑑{thQVV֎L'G  -"NۉP0 ?z%tyK==HH$ʜܘ%Ȝ2u29RsEC++ܹ]4~_Tɑt:}ƷFv]icY0tX,sɑ$teeetrrppB+IggUg\ɼ^l~1Nm=gDsC#˳RS.*9=9^SMM9ٙ3\$-5YTpHFS2 U=bŇN:75]IO9^J tʳF9si4SɃŮn}Nｐ+%9AKsC#8Qq3ffPɗ4WEě[or{wƎ}A-bb#b"b O&fRUদƖfSS`033v T(llME''%~,x> YJō3,==9a}NKW*g޼M`Ų\g>CCSI֖s*/bb|NNfǒ, gWN@Fsg`k; *gP1ݔg]< er-&vvy⻔m\ jkkjml5/Q,ϪknGArxi\e^F@P L'c'̉Dzv("b>t;%v9SOOɉKѨQZ+)1,t1^15'ǹȬos4X5a$`sSvQ__{95_PiT/P3ށ<Ԅ1#?y3bK]=u(m3z,u-oI{[n<]0Y@oP0BMtrw9P ~-gfObTr2u]_Ig%_WD,6bqN`?|dd7ٳgYf@muí#S,r$nGOo  TJ$\vTh ʳFy(YA$ğ<|YA̙!anh;[7zؘ2_~#/ 50,aG }:--%%)M{Dlbs|g9h(OnݒEBabbbbb͛}||"##Y, rX[@acm"192ylw47RRTP߄,P0Gt˿]IRů$ğ!_~7~똒GE"b)\"wC IDAT"6nQm|M))HqK:݈`Xl˟3cLIVttttK tz꾁<_ҥKZT*R Fs3~ZGSȗ.LDm%32iH7PVRE1G|qByfffFʺʝO677^Iܙ~6˨?< gOR"f!!a~ 旟,aF*]fery +XYY9l6'08sʴl޿rٮ.`:9=q7!xn[jmڴ=444666,,hSQ])HG_d`^|}֗v#ϼ:nd|<-gq1Jy֥ɑ˘؅9ҔzWXx4_򃦠 w_1_jemcmmmbb&n [\}ԱVA#s3ҹ,^mM+̙cqO)ۑhA|ny]!xÃ)=)..޳gϞ={lll¢"""p8rpDI,X UDZ٘pmhW5]Y#]t6Q7F^T/oy$9q[e$ooi..n;>*<$  nؐqQLOIȻ&D,vak"IMe€TUUE";|N9I]]ݟZ'zLum%%mgf(,,F?{|p^AA첣#YɞS) zzzQ~'r򬳧+n974rBIM6YXX ~J'.5/#M/ȿѡt==^,6fs4'8TSrTzIDRd֬Y111 ,pu/9.,G,E(b, N;,/ʸRWWK~4&v! vBJy֩G7m&Y,)U6A c[ӵYjU&#GY_a@SٙcZJ%_{qV2@y&=D>/';ׯ.M:]q:sr/ Eog0;pz&USjgjMbېSZZѮ,v нy܌Ln:VUY 66CigPS%oi&wwٳg|ofɑ @{ xdc, =ǽJ~hYO,\<ѿ^^zE%KWJ_1 lmjj,}`(~]~]mgC˽HqӳxM dk"IЊrؓ;wܹ6,,,***<)X]Y^'E6QX"dei{ 2s_xާ3%W=JwJn-,{Y.157vpvvw:{4w>N=~ɆzKO]r\VJ|N~ŕ B ll,Ϝ27zy?" O]Ϊ/HaF]2m-Uڽ nJZaȐn`mk>cn _I*-(7kN/D`92*+~I`aYs­zy [9WB|˖?6C1X__7[v$;wq3jjjHiys-F0qbq?E-СC"-R{vIFm(hi8K-īU¼g8ZY..}؉YG_~Q)oXŽ J'#B;Z)D~WmubyCAYk^Qf͞h~ dEo#yyʞQZRI$两U/֤__۱:bKeY@d٣9X[S}'c{Mi:9SFzzzu{BS h4TJT,1s $.ݞ8Yh;rj/#M/ȿѡt==^A,6wwZS{"oebb2k֬ hnG B*~c;?U҂kŽ=jjn]zrXF©7?~W9_Z,KQ|΍[MP{Rp8o>M7Jȑ+Y9/}@Aġԓ^VfYE{ nfI>#V'߿>šݡ@W<|bp[Ւ#/}KVP"9=-\tqgNjH??rMo^>~66΢lm}fh;aڿe}jv^[̈ ڶ?]\s>1c;;#cQ0*N3 M67cƍmmm5H.x/>~kY2{5hyi͢ٯ+Y"!پ珷S~J\2lBŬ`'Q|~RinFZ&7K KK+ ehh<`Zh9'lx4¸MNӔaJQ?$ĨZ.ggO9*>pSݰit2]'n7?萜9帆9}||VDVa& 딡kSR6xUԻE= 05c :;i|4_odZx_AyRʳ(zy"zzz /z=־?)jKK%756n~}io PEYe"qcgO>|}ffϽ^s)Ԍ3;DYִggҹܜU`kk  cqOD0iBKFRKU=66666vFF8>y7,}}إaX@O;+Y,a)9g>>%#~T*mnn(/u+JXt< F H2yܖfUWpubsC]g9SJ#(hsNРJ.Z֭CK-U&IJp,v4q2db3ѵMֶ])䈱׿mǢtty%1K9"Jo> 7olrFYXZXxUrӋ>ljHo>Oh<~Rv4616c•}p|fXާ|Ư(OG3scyFY̝ۚ0w>ۥ϶~6 ƆhVjqxQsg|%'kyD__oݪg"W>g>KGǿmҪzߩwW[Qsپ~=~}}89ydiiiddرc ʳ_&طn_ e<=<= }m؀:'n|#KJ+>$߷g=KÈ!T*4wvv{kɆo|VŇ$GN8kײmmt|OKCC+%9 zGGJi44`8L' 0tvv~I˳r1 0<oڴ -T"J#吵ؙl1b{SmV'fMi1~=-I*LcYx]JC.'^kXXkn=酨U| f(-:<%o}Yt#GFG{Vt~fVj!t~گ`X?_70wmKvVW!}J|NJ|vPrzX'LKv%F]MϞ!YAmڴiaaaqqqVVV T@y upTTbzz5[%F_~2ENⶶR%ǎ\q[[Nv]]mGl|18`_h7]dmNg9^ٲn>N6]Jlge}_t~fVjqh.'6rfƞj07=mywvtE6gz񿧏4n~ѧS1gڵʕ+ϟsNt2ZP]L }#+V+Βe+#dO39 SSߚN{Ҳ_\E^߷d1٧V߷VMDd`Ύc.ީ<~:i;;#cm49 M67cƍb#ƞ/S"|J:򥕍3.Tf)sbs){j߅BINL%_Zۚy2CDrFٓ nOJ-~T/61U/ V89ۑ#WnU[rAh}=55MǬ[H[_ephnڵsmܸsҤI6l8{P(}:nTk]L ZZZ,P~{ OSKYea1T{9u.uyzz?Ke,n߃y#/M++`mm3;p2 NFRkΝ111 .tqqv^/]. Gƌw Сc7?ӌyVbk呼l5p #vfw!{Qr|69$VI7T5#>SǪ=7ݼZܯ~˧nXJN761ZTؗ$ONRis9IW::WAMG*h4ڏ?H߻wo׮]vb0AAAQQQ'NV F"✝]KKK };mk˳~~'mA=P\I9~ܙcJrgQq7q¤9a䙔k{Y[[xhɊeW#I bqNvU^zJȘ̂FUW5ʙ3;⌟࡯ߟ,HҒ(H(7!*5JdѣW^F Jdn OC.RUP"nk0+23sQ.W#{vqd,"J$j"BWnfXRS%ϡg]NFPCnrZ*k/JKv)OPm?+'1$вD7ovpp Ǥ ByPguߤPS?l@V3Rb|~kjjlnjZ&&&&&&VvvL&Ӊ4m[ zq˩/&ݸ[RrZ$jhL}}|fqNy022S.i7 -0c2bsnH`xCK-(qaH^6@*ȱ2!_vK]US {Bq` ilާnS"9uK7nfLRigW(ۊZyƃ6؈A~_>Tz#^9[bxmc#6K>hhhzSUUuȑ#GZjMYdmmxKWt:}nhШA1.5/#M/ȿj[8==^,6`P# Zjtzsyr 65 uLY/[4[1$ YO=[RYI%2U͙S7Re%K.h<|P5yɑ Zgj.N<|YDgO6KFܹO?sp[[[BBBBBkfooccc3ʳFҒ+i) e*䪫e-O3k,=== ]< `DJnfSy +XXXgf93Y}&}uCK-]˘[#Pf-hiRBT*iZ {BvɆFMSZc|inaG7nf-]4"C*1k]P^Zxߝ~p]A ]ЈX?I//; BJ^R.f@tyD:MHHHRR%Ivvvvvg}fjj:a„g}666Ii¿P0luvvMMNrӯfUUR6`vs IDAT2!Zvvvaaaaaa `@$.M #ɐ'rw# V: .͊4nOJ-::<5:ݰQgI:$ϩ;#5QVYGjpwO=ĉVzzzޡs /4`Xqyh&NbBBY,#-ã총lJ{*tuDuo.30ȚCк''/x8ZVf:E.0y?u3+5(_4::$>zjmjnȑ#G! 24O:Cy Va빵NDevvӦy3MU]]M䔤D7[jm }|X;QCyB@ QCR*޻]6aהH:KT7TF'y9w*'GRrcэr08%?y?'+ UP@~)JN"o>DF:!ᨴ*8̬+E¶'ЇZ,иVaW_|\WWPt1܆oBVInF^XF bsX3&*hootk; M F7>}ph#$/sH~νgM6&|9r5P3fĠ[*j''v9ِF=2[ɹ^ 3ɑ+E(98YI(׳T.!xakod3w+|v{RmgZ#oe``9}tTgQ(͹:tkR4z7#-WVU]ҊfsC'{zvA%']8uomg0h%KWg)=0cbbtd&g2Atr© WdM_IȥYKWȚZdbbDT*Sztr/`(ge;O cԓy#Z^Of>D+$ۺ|aҗ=Q wr5A]sq˗/o۶dFDD<3l6[y (knnv 8h{ggҹ̚*UWpBBXTg)C"G^;Z)Ÿ{)8?7E8XNrK:Q$tj;#-FFFNgh@yE"./g]嶴\bsnckk3SY UWf4䏛1kk똘@m~ֶ> m퟼5,æ\+R>=ޗkvDzgܓ./ cp0/X|xr??:xCY*#}Ѫ8'tK>Ov\r$vih48/q/95Zҹzm4fY֐SSSm3Ġ< ѣu4ECC}JE^F^ǏQ7˛X,#ICyల4@YheH+YA)~NO~8r Av 3kjxAOe._±n2O:%g^!GfMPi,koOovڰmz !%R]ۿ!+u Uܦ'KBRN`w[Eo_ʷV?enAmAXZ- [@DIq뉧3SQ"<<<444<ttth<4-5Mq޹%HTNӽ}fA!s}|T( y {{joa,0{{~n19?qU![sNOTn.nq?ܸUI8Ypsj-u$g ȑl(69E߿'_m?$lj58ڌR 5Y*WwHGRs>6Ϟf [DO.~+45c<| :\3]|tmD-JāohhhLL 60@ei;a`=/${njic_y*dm8(8wv;uNI[,aIOOر###_~Yd j`trCɯ{=n.tfgn .c=|{z 9?/٨! $K#_8X*__ [Dt?Bd#ζϞtrx,k㥳Yݿ$ufϱFSJʎLHbVzAûyit?Yahe;[SPqg"}6R (m9;ˁ3r_|~ؙt#jk7Z,,,,Yk׮۷ogddر# Yf7rit/ 69l)a&::&Mu{']#%cϴ<9n,j}F=YMƚuq.nSnyՇoJ;qX$qVfƦ (킇V1Azz+ ؓVBNu)+cmfMhw44ܿ].n bsS{"l%$GBcfc);ڲlvq9~'|`mk0w{;{K^/w*m4c9vɪ;eJ]/HUYۚaGvtH~g{JWiR) - j1h{{hԈb^$:(,eg~"=]1=;_ٯ]uڰ>+cahAޖ`[[j}w,w_v O*AnN??}g֯_W5[jڴӯgmuHH-n.!!O>#F2dH.]\a37}K[o|k`xVE V<𸭿|w-2mAvApp?1v૏'~m^2iL÷ژ;re>0kvKN7 BW+-7_d5 i) _~on[0gIZ+A33K:WWYVӷ^{{eQ#"ER):tn١s˳.Rñ_VEf\|ѷծ -M R#Hj*v!5k֑GYl2Xz_|ݿۯNiӦϾ] ҫw߮zo(>ة( ^|_l24əpH[|f9klQj I ^se1MiNAAA$$LV\Wd+aGIIM:x=[زFy <)癖tQ7Ϛȗuj*z}K>%AqVJپn5Β/O?nTʘ/tڕ͚5kС{jԨq2nٲeQ ,(gϛSSRS߫w^}vѫja4 D.延 fL;ed}-wc5k<󙙙n$VZv⊁֫W mi޲]zeGc[j{3רҵWہt;Aů٦m'߸vƔy|nS߰V~>^;I6P }>;no䔤t88QꙌ{My#GUdXUq1O~0'?4OJjrNw^m{ئmR(ٲMz]oO ًPȲ߰]p֨>kNǟ94kkѺ#|={ؿ8S ͝=iҏkVo#IIIm}ӷW~ի Iv?pܹsګI^dg\2??N'5< w4Q |_/>SOi99Y[NKRkOAA7!vÈH$rI>g`3wܡCΟ_;QV׮]GG_G"Ms2u7ݘJ54iVu&M+_cŲ."sպܔjiժ7kQmuըwQ/\19/߰6kjTQ^vگurJy39~}95ZkPyO%&VeMi9V-MJJLMOIZzz߰qJ<[69Kڸ.S5h\M ˤi fN]tʍjխzػJ՝Q &0vf~ܧ[JKhc;zt$ *.!7oСC͛ /LeuaGF|QlH$b,خH$RlVݺسw}uAPW,ZiieiS^އL:qIFFnnNzzFtߠ :,95kfgg}I?._t UT^FVtU5+(lW_\e lKWׯX|8ؓnO_{//m[>m3/2t\v|D۴jŊ_~7_=|Y'xgŦigqܨ*e+.;?0eħ{VZiV\yǕ4 .v0]N/vd6FYz/bf-8@[jY?}k87A ۢEÇ>|РAUVwkx~FׯkvÆѫw=z֭Wέ¡rtݷ{;n'+2DWJ,?D"ͷWLH=xݷbl޼YNݓN9,V233}螗^|f뭧VXQ"3gL?cׯ_Wd<--=))ikӦM*1Uhꔉ?_z#)9-FVVȳO3{Vwrkf?.ʯE˖)e9ݵmd8x`e 4'vǫ jթ~q8,r&Z7ug pٯk:yއ+:3rUmyowLIMo`}nۡmkժ]JO!|ٿ/\0/Z7ǞRJtZjq⏿ݷ  ijoؑSN;线-;֥_q<1=^{{ow}JQjv9y(}lGcelٶcS/ΨV)v%-P9ijv?Ϝq-}g\^^|y4dW^͙g}3EwTZzsO?q䃇vǽ؛nȣ朄Q\r1/=w*+UY;nӣiب~[j2-OF_׫_GfSRC'̝=U6eo?[$#Z9m>O ۆ7Gi׾>v<Ȁ^{-[,خ͛wx֎ǍlII6Aە4` :tܶ}A~M6mڔ~ ֭[9??7/'kcVk k3 ؘ1QKNNZZժUOHLLIIRj5k֊D"!H$Rf TPF TTz WRVXfuiNKOOeWW۩VFlnmf$&%ի`钌rUaҥ'-7nz*f%>B|żɧU~ on))){9ˊ=ƫ4ٳ(dIeKsUvs_fVJ<7\Aջ^P5=Gf5i2dȠAԩ#{,خ-[ApACQ+deefggedAAvVvvNvvt0';.,rsN*~5'';';{u6mĶXyyyEz\۵S}Xy6l[Μ94WUჅ9ŊxG'Apđa?csb׬^}2//}7 |E;n7_-:waтye-I&…7>t^/|]f;}8!!{UQÇ ֻwoe;q^掸Wa,4Xa,4Xa,b`Q  `]yv;;vD騯t%d~g&ǟgxbVN?ߕol޼Gf[mg~S&O\8ukխWa~:!իW/\tޏn5}SN8➿=ޮ}i-۵x#^-[[֫ߠ4WrڲeKw7+oQRme,`W!(X;O,;v0A f`Y?gcY5ky)wE G[UQNv>N XnJYy999O~я׮]`ݺuaR q{噧III-Zc`ɢEFt>ࠡݺjfZj''%eɜ:u;5_FGYY/|ZJ ! Y9SL?5_,9 92}YY2'hܸq[J]>u}ԨQt[쬌Kdl/AÆeMwpPYa._[ڻ^tY~6n$===)9nzt۝?5k֊N^`޳O?ՒߴF]e9i F]֣g{ݷ1ngۛ崜b)Mh˯MI+,=x#FUĎiwn==#W^}ñǟTEv(:t\bG^\t?]y٨z5Zu{_z/7VxV<&}RjlWxV^lYuEJW̛c4;vKy͈g칚5k~OWW\u}~ZjSFͤkwC;o6"JE<==vK: * &'';LjS"vXEˌӧN{roDQ_q57Fׯ]ɧ_x]쬋.<{Æ e}#ATEjڵ*3sMlYZyYy$.ÕENZ26{f9Mbˮ=zr{;r.`㎖[nMN-HC;rꔉ򧟾o"sVXqݷFJHH8ӻtqǭZpp钌ELo$ZĖ+V./U+Wlqc-)7^u钌;t4r,K8˗-6uR_AJsc_Ǝ~y^qM%5jĖY99%Ƴbw `y[y6W^sc1 v{=#…_P[+\|fZli-u-񪫯bh٭{;~$u^i'ֱӾiSпKVƼ ֩_dKb f[ƖkV*UE߳O 3%Ώu鞔\ xVY%ƽ#%^[Oef-y/LE<|{Il)۵/2!111-UP5?sTV(33A|ự(2ȁӧO Jaꖩ mof1z:Fƭ]v{ ^}ؑ!C-MijV*Znvܿ?%ɉ٫UkǾ%sG0Oo^~}*#>6Z<e׮=4iZdNzZl-?l%Kbhܵ[O+%&%} rя=ӧM=ݪMmΌ˷`"@uYNxwQ~uڭG!7;睷<Ёu͛7x|ժ|K/<}]V/-͛3Ca\=hcO6gfff^KcGN=c6gz׹99]5PPPpӍ,^0:[J;Դh?㭧o*vho .$)P1U^rU?ܼxRf))ݺLKF 9k3Ӿ?~ EAjּSϾҼEW[r)Wëhم7npޣWF9௬Zis#~2//ӎ;m99wHq6<oFnܸaI~AcLKK` 5e5my._ABbª+a6???O<[E,#+>_fYY>]9=*Tzz6Hܿ5 ,=QbBb٬۶|p\dQBvvփ9 I*UyN<2#cqtp)3OԴǟ|qYO]p-:RPP0yҏ'mBzÍo=^GO9eKD/_'ncrjC=Ӽym.o "! i9vJ=g^yʔ `Օگ5oѴiؑU]JYƼs;jf_|wۻ?yu+MIII.Ɀ?߰*#j޼ŘWu_7mب3/V{5gӫV^aMԱ>]u39%|kpTbYE"n瞒S8r.iߡcޥD͛x7c9s0 |YsAJZj)+/edl{ױƍzѧy^FN9QӦ^}1/>S/\8Wk׮s EիW/f[|Gl.˃ ݡPacƌu$97'{Ν^mO:xZh'ݣgn{ƷիVMòeVZDԭסCD"^sޜ3fN[hƍ 5jԬ]n}4mڬ+M>uڔI+/\v:mwonܦr[y걅 iyn`[|ΐD"'s`gOŻ `:rxw.O>-b>0D"v`ִiSY>]΀T-[iٺMEV(#СSv} 4Xh{FÆ x7յkxںt=gkժUn޽{-';۷oƍ駥Knڴ) vڵUVݖx֭[n:]n ,PgB< YHw7oʎwڼys[6,3gw.JpCxRZ[Dj@Qvw^{u4TPbbb@QYTFƻ 7x@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB!kIDAT ,PgB< In_wG{@(ijB! ,P$Ż*Y$IJJڴiSt$-%x;;gԩݟ DY;ƻ`#(xniӦ>BթSƍǻ %ŻBѭ[O>}ʕnM$&&֭[cǎM6w/ m5k֬Yf\7x@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,PgB< Yx@(ijB! ,ڹC0< ;54bpHʪޒsܟg<,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ,< !Hȳ???f{@Bg$Yc۶{޽p c˲z9ry w|:Zz~nm۫KceYu=4},>47{IENDB`distributed-1.14.3/docs/source/images/scheduler-detailed.svg000066400000000000000000001710531301211143400240640ustar00rootroot00000000000000 image/svg+xml 192.168.0.1 192.168.0.2 192.168.0.3 192.168.0.4 worker queues worker core worker core worker core worker core worker core worker core scheduler queue report queue Computationgraph whohaswhat center/deletion sockets Scheduler distributed-1.14.3/docs/source/images/task-state.dot000066400000000000000000000007341301211143400224010ustar00rootroot00000000000000digraph{ graph [ bgcolor="#FFFFFFF00", rankdir=LR, ]; released1 [label=released]; released2 [label=released]; new -> released1; released1 -> waiting; waiting -> queue; waiting -> stacks; waiting -> "no-worker"; "no-worker" -> waiting; queue -> processing; stacks -> processing; processing -> memory; processing -> error; error -> forgotten; memory -> released2; released2 -> forgotten; } distributed-1.14.3/docs/source/images/task-state.svg000066400000000000000000000171471301211143400224200ustar00rootroot00000000000000 %3 released1 released waiting waiting released1->waiting released2 released forgotten forgotten released2->forgotten new new new->released1 queue queue waiting->queue stacks stacks waiting->stacks no-worker no-worker waiting->no-worker processing processing queue->processing stacks->processing no-worker->waiting memory memory processing->memory error error processing->error memory->released2 error->forgotten distributed-1.14.3/docs/source/index.rst000066400000000000000000000101341301211143400202000ustar00rootroot00000000000000Dask.distributed ================ Dask.distributed is a lightweight library for distributed computing in Python. It extends both the ``concurrent.futures`` and ``dask`` APIs to moderate sized clusters. See :doc:`the quickstart ` to get started. Motivation ---------- ``Distributed`` serves to complement the existing PyData analysis stack. In particular it meets the following needs: * **Low latency:** Each task suffers about 1ms of overhead. A small computation and network roundtrip can complete in less than 10ms. * **Peer-to-peer data sharing:** Workers communicate with each other to share data. This removes central bottlenecks for data transfer. * **Complex Scheduling:** Supports complex workflows (not just map/filter/reduce) which are necessary for sophisticated algorithms used in nd-arrays, machine learning, image processing, and statistics. * **Pure Python:** Built in Python using well-known technologies. This eases installation, improves efficiency (for Python users), and simplifies debugging. * **Data Locality:** Scheduling algorithms cleverly execute computations where data lives. This minimizes network traffic and improves efficiency. * **Familiar APIs:** Compatible with the `concurrent.futures`_ API in the Python standard library. Compatible with `dask`_ API for parallel algorithms * **Easy Setup:** As a Pure Python package distributed is ``pip`` installable and easy to :doc:`set up ` on your own cluster. .. _`concurrent.futures`: https://www.python.org/dev/peps/pep-3148/ .. _`dask`: http://dask.pydata.org/en/latest/ Architecture ------------ Dask.distributed is a centrally managed, distributed, dynamic task scheduler. The central ``dask-scheduler`` process coordinates the actions of several ``dask-worker`` processes spread across multiple machines and the concurrent requests of several clients. The scheduler is asynchronous and event driven, simultaneously responding to requests for computation from multiple clients and tracking the progress of multiple workers. The event-driven and asynchronous nature makes it flexible to concurrently handle a variety of workloads coming from multiple users at the same time while also handling a fluid worker population with failures and additions. Workers communicate amongst each other for bulk data transfer over TCP. Internally the scheduler tracks all work as a constantly changing directed acyclic graph of tasks. A task is a Python function operating on Python objects, which can be the results of other tasks. This graph of tasks grows as users submit more computations, fills out as workers complete tasks, and shrinks as users leave or become disinterested in previous results. Users interact by connecting a local Python session to the scheduler and submitting work, either by individual calls to the simple interface ``client.submit(function, *args, **kwargs)`` or by using the large data collections and parallel algorithms of the parent ``dask`` library. The collections in the dask_ library like dask.array_ and dask.dataframe_ provide easy access to sophisticated algorithms and familiar APIs like NumPy and Pandas, while the simple ``client.submit`` interface provides users with custom control when they want to break out of canned "big data" abstractions and submit fully custom workloads. .. _dask.array: http://dask.pydata.org/en/latest/array.html .. _dask.dataframe: http://dask.pydata.org/en/latest/dataframe.html Contents -------- .. toctree:: :maxdepth: 1 :caption: Getting Started install quickstart setup client api faq .. toctree:: :maxdepth: 1 :caption: Build Understanding efficiency locality manage-computation memory resilience scheduling-policies scheduling-state worker work-stealing related-work .. toctree:: :maxdepth: 1 :caption: Additional Features local-cluster ec2 web joblib ipython publish queues task-launch adaptive submitting-applications .. toctree:: :maxdepth: 1 :caption: Developer Documentation develop foundations journey protocol plugins distributed-1.14.3/docs/source/install.rst000066400000000000000000000023301301211143400205360ustar00rootroot00000000000000Install Dask.Distributed ======================== You can install dask.distributed with ``conda``, with ``pip``, or by installing from source. Conda ----- To install the latest version of dask.distributed from the `conda-forge `_ repository using `conda `_:: conda install dask distributed -c conda-forge Pip --- Or install distributed with ``pip``:: pip install dask distributed --upgrade Source ------ To install distributed from source, clone the repository from `github `_:: git clone https://github.com/dask/distributed.git cd distributed python setup.py install Notes ----- **Note for Macports users:** There `is a known issue `_. with python from macports that makes executables be placed in a location that is not available by default. A simple solution is to extend the ``PATH`` environment variable to the location where python from macports install the binaries:: $ export PATH=/opt/local/Library/Frameworks/Python.framework/Versions/3.5/bin:$PATH or $ export PATH=/opt/local/Library/Frameworks/Python.framework/Versions/2.7/bin:$PATH distributed-1.14.3/docs/source/ipython.rst000066400000000000000000000051341301211143400205670ustar00rootroot00000000000000IPython Integration =================== Dask.distributed integrates with IPython in three ways: 1. You can launch a Dask.distributed cluster from an IPyParallel_ cluster 2. You can launch IPython kernels from Dask Workers and Schedulers to assist with debugging 3. They both support the common concurrent.futures_ interface .. _IPyParallel: https://ipyparallel.readthedocs.io/en/latest/ .. _concurrent.futures: https://docs.python.org/3/library/concurrent.futures.html Launch Dask from IPyParallel ---------------------------- IPyParallel is IPython's distributed computing framework that allows you to easily manage many IPython engines on different computers. An IPyParallel ``Client`` can launch a ``dask.distributed`` Scheduler and Workers on those IPython engines, effectively launching a full dask.distributed system. This is possible with the Client.become_distributed_ method:: $ ipcluster start .. code-block:: python >>> from ipyparallel import Client >>> c = Client() # connect to IPyParallel cluster >>> e = c.become_distributed() # start dask on top of IPyParallel >>> e .. _Client.become_distributed: https://ipyparallel.readthedocs.io/en/latest/api/ipyparallel.html#ipyparallel.Client.become_distributed Launch IPython within Dask Workers ---------------------------------- It is sometimes convenient to inspect the ``Worker`` or ``Scheduler`` process interactively. Fortunately IPython gives us a way to launch interactive sessions within Python processes. This is available through the following methods: .. currentmodule:: distributed.client .. autosummary:: Client.start_ipython_workers Client.start_ipython_scheduler These methods start IPython kernels running in a separate thread within the specified Worker or Schedulers. These kernels are accessible either through IPython magics or a QT-Console. Example with IPython Magics ~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: python >>> e.start_ipython_scheduler() >>> %scheduler scheduler.processing {'127.0.0.1:3595': ['inc-1', 'inc-2'], '127.0.0.1:53589': ['inc-2', 'add-5']} >>> info = e.start_ipython_workers() >>> %remote info['127.0.0.1:3595'] worker.active {'inc-1', 'inc-2'} Example with qt-console ~~~~~~~~~~~~~~~~~~~~~~~ You can also open up a full interactive `IPython qt-console`_ on the scheduler or each of the workers: .. code-block:: python >>> e.start_ipython_scheduler(qtconsole=True) >>> e.start_ipython_workers(qtconsole=True) .. _`IPython qt-console`: https://ipython.org/ipython-doc/3/interactive/qtconsole.html distributed-1.14.3/docs/source/joblib.rst000066400000000000000000000040701301211143400203340ustar00rootroot00000000000000Joblib Integration ================== Dask.distributed integrates with Joblib_ by providing an alternative cluster-computing backend, alongside Joblib's builtin threading and multiprocessing backends. Joblib_ is a library for simple parallel programming primarily developed and used by the Scikit Learn community. As of version 0.10.0 it contains a plugin mechanism to allow Joblib code to use other parallel frameworks to execute computations. The ``dask.distributed`` scheduler implements such a plugin in the ``distributed.joblib`` module and registers it appropriately with Joblib. As a result, any joblib code (including many scikit-learn algorithms) will run on the distributed scheduler if you enclose it in a context manager as follows: .. code-block:: python import distributed.joblib from joblib import Parallel, parallel_backend with parallel_backend('dask.distributed', scheduler_host='HOST:PORT'): # normal Joblib code Note that scikit-learn bundles joblib internally, so if you want to specify the joblib backend you'll need to import ``parallel_backend`` from scikit-learn instead of ``joblib``. As an example you might distributed a randomized cross validated parameter search as follows. .. code-block:: python import distributed.joblib # Scikit-learn bundles joblib, so you need to import from # `sklearn.externals.joblib` instead of `joblib` directly from sklearn.externals.joblib import parallel_backend from sklearn.datasets import load_digits from sklearn.grid_search import RandomizedSearchCV from sklearn.svm import SVC import numpy as np digits = load_digits() param_space = { 'C': np.logspace(-6, 6, 13), 'gamma': np.logspace(-8, 8, 17), 'tol': np.logspace(-4, -1, 4), 'class_weight': [None, 'balanced'], } model = SVC(kernel='rbf') search = RandomizedSearchCV(model, param_space, cv=3, n_iter=50, verbose=10) with parallel_backend('dask.distributed', scheduler_host='localhost:8786'): search.fit(digits.data, digits.target) .. _Joblib: https://pythonhosted.org/joblib/ distributed-1.14.3/docs/source/journey.rst000066400000000000000000000161341301211143400205720ustar00rootroot00000000000000Journey of a Task ================= We follow a single task through the user interface, scheduler, worker nodes, and back. Hopefully this helps to illustrate the inner workings of the system. User code --------- A user computes the addition of two variables already on the cluster, then pulls the result back to the local process. .. code-block:: python client = Client('host:port') x = e.submit(...) y = e.submit(...) z = client.submit(add, x, y) # we follow z print(z.result()) Step 1: Client ---------------- ``z`` begins its life when the ``Client.submit`` function sends the following message to the ``Scheduler``:: {'op': 'update-graph', 'tasks': {'z': (add, x, y)}, 'keys': ['z']} The client then creates a ``Future`` object with the key ``'z'`` and returns that object back to the user. This happens even before the message has been received by the scheduler. The status of the future says ``'pending'``. Step 2: Arrive in the Scheduler ------------------------------- A few milliseconds later, the scheduler receives this message on an open socket. The scheduler updates its state with this little graph that shows how to compute ``z``.:: scheduler.tasks.update(msg['tasks']) The scheduler also updates *a lot* of other state. Notably, it has to identify that ``x`` and ``y`` are themselves variables, and connect all of those dependencies. This is a long and detail oriented process that involves updating roughly 10 sets and dictionaries. Interested readers should investigate ``distributed/scheduler.py::update_state()``. While this is fairly complex and tedious to describe rest assured that it all happens in constant time and in about a millisecond. Step 3: Select a Worker ----------------------- Once the latter of ``x`` and ``y`` finishes, the scheduler notices that all of ``z``'s dependencies are in memory and that ``z`` itself may now run. Which worker should ``z`` select? We consider a sequence of criteria: 1. First, we quickly downselect to only those workers that have either ``x`` or ``y`` in local memory. 2. Then, we select the worker that would have to gather the least number of bytes in order to get both ``x`` and ``y`` locally. E.g. if two different workers have ``x`` and ``y`` and if ``y`` takes up more bytes than ``x`` then we select the machine that holds ``y`` so that we don't have to communicate as much. 3. If there are multiple workers that require the minimum number of communication bytes then we select the worker that is the least busy ``z`` considers the workers and chooses one based on the above criteria. In the common case the choice is pretty obvious after step 1. ``z`` waits on a stack associated with the chosen worker. The worker may still be busy though, so ``z`` may wait a while. *Note: This policy is under flux and this part of this document is quite possibly out of date.* Step 4: Transmit to the Worker ------------------------------ Eventually the worker finishes a task, has a spare core, and ``z`` finds itself at the top of the stack (note, that this may be some time after the last section if other tasks placed themselves on top of the worker's stack in the meantime.) We place ``z`` into a ``worker_queue`` associated with that worker and a ``worker_core`` coroutine pulls it out. ``z``'s function, the keys associated to its arguments, and the locations of workers that hold those keys are packed up into a message that looks like this:: {'op': 'compute', 'function': execute_task, 'args': ((add, 'x', 'y'),), 'who_has': {'x': {(worker_host, port)}, 'y': {(worker_host, port), (worker_host, port)}}, 'key': 'z'} This message is serialized and sent across a TCP socket to the worker. Step 5: Execute on the Worker ----------------------------- The worker unpacks the message, and notices that it needs to have both ``x`` and ``y``. If the worker does not already have both of these then it gathers them from the workers listed in the ``who_has`` dictionary also in the message. For each key that it doesn't have it selects a valid worker from ``who_has`` at random and gathers data from it. After this exchange, the worker has both the value for ``x`` and the value for ``y``. So it launches the computation ``add(x, y)`` in a local ``ThreadPoolExecutor`` and waits on the result. *In the mean time the worker repeats this process concurrently for other tasks. Nothing blocks.* Eventually the computation completes. The Worker stores this result in its local memory:: data['x'] = ... And transmits back a success, and the number of bytes of the result:: Worker: Hey Scheduler, 'z' worked great. I'm holding onto it. It takes up 64 bytes. The worker does not transmit back the actual value for ``z``. Step 6: Scheduler Aftermath ---------------------------- The scheduler receives this message and does a few things: 1. It notes that the worker has a free core, and sends up another task if available 2. If ``x`` or ``y`` are no longer needed then it sends a message out to relevant workers to delete them from local memory. 3. It sends a message to all of the clients that ``z`` is ready and so all client ``Future`` objects that are currently waiting should, wake up. In particular, this wakes up the ``z.result()`` command executed by the user originally. Step 7: Gather --------------- When the user calls ``z.result()`` they wait both on the completion of the computation and for the computation to be sent back over the wire to the local process. Usually this isn't necessary, usually you don't want to move data back to the local process but instead want to keep in on the cluster. But perhaps the user really wanted to actually know this value, so they called ``z.result()``. The scheduler checks who has ``z`` and sends them a message asking for the result. This message doesn't wait in a queue or for other jobs to complete, it starts instantly. The value gets serialized, sent over TCP, and then deserialized and returned to the user (passing through a queue or two on the way.) Step 8: Garbage Collection --------------------------- The user leaves this part of their code and the local variable ``z`` goes out of scope. The Python garbage collector cleans it up. This triggers a decremented reference on the client (we didn't mention this, but when we created the ``Future`` we also started a reference count.) If this is the only instance of a Future pointing to ``z`` then we send a message up to the scheduler that it is OK to release ``z``. The user no longer requires it to persist. The scheduler receives this message and, if there are no computations that might depend on ``z`` in the immediate future, it removes elements of this key from local scheduler state and adds the key to a list of keys to be deleted periodically. Every 500 ms a message goes out to relevant workers telling them which keys they can delete from their local memory. The graph/recipe to create the result of ``z`` persists in the scheduler for all time. Overhead -------- The user experiences this in about 10 milliseconds, depending on network latency. distributed-1.14.3/docs/source/local-cluster.rst000066400000000000000000000015541301211143400216500ustar00rootroot00000000000000Local Cluster ============= For convenience you can start a local cluster from your Python session. .. code-block:: python >>> from distributed import Client, LocalCluster >>> cluster = LocalCluster() LocalCluster("127.0.0.1:8786", workers=8, ncores=8) >>> client = Client(cluster) You can dynamically scale this cluster up and down: .. code-block:: python >>> worker = cluster.add_worker() >>> cluster.remove_worker(worker) Alternatively, a ``LocalCluster`` is made for you automatically if you create an ``Client`` with no arguments: .. code-block:: python >>> from distributed import Client >>> client = Client() >>> client API --- .. currentmodule:: distributed.deploy.local .. autoclass:: LocalCluster :members: distributed-1.14.3/docs/source/locality.rst000066400000000000000000000144611301211143400207200ustar00rootroot00000000000000Data Locality ============= *Data movement often needlessly limits performance.* This is especially true for analytic computations. Dask.distributed minimizes data movement when possible and enables the user to take control when necessary. This document describes current scheduling policies and user API around data locality. Current Policies ---------------- Task Submission ``````````````` In the common case distributed runs tasks on workers that already hold dependent data. If you have a task ``f(x)`` that requires some data ``x`` then that task will very likely be run on the worker that already holds ``x``. If a task requires data split among multiple workers, then the scheduler chooses to run the task on the worker that requires the least data transfer to it. The size of each data element is measured by the workers using the ``sys.getsizeof`` function, which depends on the ``__sizeof__`` protocol generally available on most relevant Python objects. Data Scatter ```````````` When a user scatters data from their local process to the distributed network this data is distributed in a round-robin fashion grouping by number of cores. So for example If we have two workers ``Alice`` and ``Bob``, each with two cores and we scatter out the list ``range(10)`` as follows: .. code-block:: python futures = client.scatter(range(10)) Then Alice and Bob receive the following data * Alice: ``[0, 1, 4, 5, 8, 9]`` * Bob: ``[2, 3, 6, 7]`` User Control ------------ Complex algorithms may require more user control. For example the existence of specialized hardware such as GPUs or database connections may restrict the set of valid workers for a particular task. In these cases use the ``workers=`` keyword argument to the ``submit``, ``map``, or ``scatter`` functions, providing a hostname, IP address, or alias as follows: .. code-block:: python future = client.submit(func, *args, workers=['Alice']) * Alice: ``[0, 1, 4, 5, 8, 9, new_result]`` * Bob: ``[2, 3, 6, 7]`` Required data will always be moved to these workers, even if the volume of that data is significant. If this restriction is only a preference and not a strict requirement, then add the ``allow_other_workers`` keyword argument to signal that in extreme cases such as when no valid worker is present, another may be used. .. code-block:: python future = client.submit(func, *args, workers=['Alice'], allow_other_workers=True) Additionally the ``scatter`` function supports a ``broadcast=`` keyword argument to enforce that the all data is sent to all workers rather than round-robined. If new workers arrive they will not automatically receive this data. .. code-block:: python futures = client.scatter([1, 2, 3], broadcast=True) # send data to all workers * Alice: ``[1, 2, 3]`` * Bob: ``[1, 2, 3]`` Valid arguments for ``workers=`` include the following: * A single IP addresses, IP/Port pair, or hostname like the following:: 192.168.1.100, 192.168.1.100:8989, alice, alice:8989 * A list or set of the above:: ['alice'], ['192.168.1.100', '192.168.1.101:9999'] If only a hostname or IP is given then any worker on that machine will be considered valid. Additionally, you can provide aliases to workers upon creation.:: $ dask-worker scheduler_address:8786 --name worker_1 And then use this name when specifying workers instead. .. code-block:: python client.map(func, sequence, workers='worker_1') Specify workers with Compute/Persist ------------------------------------ The ``workers=`` keyword in ``scatter``, ``submit``, and ``map`` is fairly straightforward, taking either a worker hostname, host:port pair or a sequence of those as valid inputs: .. code-block:: python client.submit(f, x, workers='127.0.0.1') client.submit(f, x, workers='127.0.0.1:55852') client.submit(f, x, workers=['192.168.1.101', '192.168.1.100']) For more complex computations, such as occur with dask collections like dask.dataframe or dask.delayed, we sometimes want to specify that certain parts of the computation run on certain workers while other parts run on other workers. .. code-block:: python x = delayed(f)(1) y = delayed(f)(2) z = delayed(g)(x, y) future = client.compute(z, workers={z: '127.0.0.1', x: '192.168.0.1'}) Here the values of the dictionary are of the same form as before, a host, a host:port pair, or a list of these. The keys in this case are either dask collections or tuples of dask collections. All of the *final* keys of these collections will run on the specified machines; dependencies can run anywhere unless they are also listed in ``workers=``. We explore this through a set of examples: The computation ``z = f(x, y)`` runs on the host ``127.0.0.1``. The other two computations for ``x`` and ``y`` can run anywhere. .. code-block:: python future = client.compute(z, workers={z: '127.0.0.1'}) The computations for both ``z`` and ``x`` must run on ``127.0.0.1`` .. code-block:: python future = client.compute(z, workers={z: '127.0.0.1', x: '127.0.0.1'}) Use a tuple to group collections. This is shorthand for the above. .. code-block:: python future = client.compute(z, workers={(x, y): '127.0.0.1'}) Recall that all options for ``workers=`` in ``scatter/submit/map`` hold here as well. .. code-block:: python future = client.compute(z, workers={(x, y): ['192.168.1.100', '192.168.1.101:9999']}) Set ``allow_other_workers=True`` to make these loose restrictions rather than hard requirements. .. code-block:: python future = client.compute(z, workers={(x, y): '127.0.0.1'}, allow_other_workers=True) Provide a collection to ``allow_other_workers=[...]`` to say that the keys for only some of the collections are loose. In the case below ``z`` *must* run on ``127.0.0.1`` while ``x`` *should* run on ``127.0.0.1`` but can run elsewhere if necessary: .. code-block:: python future = client.compute(z, workers={(x, y): '127.0.0.1'}, allow_other_workers=[x]) This works fine with ``persist`` and with any dask collection (any object with a ``._keys()`` method): .. code-block:: python df = dd.read_csv('s3://...') df = client.persist(df, workers={df: ...}) See the :doc:`efficiency ` page to learn about best practices. distributed-1.14.3/docs/source/manage-computation.rst000066400000000000000000000146221301211143400226670ustar00rootroot00000000000000Managing Computation ==================== Data and Computation in Dask.distributed are always in one of three states 1. Concrete values in local memory. Example include the integer ``1`` or a numpy array in the local process. 2. Lazy computations in a dask graph, perhaps stored in a ``dask.delayed`` or ``dask.dataframe`` object. 3. Running computations or remote data, represented by ``Future`` objects pointing to computations currently in flight. All three of these forms are important and there are functions that convert between all three states. Dask Collections to Concrete Values ----------------------------------- You can turn any dask collection into a concrete value by calling the ``.compute()`` method or ``dask.compute(...)`` function. This function will block until the computation is finished, going straight from a lazy dask collection to a concrete value in local memory. This approach is the most familiar and straightforward, especially for people coming from the standard single-machine Dask experience or from just normal programming. It is great when you have data already in memory and want to get small fast results right to your local process. .. code-block:: python >>> df = dd.read_csv('s3://...') >>> df.value.sum().compute() 100000000 However, this approach often breaks down if you try to bring the entire dataset back to local RAM .. code-block:: python >>> df.compute() MemoryError(...) It also forces you to wait until the computation finishes before handing back control of the interpreter. Dask Collections to Futures --------------------------- You can asynchronously submit lazy dask graphs to run on the cluster with the ``client.compute`` and ``client.persist`` methods. These functions return Future objects immediately. These futures can then be queried to determine the state of the computation. client.compute ~~~~~~~~~~~~~~ The ``.compute`` method takes a collection and returns a single future. .. code-block:: python >>> df = dd.read_csv('s3://...') >>> total = client.compute(df.sum()) # Return a single future >>> total Future(..., status='pending') >>> total.result() # Block until finished 100000000 Because this is a single future the result must fit on a single worker machine. Like ``dask.compute`` above, the ``client.compute`` method is only appropriate when results are small and should fit in memory. The following would likely fail: .. code-block:: python >>> future = client.compute(df) # Blows up memory Instead, you should use ``client.persist`` client.persist ~~~~~~~~~~~~~~ The ``.persist`` method submits the task graph behind the Dask collection to the scheduler, obtaining Futures for all of the top-most tasks (for example one Future for each Pandas DataFrame in a Dask DataFrame). It then returns a copy of the collection pointing to these futures instead of the previous graph. This new collection is semantically equivalent but now points to actively running data rather than a lazy graph. If you look at the dask graph within the collection you will see the Future objects directly: .. code-block:: python >>> df = dd.read_csv('s3://...') >>> df.dask # Recipe to compute df in chunks {('read', 0): (load_s3_bytes, ...), ('parse', 0): (pd.read_csv, ('read', 0)), ('read', 1): (load_s3_bytes, ...), ('parse', 1): (pd.read_csv, ('read', 1)), ... } >>> df = client.persist(df) # Start computation >>> df.dask # Now points to running futures {('parse', 0): Future(..., status='finished'), ('parse', 1): Future(..., status='pending'), ... } The collection is returned immediately and the computation happens in the background on the cluster. Eventually all of the futures of this collection will be completed at which point further queries on this collection will likely be very fast. Typically the workflow is to define a computation with a tool like ``dask.dataframe`` or ``dask.delayed`` until a point where you have a nice dataset to work from, then persist that collection to the cluster and then perform many fast queries off of the resulting collection. Concrete Values to Futures -------------------------- We obtain futures through a few different ways. One is the mechanism above, by wrapping Futures within Dask collections. Another is by submitting data or tasks directly to the cluster with ``client.scatter``, ``client.submit`` or ``client.map``. .. code-block:: python futures = client.scatter(args) # Send data future = client.submit(function, *args, **kwrags) # Send single task futures = client.map(function, sequence, **kwargs) # Send many tasks In this case ``*args`` or ``**kwargs`` can be normal Python objects, like ``1`` or ``'hello'``, or they can be other ``Future`` objects if you want to link tasks together with dependencies. Unlike Dask collections like dask.delayed these task submissions happen immediately. The concurrent.futures interface is very similar to dask.delayed except that execution is immediate rather than lazy. Futures to Concrete Values -------------------------- You can turn an individual ``Future`` into a concrete value in the local process by calling the ``Future.result()`` method. You can convert a collection of futures into concrete values by calling the ``client.gather`` method. .. code-block:: python >>> future.result() 1 >>> client.gather(futures) [1, 2, 3, 4, ...] Futures to Dask Collections --------------------------- As seen in the Collection to futures section it is common to have currently computing ``Future`` objects within Dask graphs. This lets us build further computations on top of currently running computations. This is most often done with dask.delayed workflows on custom computations: .. code-block:: python >>> x = delayed(sum)(futures) >>> y = delayed(product)(futures) >>> future = client.compute(x + y) Mixing the two forms allow you to build and submit a computation in stages like ``sum(...) + product(...)``. This is often valuable if you want to wait to see the values of certain parts of the computation before determining how to proceed. Submitting many computations at once allows the scheduler to be slightly more intelligent when determining what gets run. *If this page interests you then you may also want to check out the doc page on* :doc:`Managing Memory` distributed-1.14.3/docs/source/memory.rst000066400000000000000000000162351301211143400204110ustar00rootroot00000000000000Managing Memory =============== Dask.distributed stores the results of tasks in the distributed memory of the worker nodes. The central scheduler tracks all data on the cluster and determines when data should be freed. Completed results are usually cleared from memory as quickly as possible in order to make room for more computation. The result of a task is kept in memory if either of the following conditions hold: 1. A client holds a future pointing to this task. The data should stay in RAM so that the client can gather the data on demand. 2. The task is necessary for ongoing computations that are working to produce the final results pointed to by futures. These tasks will be removed once no ongoing tasks require them. When users hold Future objects or persisted collections (which contain many such Futures inside their ``.dask`` attribute) they pin those results to active memory. When the user deletes futures or collections from their local Python process the scheduler removes the associated data from distributed RAM. Because of this relationship, distributed memory reflects the state of local memory. A user may free distributed memory on the cluster by deleting persisted collections in the local session. Creating Futures ---------------- The following functions produce Futures .. currentmodule:: distributed.client .. autosummary:: Client.submit Client.map Client.compute Client.persist Client.scatter The ``submit`` and ``map`` methods handle raw Python functions. The ``compute`` and ``persist`` methods handle Dask collections like arrays, bags, delayed values, and dataframes. The ``scatter`` method sends data directly from the local process. Persisting Collections ---------------------- Calls to ``Client.compute`` or ``Client.persist`` submit task graphs to the cluster and return ``Future`` objects that point to particular output tasks. Compute returns a single future per input, persist returns a copy of the collection with each block or partition replaced by a single future. In short, use ``persist`` to keep full collection on the cluster and use ``compute`` when you want a small result as a single future. Persist is more common and is often used as follows with collections: .. code-block:: python >>> # Construct dataframe, no work happens >>> df = dd.read_csv(...) >>> df = df[df.x > 0] >>> df = df.assign(z = df.x + df.y) >>> # Pin data in distributed ram, this triggers computation >>> df = client.persist(df) >>> # continue operating on df *Note for Spark users: this differs from what you're accustomed to. Persist is an immediate action. However, you'll get control back immediately as computation occurs in the background.* In this example we build a computation by parsing CSV data, filtering rows, and then adding a new column. Up until this point all work is lazy; we've just built up a recipe to perform the work as a graph in the ``df`` object. When we call ``df = client.persist(df)`` we cut this graph off of the ``df`` object, send it up to the scheduler, receive ``Future`` objects in return and create a new dataframe with a very shallow graph that points directly to these futures. This happens more or less immediately (as long as it takes to serialize and send the graph) and we can continue working on our new ``df`` object while the cluster works to evaluate the graph in the background. Difference with dask.compute ---------------------------- The operations ``client.persist(df)`` and ``client.compute(df)`` are asynchronous and so differ from the traditional ``df.compute()`` method or ``dask.compute`` function, which blocks until a result is available. The ``.compute()`` method does not persist any data on the cluster. The ``.compute()`` method also brings the entire result back to the local machine, so it is unwise to use it on large datasets. However, ``.compute()`` is very convenient for smaller results particularly because it does return concrete results in a way that most other tools expect. Typically we use asynchronous methods like ``client.persist`` to set up large collections and then use ``df.compute()`` for fast analyses. .. code-block:: python >>> # df.compute() # This is bad and would likely flood local memory >>> df = client.persist(df) # This is good and asynchronously pins df >>> df.x.sum().compute() # This is good because the result is small >>> future = client.compute(df.x.sum()) # This is also good but less intuitive Clearing data ------------- We remove data from distributed ram by removing the collection from our local process. Remote data is removed once all Futures pointing to that data are removed from all client machines. .. code-block:: python >>> del df # Deleting local data often deletes remote data If this is the only copy then this will likely trigger the cluster to delete the data as well. However if we have multiple copies or other collections based on this one then we'll have to delete them all. .. code-block:: python >>> df2 = df[df.x < 10] >>> del df # would not delete data, because df2 still tracks the futures Aggressively Clearing Data -------------------------- To definitely remove a computation and all computations that depend on it you can always ``cancel`` the futures/collection. .. code-block:: python >>> client.cancel(df) # kills df, df2, and every other dependent computation Alternatively, if you want a clean slate, you can restart the cluster. This clears all state and does a hard restart of all worker processes. It generally completes in around a second. .. code-block:: python >>> client.restart() Resilience ---------- Results are not intentionally copied unless necessary for computations on other worker nodes. Resilience is achieved through recomputation by maintaining the provenance of any result. If a worker node goes down the scheduler is able to recompute all of its results. The complete graph for any desired Future is maintained until no references to that future exist. For more information see :doc:`Resilience `. Advanced techniques ------------------- At first the result of a task is not intentionally copied, but only persists on the node where it was originally computed or scattered. However result may be copied to another worker node in the course of normal computation if that result is required by another task that is intended to by run by a different worker. This occurs if a task requires two pieces of data on different machines (at least one must move) or through work stealing. In these cases it is the policy for the second machine to maintain its redundant copy of the data. This helps to organically spread around data that is in high demand. However, advanced users may want to control the location, replication, and balancing of data more directly throughout the cluster. They may know ahead of time that certain data should be broadcast throughout the network or that their data has become particularly imbalanced, or that they want certain pieces of data to live on certain parts of their network. These considerations are not usually necessary. .. currentmodule:: distributed.client .. autosummary:: Client.rebalance Client.replicate Client.scatter distributed-1.14.3/docs/source/plugins.rst000066400000000000000000000001601301211143400205500ustar00rootroot00000000000000Scheduler Plugins ================= .. autoclass:: distributed.diagnostics.plugin.SchedulerPlugin :members: distributed-1.14.3/docs/source/protocol.rst000066400000000000000000000210721301211143400207350ustar00rootroot00000000000000Protocol ======== The scheduler, workers, and clients pass messages between each other. Semantically these messages encode commands, status updates, and data, like the following: * Please compute the function ``sum`` on the data ``x`` and store in ``y`` * The computation ``y`` has been completed * Be advised that a new worker named ``alice`` is available for use * Here is the data for the keys ``'x'``, and ``'y'`` In practice we represent these messages with dictionaries/mappings:: {'op': 'compute', 'function': ... 'args': ['x']} {'op': 'task-complete', 'key': 'y', 'nbytes': 26} {'op': 'register-worker', 'address': '192.168.1.42', 'name': 'alice', 'ncores': 4} {'x': b'...', 'y': b'...'} When we communicate these messages between nodes we need to serialize these messages down to a string of bytes that can then be deserialized on the other end to their in-memory dictionary form. For simple cases several options exist like JSON, MsgPack, Protobuffers, and Thrift. The situation is made more complex by concerns like serializing Python functions and Python objects, optional compression, cross-language support, large messages, and efficiency. This document describes the protocol used by ``dask.distributed`` today. Be advised that this protocol changes rapidly as we continue to optimize for performance. Overview -------- We may split a single message into multiple message-part to suit different protocols. Generally small bits of data are encoded with MsgPack while large bytestrings and complex datatypes are handled by a custom format. Each message-part gets its own header, which is always encoded as msgpack. After serializing all message parts we have a sequence of bytestrings or *frames* which we send along the wire, prepended with length information. The application doesn't know any of this, it just sends us Python dictionaries with various datatypes and we produce a list of bytestrings that get written to a socket. This format is fast both for many frequent messages and for large messages. MsgPack for Messages -------------------- Most messages are encoded with MsgPack_, a self describing semi-structured serialization format that is very similar to JSON, but smaller, faster, not human-readable, and supporting of bytestrings and (soon) timestamps. We chose MsgPack as a base serialization format for the following reasons: * It does not require separate headers, and so is easy and flexible to use which is particularly important in an early stage project like ``dask.distributed`` * It is very fast, much faster than JSON, and there are nicely optimized implementations, particularly within the ``pandas.msgpack`` module. With few exceptions (described later) MsgPack does not come anywhere near being a bottleneck, even under heavy use. * Unlike JSON it supports bytestrings * It covers the standard set of types necessary to encode most information * It is widely implemented in a number of languages (see cross language section below) However, MsgPack fails (correctly) in the following ways: * It does not provide any way for us to encode Python functions or user defined data types * It does not support bytestrings greater than 4GB and is generally inefficient for very large messages. Because of these failings we supplement it with a language-specific protocol and a special case for large bytestrings. CloudPickle for Functions and Some Data --------------------------------------- Pickle and CloudPickle are Python libraries to serialize almost any Python object, including functions. We use these libraries to transform the users' functions and data into bytes before we include them in the dictionary/map that we pass off to msgpack. In the introductory example you may have noticed that we skipped providing an example for the function argument:: {'op': 'compute', 'function': ... 'args': ['x']} That is because this value ``...`` will actually be the result of calling ``cloudpickle.dumps(myfunction)``. Those bytes will then be included in the dictionary that we send off to msgpack, which will only have to deal with bytes rather than obscure Python functions. *Note: we actually call some combination of pickle and cloudpickle, depending on the situation. This is for performance reasons.* Cross Language Specialization ----------------------------- The Client and Workers must agree on a language-specific serialization format. In the standard ``dask.distributed`` client and worker objects this ends up being the following:: bytes = cloudpickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL) obj = cloudpickle.loads(bytes) This varies between Python 2 and 3 and so your client and workers must match their Python versions and software environments. However, the Scheduler never uses the language-specific serialization and instead only deals with MsgPack. If the client sends a pickled function up to the scheduler the scheduler will not unpack function but will instead keep it as bytes. Eventually those bytes will be sent to a worker, which will then unpack the bytes into a proper Python function. Because the Scheduler never unpacks language-specific serialized bytes it may be in a different language. **The client and workers must share the same language and software environment, the scheduler may differ.** This has a few advantages: 1. The Scheduler is protected from unpickling unsafe code 2. The Scheduler can be run under ``pypy`` for improved performance. This is only useful for larger clusters. 3. We could conceivably implement workers and clients for other languages (like R or Julia) and reuse the Python scheduler. The worker and client code is fairly simple and much easier to reimplement than the scheduler, which is complex. 4. The scheduler might some day be rewritten in more heavily optimized C or Go Compression ----------- Fast compression libraries like LZ4 or Snappy may increase effective bandwidth by compressing data before sending and decompressing it after reception. This is especially valuable on lower-bandwidth networks. If either of these libraries is available (we prefer LZ4 to Snappy) then for every message greater than 1kB we try to compress the message and, if the compression is at least a 10% improvement, we send the compressed bytes rather than the original payload. We record the compression used within the header as a string like ``'lz4'`` or ``'snappy'``. To avoid compressing large amounts of uncompressable data we first try to compress a sample. We take 10kB chunks from five locations in the dataset, arrange them together, and try compressing the result. If this doesn't result in significant compression then we don't try to compress the full result. Header ------ The header is a small dictionary encoded in msgpack that includes some metadata about the message, such as compression. Serializing Data ---------------- For administrative messages like updating status msgpack is sufficient. However for large results or Python specific data, like NumPy arrays or Pandas Dataframes, or for larger results we need to use something else to convert Python objects to bytestrings. Exactly how we do this is described more in the `:doc:Serialization documentation `. The application code marks Python specific results with the ``to_serialize`` function: .. code-block:: python >>> import numpy as np >>> x = np.ones(5) >>> from distributed.protocol import to_serialize >>> msg = {'status': 'OK', 'data': to_serialize(x)} >>> msg {'data': , 'status': 'OK'} We separate the message into two messages, one encoding all of the data to be serialized and, and one encoding everything else:: {'key': 'x', 'addresss': 'alice'} {'data': } The first message we pass normally with msgpack. The second we pass in multiple parts, one part for each serialized piece of data (see `:doc: serialization `) and one header including types, compression, etc. used for each value:: {'keys': ['data'], 'compression': ['lz4']} b'...' b'...' Frames ------ At the end of the pipeline we have a sequence of bytestrings or frames. We need to tell the receiving end how many frames there are and how long each these frames are. We order the frames and lengths of frames as follows: 1. The number of frames, stored as an 8 byte unsigned integer 2. The length of each frame, each stored as an 8 byte unsigned integer 3. Each of the frames In the following sections we describe how we create these frames. .. _MsgPack: http://msgpack.org/index.html distributed-1.14.3/docs/source/publish.rst000066400000000000000000000051731301211143400205460ustar00rootroot00000000000000Publish Datasets ================ A *published dataset* is a named reference to a Dask collection or list of futures that has been published to the cluster. It is available for any client to see and persists beyond the scope of an individual session. Publishing datasets is useful in the following cases: * You want to share computations with colleagues * You want to persist results on the cluster between interactive sessions Motivating Example ------------------ In this example we load a dask.dataframe from S3, manipulate it, and then publish the result. **Connect and Load** .. code-block:: python from dask.distributed import Client client = Client('scheduler-address:8786') import dask.dataframe as dd df = dd.read_csv('s3://my-bucket/*.csv') df2 = df[df.balance < 0] df2 = client.persist(df2) >>> df2.head() name balance 0 Alice -100 1 Bob -200 2 Charlie -300 3 Dennis -400 4 Edith -500 **Publish** To share this collection with a colleague we publish it under the name ``'negative_accounts'`` .. code-block:: python client.publish_dataset(negative_accounts=df2) **Load published dataset from different client** Now any other client can connect to the scheduler and retrieve this published dataset. .. code-block:: python >>> from dask.distributed import Client >>> client = Client('scheduler-address:8786') >>> client.list_datasets() ['negative_accounts'] >>> df = client.get_dataset('negative_accounts') >>> df.head() name balance 0 Alice -100 1 Bob -200 2 Charlie -300 3 Dennis -400 4 Edith -500 This allows users to easily share results. It also allows for the persistence of important and commonly used datasets beyond a single session. Published datasets continue to reside in distributed memory even after all clients requesting them have disconnected. Notes ----- Published collections are not automatically persisted. If you publish an un-persisted collection then others will still be able to get the collection from the scheduler, but operations on that collection will start from scratch. This allows you to publish views on data that do not permanently take up cluster memory but can be surprising if you expect "publishing" to automatically make a computed dataset rapidly available. Any client can publish or unpublish a dataset. Publishing too many large datasets can quickly consume a cluster's RAM. API --- .. currentmodule:: distributed.client .. autosummary:: Client.publish_dataset Client.list_datasets Client.get_dataset Client.unpublish_dataset distributed-1.14.3/docs/source/queues.rst000066400000000000000000000114321301211143400204020ustar00rootroot00000000000000Data Streams with Queues ======================== The ``Client`` methods ``scatter``, ``map``, and ``gather`` can consume and produce standard Python ``Queue`` objects. This is useful for processing continuous streams of data. However, it does not constitute a full streaming data processing pipeline like Storm. .. raw:: html Example ------- We connect to a local Client. .. code-block:: python >>> from distributed import Client >>> client = Client('127.0.0.1:8786') >>> client We build a couple of toy data processing functions: .. code-block:: python from time import sleep from random import random def inc(x): from random import random sleep(random() * 2) return x + 1 def double(x): from random import random sleep(random()) return 2 * x And we set up an input Queue and map our functions across it. .. code-block:: python >>> from queue import Queue >>> input_q = Queue() >>> remote_q = client.scatter(input_q) >>> inc_q = client.map(inc, remote_q) >>> double_q = client.map(double, inc_q) We will fill the ``input_q`` with local data from some stream, and then ``remote_q``, ``inc_q`` and ``double_q`` will fill with ``Future`` objects as data gets moved around. We gather the futures from the ``double_q`` back to a queue holding local data in the local process. .. code-block:: python >>> result_q = client.gather(double_q) Insert Data Manually ~~~~~~~~~~~~~~~~~~~~ Because we haven't placed any data into any of the queues everything is empty, including the final output, ``result_q``. .. code-block:: python >>> result_q.qsize() 0 But when we insert an entry into the ``input_q``, it starts to make its way through the pipeline and ends up in the ``result_q``. .. code-block:: python >>> input_q.put(10) >>> result_q.get() 22 Insert data in a separate thread ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ We simulate a slightly more realistic situation by dumping data into the ``input_q`` in a separate thread. This simulates what you might get if you were to read from an active data source. .. code-block:: python def load_data(q): i = 0 while True: q.put(i) sleep(random()) i += 1 >>> from threading import Thread >>> load_thread = Thread(target=load_data, args=(input_q,)) >>> load_thread.start() >>> result_q.qsize() 4 >>> result_q.qsize() 9 We consume data from the ``result_q`` and print results to the screen. .. code-block:: python >>> while True: ... item = result_q.get() ... print(item) 2 4 6 8 10 12 ... Limitations ----------- * This doesn't do any sort of auto-batching of computations, so ideally you batch your data to take significantly longer than 1ms to run. * This isn't a proper streaming system. There is no support outside of what you see here. In particular there are no policies for dropping data, joining over time windows, etc.. Extensions ---------- We can extend this small example to more complex systems that have buffers, split queues, merge queues, etc. all by manipulating normal Python Queues. Here are a couple of useful function to multiplex and merge queues: .. code-block:: python from queue import Queue from threading import Thread def multiplex(n, q, **kwargs): """ Convert one queue into several equivalent Queues >>> q1, q2, q3 = multiplex(3, in_q) """ out_queues = [Queue(**kwargs) for i in range(n)] def f(): while True: x = q.get() for out_q in out_queues: out_q.put(x) t = Thread(target=f) t.daemon = True t.start() return out_queues def push(in_q, out_q): while True: x = in_q.get() out_q.put(x) def merge(*in_qs, **kwargs): """ Merge multiple queues together >>> out_q = merge(q1, q2, q3) """ out_q = Queue(**kwargs) threads = [Thread(target=push, args=(q, out_q)) for q in in_qs] for t in threads: t.daemon = True t.start() return out_q With useful functions like these we can build out more sophisticated data processing pipelines that split off and join back together. By creating queues with ``maxsize=`` we can control buffering and apply back pressure. .. raw:: html distributed-1.14.3/docs/source/quickstart.rst000066400000000000000000000054331301211143400212710ustar00rootroot00000000000000Quickstart ========== Install ------- :: $ pip install distributed --upgrade See :doc:`installation ` document for more information. Setup Dask.distributed the Easy Way ----------------------------------- If you create an client without providing an address it will start up a local scheduler and worker for you. .. code-block:: python >>> from distributed import Client >>> client = Client() >>> client Setup Dask.distributed the Hard Way ----------------------------------- This allows dask.distributed to use multiple machines as workers. Set up scheduler and worker processes on your local computer:: $ dask-scheduler Scheduler started at 127.0.0.1:8786 $ dask-worker 127.0.0.1:8786 $ dask-worker 127.0.0.1:8786 $ dask-worker 127.0.0.1:8786 .. note:: At least one ``dask-worker`` must be running after launching a scheduler. Launch an Client and point it to the IP/port of the scheduler. .. code-block:: python >>> from distributed import Client >>> client = Client('127.0.0.1:8786') See :doc:`setup ` for advanced use. Map and Submit Functions ~~~~~~~~~~~~~~~~~~~~~~~~ Use the ``map`` and ``submit`` methods to launch computations on the cluster. The ``map/submit`` functions send the function and arguments to the remote workers for processing. They return ``Future`` objects that refer to remote data on the cluster. The ``Future`` returns immediately while the computations run remotely in the background. .. code-block:: python >>> def square(x): return x ** 2 >>> def neg(x): return -x >>> A = client.map(square, range(10)) >>> B = client.map(neg, A) >>> total = client.submit(sum, B) >>> total.result() -285 Gather ~~~~~~ The ``map/submit`` functions return ``Future`` objects, lightweight tokens that refer to results on the cluster. By default the results of computations *stay on the cluster*. .. code-block:: python >>> total # Function hasn't yet completed >>> total # Function completed, result ready on remote worker Gather results to your local machine either with the ``Future.result`` method for a single future, or with the ``Client.gather`` method for many futures at once. .. code-block:: python >>> total.result() # result for single future -285 >>> client.gather(A) # gather for many futures [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] Restart ~~~~~~~ When things go wrong, or when you want to reset the cluster state, call the ``restart`` method. .. code-block:: python >>> client.restart() See :doc:`client ` for advanced use. distributed-1.14.3/docs/source/related-work.rst000066400000000000000000000205731301211143400215010ustar00rootroot00000000000000Related Work ============ Writing the "related work" for a project called "distributed", is a Sisyphean task. We'll list a few notable projects that you've probably already heard of down below. You may also find the `dask comparison with spark`_ of interest. .. _`dask comparison with spark`: http://dask.pydata.org/en/latest/spark.html Big Data World -------------- * The venerable Hadoop_ provides batch processing with the MapReduce programming paradigm. Python users typically use `Hadoop Streaming`_ or MRJob_. * Spark builds on top of HDFS systems with a nicer API and in-memory processing. Python users typically use PySpark_. * Storm_ provides streaming computation. Python users typically use streamparse_. This is a woefully inadequate representation of the excellent work blossoming in this space. A variety of projects have come into this space and rival or complement the projects above. Still, most "Big Data" processing hype probably centers around the three projects above, or their derivatives. .. _Hadoop: https://hadoop.apache.org/ .. _MRJob: https://pythonhosted.org/mrjob/ .. _`Hadoop Streaming`: https://hadoop.apache.org/docs/r1.2.1/streaming.html .. _Spark: http://spark.apache.org/ .. _PySpark: http://spark.apache.org/docs/latest/api/python/ .. _storm: http://storm.apache.org/ .. _streamparse: https://streamparse.readthedocs.io/en/latest/index.html .. _Disco: http://discoproject.org/ Python Projects --------------- There are dozens of Python projects for distributed computing. Here we list a few of the more prominent projects that we see in active use today. Task scheduling ~~~~~~~~~~~~~~~ * Celery_: An asynchronous task scheduler, focusing on real-time processing. * Luigi_: A bulk big-data/batch task scheduler, with hooks to a variety of interesting data sources. Ad hoc computation ~~~~~~~~~~~~~~~~~~ * `IPython Parallel`_: Allows for stateful remote control of several running ipython sessions. * Scoop_: Implements the `concurrent.futures`_ API on distributed workers. Notably allows tasks to spawn more tasks. Direct Communication ~~~~~~~~~~~~~~~~~~~~ * MPI4Py_: Wraps the Message Passing Interface popular in high performance computing. * PyZMQ_: Wraps ZeroMQ, the gentleman's socket. Venerable ~~~~~~~~~ There are a couple of older projects that often get mentioned * Dispy_: Embarrassingly parallel function evaluation * Pyro_: Remote objects / RPC .. _Luigi: https://luigi.readthedocs.io/en/latest/ .. _MPI4Py: http://pythonhosted.org/mpi4py/ .. _PyZMQ: https://github.com/zeromq/pyzmq .. _Celery: http://www.celeryproject.org/ .. _`IPython Parallel`: https://ipyparallel.readthedocs.io/en/latest/ .. _Scoop: https://github.com/soravux/scoop/ .. _`concurrent.futures`: https://docs.python.org/3/library/concurrent.futures.html .. _Dispy: http://dispy.sourceforge.net/ .. _Pyro: https://pythonhosted.org/Pyro4/ Relationship ------------ In relation to these projects ``distributed``... * Supports data-local computation like Hadoop and Spark * Uses a task graph with data dependencies abstraction like Luigi * In support of ad-hoc applications, like IPython Parallel and Scoop In depth comparison to particular projects ------------------------------------------ IPython Parallel ~~~~~~~~~~~~~~~~ **Short Description** `IPython Parallel`_ is a distributed computing framework from the IPython project. It uses a centralized hub to farm out jobs to several ``ipengine`` processes running on remote workers. It communicates over ZeroMQ sockets and centralizes communication through the central hub. IPython parallel has been around for a while and, while not particularly fancy, is quite stable and robust. IPython Parallel offers parallel ``map`` and remote ``apply`` functions that route computations to remote workers .. code-block:: python >>> view = Client(...)[:] >>> results = view.map(func, sequence) >>> result = view.apply(func, *args, **kwargs) >>> future = view.apply_async(func, *args, **kwargs) It also provides direct execution of code in the remote process and collection of data from the remote namespace. .. code-block:: python >>> view.execute('x = 1 + 2') >>> view['x'] [3, 3, 3, 3, 3, 3] **Brief Comparison** Distributed and IPython Parallel are similar in that they provide ``map`` and ``apply/submit`` abstractions over distributed worker processes running Python. Both manage the remote namespaces of those worker processes. They are dissimilar in terms of their maturity, how worker nodes communicate to each other, and in the complexity of algorithms that they enable. **Distributed Advantages** The primary advantages of ``distributed`` over IPython Parallel include 1. Peer-to-peer communication between workers 2. Dynamic task scheduling ``Distributed`` workers share data in a peer-to-peer fashion, without having to send intermediate results through a central bottleneck. This allows ``distributed`` to be more effective for more complex algorithms and to manage larger datasets in a more natural manner. IPython parallel does not provide a mechanism for workers to communicate with each other, except by using the central node as an intermediary for data transfer or by relying on some other medium, like a shared file system. Data transfer through the central node can easily become a bottleneck and so IPython parallel has been mostly helpful in embarrassingly parallel work (the bulk of applications) but has not been used extensively for more sophisticated algorithms that require non-trivial communication patterns. The distributed client includes a dynamic task scheduler capable of managing deep data dependencies between tasks. The IPython parallel docs include `a recipe`_ for executing task graphs with data dependencies. This same idea is core to all of ``distributed``, which uses a dynamic task scheduler for all operations. Notably, ``distributed.Future`` objects can be used within ``submit/map/get`` calls before they have completed. .. code-block:: python >>> x = client.submit(f, 1) # returns a future >>> y = client.submit(f, 2) # returns a future >>> z = client.submit(add, x, y) # consumes futures The ability to use futures cheaply within ``submit`` and ``map`` methods enables the construction of very sophisticated data pipelines with simple code. Additionally, distributed can serve as a full dask task scheduler, enabling support for distributed arrays, dataframes, machine learning pipelines, and any other application build on dask graphs. The dynamic task schedulers within ``distributed`` are adapted from the dask_ task schedulers and so are fairly sophisticated/efficient. **IPython Parallel Advantages** IPython Parallel has the following advantages over ``distributed`` 1. Maturity: IPython Parallel has been around for a while. 2. Explicit control over the worker processes: IPython parallel allows you to execute arbitrary statements on the workers, allowing it to serve in system administration tasks. 3. Deployment help: IPython Parallel has mechanisms built-in to aid deployment on SGE, MPI, etc.. Distributed does not have any such sugar, though is fairly simple to :doc:`set up ` by hand. 4. Various other advantages: Over the years IPython parallel has accrued a variety of helpful features like IPython interaction magics, ``@parallel`` decorators, etc.. .. _`a recipe`: https://ipython.org/ipython-doc/3/parallel/dag_dependencies.html#dag-dependencies .. _dask: http://dask.pydata.org/en/latest/ concurrent.futures ~~~~~~~~~~~~~~~~~~ The ``distributed.Client`` API is modeled after ``concurrent.futures`` and PEP-3184_. It has a few notable differences: * ``distributed`` accepts ``Future`` objects within calls to ``submit/map``. It is preferable to submit Future objects directly rather than wait on them before submission. * The ``map`` function returns ``Future`` objects, not concrete results. The ``map`` function returns immediately. * It is not yet possible to cancel a ``Future`` (though this is theoretically possible please raise an issue if this is of concrete importance to you.) * Distributed generally does not support timeouts or callbacks ``distributed.CompatibleExecutor`` is a subclass of ``distributed.Client`` that does conform to the ``concurrent.futures`` API, allowing it to be used as a drop-in replacement for other Executors using the common API. .. _PEP-3184: https://www.python.org/dev/peps/pep-3148/ distributed-1.14.3/docs/source/resilience.rst000066400000000000000000000061611301211143400212200ustar00rootroot00000000000000Resilience ========== Software fails, Hardware fails, network connections fail, user code fails. This document describes how ``dask.distributed`` responds in the face of these failures and other known bugs. User code failures ------------------ When a function raises an error that error is kept and transmitted to the client on request. Any attempt to gather that result *or any dependent result* will raise that exception. .. code-block:: python >>> def div(a, b): ... return a / b >>> x = client.submit(div, 1, 0) >>> x.result() ZeroDivisionError: division by zero >>> y = client.submit(add, x, 10) >>> y.result() # same error as above ZeroDivisionError: division by zero This does not affect the smooth operation of the scheduler or worker in any way. Closed Network Connections -------------------------- If the connection to a remote worker unexpectedly closes and the local process appropriately raises an ``IOError`` then the scheduler will reroute all pending computations to other workers. If the lost worker was the only worker to hold vital results necessary for future computations then those results will be recomputed by surviving workers. The scheduler maintains a full history of how each result was produced and so is able to reproduce those same computations on other workers. This has some fail cases. 1. If results depend on impure functions then you may get a different (although still entirely accurate) result 2. If the worker failed due to a bad function, for example a function that causes a segmentation fault, then that bad function will repeatedly be called on other workers. This function will be marked as "bad" after it kills a fixed number of workers (defaults to three). 3. Data scattered out to the workers is not kept in the scheduler (it is often quite large) and so the loss of this data is irreparable. You may wish to call ``Client.replicate`` on the data with a suitable replication factor to ensure that it remains long-lived or else back the data off of some resilient store, like a file system. Hardware Failures ----------------- It is not clear under which circumstances the local process will know that the remote worker has closed the connection. If the socket does not close cleanly then the system will wait for a timeout, roughly three seconds, before marking the worker as failed and resuming smooth operation. Scheduler Failure ----------------- The process containing the scheduler might die. There is currently no persistence mechanism to record and recover the scheduler state. The workers and clients will all reconnect to the scheduler after it comes back online but records of ongoing computations will be lost. Restart and Nanny Processes --------------------------- The client provides a mechanism to restart all of the workers in the cluster. This is convenient if, during the course of experimentation, you find your workers in an inconvenient state that makes them unresponsive. The ``Client.restart`` method kills all workers, flushes all scheduler state, and then brings all workers back online, resulting in a clean cluster. distributed-1.14.3/docs/source/scheduling-policies.rst000066400000000000000000000120131301211143400230210ustar00rootroot00000000000000Scheduling Policies =================== This document describes the policies used to select the preference of tasks and to select the preference of workers used by Dask's distributed scheduler. For more information on how this these policies are enacted efficiently see :doc:`Scheduling State`. Choosing Workers ---------------- When a task transitions from waiting to a ready state we may decide a suitable worker for that task. If the task has significant data dependencies or if the workers are under heavy load then this choice of worker can strongly impact global performance. Currently workers for tasks are determined as follows: 1. If the task has no major dependencies and no restrictions then it goes into a common pool of tasks to be used by the next worker with nothing better to do. 2. Otherwise, if a task has user-provided restrictions (for example it must run on a machine with a GPU) then we restrict the available pool of workers to just that set, otherwise we consider all workers 3. From among this pool of workers we determine the workers to whom the least amount of data would need to be transferred. 4. We break ties by choosing the worker that currently has the fewest tasks, counting both those tasks in memory and those tasks processing currently. This process is easy to change (and indeed this document may be outdated). We encourage readers to inspect the ``decide_worker`` function in scheduler.py .. currentmodule:: distributed.scheduler .. autosummary:: decide_worker Choosing Tasks -------------- We often have a choice between running many valid tasks. There are a few competing interests that might motivate our choice: 1. Run tasks on a first-come-first-served basis for fairness between multiple clients 2. Run tasks that are part of the critical path in an effort to reduce total running time and minimize straggler workloads 3. Run tasks that allow us to release many dependencies in an effort to keep the memory footprint small 4. Run tasks that are related so that large chunks of work can be completely eliminated before running new chunks of work Accomplishing all of these objectives simultaneously is impossible. Optimizing for any of these objectives perfectly can result in costly overhead. The heuristics with the scheduler do a decent but imperfect job of optimizing for all of these (they all come up in important workloads) quickly. Last in, first out ~~~~~~~~~~~~~~~~~~ When a worker finishes a task the immediate dependencies of that task get top priority. This encourages a behavior of finishing ongoing work immediately before starting new work. This often conflicts with the first-come-first-served objective but often results in shorter total runtimes and significantly reduced memory footprints. Break ties with children and depth ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Often a task has multiple dependencies and we need to break ties between them with some other objective. Breaking these ties has a surprisingly strong impact on performance and memory footprint. When a client submits a graph we perform a few linear scans over the graph to determine something like the number of descendants of each node (not quite, because it's a DAG rather than a tree, but this is a close proxy). This number can be used to break ties and helps us to prioritize nodes with longer critical paths and nodes with many children. The actual algorithms used are somewhat more complex and are described in detail in `dask/order.py`_ .. _`dask/order.py`: https://github.com/dask/dask/blob/master/dask/order.py Initial Task Placement ~~~~~~~~~~~~~~~~~~~~~~ When a new large batch of tasks come in and there are many idle workers then we want to give each worker a set of tasks that are close together/related and unrelated from the tasks given to other workers. This usually avoids inter-worker communication down the line. The same depth-first-with-child-weights priority given to workers described above can usually be used to properly segment the leaves of a graph into decently well separated sub-graphs with relatively low inter-sub-graph connectedness. First-Come-First-Served, Coarsely ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The last-in-first-out behavior used by the workers to minimize memory footprint can distort the task order provided by the clients. Tasks submitted recently may run sooner than tasks submitted long ago because they happen to be more convenient given the current data in memory. This behavior can be *unfair* but improves global runtimes and system efficiency, sometimes quite significantly. However, workers inevitably run out of tasks that were related to tasks they were just working on and the last-in-first-out policy eventually exhausts itself. In these cases workers often pull tasks from the common task pool. The tasks in this pool *are* ordered in a first-come-first-served basis and so workers do behave in a fair scheduling manner at a *coarse* level if not a fine grained one. Dask's scheduling policies are short-term-efficient and long-term-fair. distributed-1.14.3/docs/source/scheduling-state.rst000066400000000000000000000366251301211143400223510ustar00rootroot00000000000000Scheduling State ================ Overview -------- The life of a computation with Dask can be described in the following stages: 1. The user authors a graph using some library, perhaps Dask.delayed or dask.dataframe or the ``submit/map`` functions on the client. They submit these tasks to the scheduler. 2. The schedulers assimilates these tasks into its graph of all tasks to track and as their dependencies become available it asks workers to run each of these tasks. 3. The worker receives information about how to run the task, communicates with its peer workers to collect dependencies, and then runs the relevant function on the appropriate data. It reports back to the scheduler that it has finished. 4. The scheduler reports back to the user that the task has completed. If the user desires, it then fetches the data from the worker through the scheduler. Most relevant logic is in tracking tasks as they evolve from newly submitted, to waiting for dependencies, to ready to run, to actively running on some worker, to finished in memory, to garbage collected. Tracking this process, and tracking all effects that this task has on other tasks that might depend on it, is the majority of the complexity of the dynamic task scheduler. This section describes the system used to perform this tracking. For more abstract information about the policies used by the scheduler, see :doc:`Scheduling Policies`. State Variables --------------- We start with a description of the state that the scheduler keeps on each task. Each of the following is a dictionary keyed by task name (described below): * **tasks:** ``{key: task}``: Dictionary mapping key to a serialized task. A key is the name of a task, generally formed from the name of the function, followed by a hash of the function and arguments, like ``'inc-ab31c010444977004d656610d2d421ec'``. The value of this dictionary is the task, which is an unevaluated function and arguments. This is stored in one of two forms: * ``{'function': inc, 'args': (1,), 'kwargs': {}}``; a dictionary with the function, arguments, and keyword arguments (kwargs). However in the scheduler these are stored serialized, as they were sent from the client, so it looks more like ``{'function': b'\x80\x04\x95\xcb\...', 'args': b'...', }`` * ``{'task': (inc, 1)}``: a tuple satisfying the dask graph protocol. This again is stored serialized. These are the values that will eventually be sent to a worker when the task is ready to run. * **dependencies and dependents:** ``{key: {keys}}``: These are dictionaries which show which tasks depend on which others. They contain redundant information. If ``dependencies[a] == {b, c}`` then the task with the name of ``a`` depends on the results of the two tasks with the names of ``b`` and ``c``. There will be complimentary entries in dependents such that ``a in dependents[b]`` and ``a in dependents[c]`` such as ``dependents[b] == {a, d}``. Keeping the information around twice allows for constant-time access for either direction of query, so we can both look up a task's out-edges or in-edges efficiently. * **waiting and waiting_data:** ``{key: {keys}}``: These are dictionaries very similar to dependencies and dependents, but they only track keys that are still in play. For example ``waiting`` looks like ``dependencies``, tracking all of the tasks that a certain task requires before it can run. However as tasks are completed and arrive in memory they are removed from their dependents sets in ``waiting``, so that when a set becomes empty we know that a key is ready to run and ready to be allocated to a worker. The ``waiting_data`` dictionary on the other hand holds all of the dependents of a key that have yet to run and still require that this task stay in memory in services of tasks that may depend on it (its ``dependents``). When a value set in this dictionary becomes empty its task may be garbage collected (unless some client actively desires that this task stay in memory). * **task_state:** ``{key: string}``: The ``task_state`` dictionary holds the current state of every key. Current valid states include released, waiting, queue, stacks, no-worker, processing, memory, and erred. These states are explained further below. * **priority:** ``{key: tuple}``: The ``priority`` dictionary provides each key with a relative ranking. This ranking is generally a tuple of two parts. The first (and dominant) part corresponds to when it was submitted. Generally earlier tasks take precedence. The second part is determined by the client, and is a way to prioritize tasks within a large graph that may be important, such as if they are on the critical path, or good to run in order to release many dependencies. This is explained further in :doc:`Scheduling Policy ` A key's priority is only used to break ties, when many keys are being considered for execution. The priority does *not* determine running order, but does exert some subtle influence that does significantly shape the long term performance of the cluster. * **ready:** ``deque(key)`` A deque of keys that are ready to run now but haven't yet been sent to a worker to run. These keys show no affinity to any particular worker and so can be run equally well by anyone. This is a common pool of tasks. * **stacks:** ``{worker: [keys]}``: Keys that are ready to run and show some affinity to a particular worker. These keys typically have dependencies that are known to be on that worker or have user-defined restrictions that require them to run on certain nodes. A worker pulls from this stack first before taking tasks from the common pool of ``ready`` tasks. * **processing:** ``{worker: {key: cost}}``: Keys that are currently running on a worker. This is keyed by worker address and contains the expected cost in seconds of running that task. * **rprocessing:** ``{key: {worker}}``: The reverse of the ``processing`` dictionary. This is all keys that are currently running with a set of all workers that are currently running them. This is redundant with ``processing`` and just here for faster indexed querying. It is rare for these sets to be larger than size one but does happen in rare cases when a task might be submitted redundantly to multiple workers. * **who_has:** ``{key: {worker}}``: For keys that are in memory this shows on which workers they currently reside. * **has_what:** ``{worker: {key}}``: This is the transpose of ``who_has``, showing all keys that currently reside on each worker. * **released:** ``{keys}`` The set of keys that are known, but released from memory. These have typically run to completion and are no longer necessary. * **unrunnable:** ``{key}`` The set ``unrunnable`` contains keys that are not currently able to run, probably because they have a user defined restriction (described below) that is not met by any available worker. These keys are waiting for an appropriate worker to join the network before computing. * **restrictions:** ``{key: {hostnames}}``: A set of hostnames per key of where that key can be run. Usually this is empty unless a key has been specifically restricted to only run on certain hosts. These restrictions don't include a worker port. Any worker on that hostname is deemed valid. * **loose_restrictions:** ``{key}``: Set of keys for which we are allowed to violate restrictions (see above) if not valid workers are present and the task would otherwise go into the ``unrunnable`` set. * **exceptions and tracebacks:** ``{key: Exception/Traceback}``: Dictionaries mapping keys to remote exceptions and tracebacks. When tasks fail we store their exceptions and tracebacks (serialized from the worker) here so that users may gather the exceptions to see the error. * **exceptions_blame:** ``{key: key}``: If a task fails then we mark all of its dependent tasks as failed as well. This dictionary lets any failed task see which task was the origin of its failure. * **suspicious_tasks:** ``{key: int}`` Number of times a task has been involved in a worker failure. Some tasks may cause workers to fail (such as ``sys.exit(0)``). When a worker fails all of the tasks on that worker are reassigned to others. This combination of behaviors can cause a bad task to catastrophically destroy all workers on the cluster, one after another. Whenever a worker fails we mark each task currently running on that worker as suspicious. If a task is involved in three failures (or some other fixed constant) then we mark the task as failed. * **who_wants:** ``{key: {client}}``: When a client submits a graph to the scheduler it also specifies which output keys it desires. Those keys are tracked here where each desired key knows which clients want it. These keys will not be released from memory and, when they complete, messages will be sent to all of these clients that the task is ready. * **wants_what:** ``{client: {key}}``: The transpose of ``who_wants``. * **nbytes:** ``{key: int}``: The number of bytes, as determined by ``sizeof``, of the result of each finished task. This number is used for diagnostics and to help prioritize work. * **stealable:** ``[[key]]`` A list of stacks of stealable keys, ordered by stealability. For more information see :doc:`Work Stealing ` Example Event and Response -------------------------- Whenever an event happens, like when a client sends up more tasks, or when a worker finishes a task, the scheduler changes the state above. For example when a worker reports that a task has finished we perform actions like the following: **Task `key` finished by `worker`**: .. code-block:: python task_state[key] = 'memory' who_has[key].add(worker) has_what[worker].add(key) nbytes[key] = nbytes processing[worker].remove(key) del rprocessing[key] if key in who_wants: send_done_message_to_clients(who_wants[key]) for dep in dependencies[key]: waiting_data[dep].remove(key) for dep in dependents[key]: waiting[dep].remove(key) if stacks[worker]: next_task = stacks[worker].pop() elif ready: next_task = ready.pop() else: idle.add(worker) State Transitions ----------------- The code presented in the section above is just for demonstration. In practice writing this code for every possible event is highly error prone, resulting in hard-to-track-down bugs. Instead the scheduler moves tasks between a fixed set of states, notably ``'released', 'waiting', 'queue', 'stacks', 'no-worker', 'processing', 'memory', 'error'``. Transitions between common pairs of states are well defined and, if no path exists between a pair, the graph of transitions can be traversed to find a valid sequence of transitions. Along with these transitions come consistent logging and optional runtime checks that are useful in testing. Tasks fall into the following states with the following allowed transitions .. image:: images/task-state.svg :alt: Dask scheduler task states * Released: known but not actively computing or in memory * Waiting: On track to be computed, waiting on dependencies to arrive in memory * Queue (ready): Ready to be computed by any worker * Stacks (ready): Ready to be computed by a particular preferred worker * No-worker (ready, rare): Ready to be computed, but no appropriate worker exists * Processing: Actively being computed by one or more workers * Memory: In memory on one or more workers * Erred: Task has computed and erred * Forgotten (not actually a state): Task is no longer needed by any client and so it removed from state Every transition between states is a separate method in the scheduler. These task transition functions are prefixed with ``transition`` and then have the name of the start and finish task state like the following. .. code-block:: python def transition_released_waiting(self, key): def transition_processing_memory(self, key): def transition_processing_erred(self, key): These functions each have three effects. 1. They perform the necessary transformations on the scheduler state (the 20 dicts/lists/sets) to move one key between states. 2. They return a dictionary of recommended ``{key: state}`` transitions to enact directly afterwards on other keys. For example after we transition a key into memory we may find that many waiting keys are now ready to transition from waiting to a ready state. 3. Optionally they include a set of validation checks that can be turned on for testing. Rather than call these functions directly we call the central function ``transition``: .. code-block:: python def transition(self, key, final_state): """ Transition key to the suggested state """ This transition function finds the appropriate path from the current to the final state. It also serves as a central point for logging and diagnostics. Often we want to enact several transitions at once or want to continually respond to new transitions recommended by initial transitions until we reach a steady state. For that we use the ``transitions`` function (note the plural ``s``). .. code-block:: python def transitions(self, recommendations): recommendations = recommendations.copy() while recommendations: key, finish = recommendations.popitem() new = self.transition(key, finish) recommendations.update(new) This function runs ``transition``, takes the recommendations and runs them as well, repeating until no further task-transitions are recommended. Stimuli ------- Transitions occur from stimuli, which are state-changing messages to the scheduler from workers or clients. The scheduler responds to the following stimuli: * **Workers** * Task finished: A task has completed on a worker and is now in memory * Task erred: A task ran and erred on a worker * Task missing data: A task tried to run but was unable to find necessary data on other workers * Worker added: A new worker was added to the network * Worker removed: An existing worker left the network * **Clients** * Update graph: The client sends more tasks to the scheduler * Release keys: The client no longer desires the result of certain keys Stimuli functions are prepended with the text ``stimulus``, and take a variety of keyword arguments from the message as in the following examples: .. code-block:: python def stimulus_task_finished(self, key=None, worker=None, nbytes=None, type=None, compute_start=None, compute_stop=None, transfer_start=None, transfer_stop=None): def stimulus_task_erred(self, key=None, worker=None, exception=None, traceback=None) These functions change some non-essential administrative state and then call transition functions. Note that there are several other non-state-changing messages that we receive from the workers and clients, such as messages requesting information about the current state of the scheduler. These are not considered stimuli. API --- .. currentmodule:: distributed.scheduler .. autoclass:: Scheduler :members: .. autofunction:: decide_worker distributed-1.14.3/docs/source/serialization.rst000066400000000000000000000030141301211143400217450ustar00rootroot00000000000000Custom Serialization ==================== When we communicate data between computers we first convert that data into a sequence of bytes that can be communicated across the network. Dask can convert data to bytes using the standard solutions of Pickle and Cloudpickle. However, sometimes pickle and cloudpickle are suboptimal so Dask also supports custom serialization formats for special types. This helps Dask to be faster on common formats like NumPy and Pandas and gives power-users more control about how their objects get moved around on the network if they want to extend the system. We include a small example and then follow with the full API documentation describing the ``serialize`` and ``deserialize`` functions, which convert objects into a msgpack header and a list of bytestrings and back. Example ------- Here is how we special case handling raw Python bytes objects. In this case there is no need to call ``pickle.dumps`` on the object. The object is already a sequnce of bytes. .. code-block:: python def serialize_bytes(obj): header = {} # no special metadata frames = [obj] return header, frames def deserialize_bytes(header, frames): return frames[0] register_serialization(bytes, serialize_bytes, deserialize_bytes) API --- .. currentmodule:: distributed.protocol.serialize .. autosummary:: register_serialization serialize deserialize .. autofunction:: register_serialization .. autofunction:: serialize .. autofunction:: deserialize distributed-1.14.3/docs/source/setup.rst000066400000000000000000000134661301211143400202440ustar00rootroot00000000000000Setup Network ============= A ``dask.distributed`` network consists of one ``Scheduler`` node and several ``Worker`` nodes. One can set these up in a variety of ways Using the Command Line ---------------------- We launch the ``dask-scheduler`` executable in one process and the ``dask-worker`` executable in several processes, possibly on different machines. Launch ``dask-scheduler`` on one node:: $ dask-scheduler Start scheduler at 192.168.0.1:8786 Then launch ``dask-worker`` on the rest of the nodes, providing the address to the node that hosts ``dask-scheduler``:: $ dask-worker 192.168.0.1:8786 Start worker at: 192.168.0.2:12345 Registered with center at: 192.168.0.1:8786 $ dask-worker 192.168.0.1:8786 Start worker at: 192.168.0.3:12346 Registered with center at: 192.168.0.1:8786 $ dask-worker 192.168.0.1:8786 Start worker at: 192.168.0.4:12347 Registered with center at: 192.168.0.1:8786 There are various mechanisms to deploy these executables on a cluster, ranging from manualy SSH-ing into all of the nodes to more automated systems like SGE/SLURM/Torque or Yarn/Mesos. Additionally, cluster SSH tools exist to send the same commands to many machines. One example is `tmux-cssh`__. __ https://github.com/dennishafemann/tmux-cssh Using SSH --------- The convenience script ``dask-ssh`` opens several SSH connections to your target computers and initializes the network accordingly. You can give it a list of hostnames or IP addresses:: $ dask-ssh 192.168.0.1 192.168.0.2 192.168.0.3 192.168.0.4 Or you can use normal UNIX grouping:: $ dask-ssh 192.168.0.{1,2,3,4} Or you can specify a hostfile that includes a list of hosts:: $ cat hostfile.txt 192.168.0.1 192.168.0.2 192.168.0.3 192.168.0.4 $ dask-ssh --hostfile hostfile.txt The ``dask-ssh`` utility depends on the ``paramiko``:: pip install paramiko Using the Python API -------------------- Alternatively you can start up the ``distributed.scheduler.Scheduler`` and ``distributed.worker.Worker`` objects within a Python session manually. Both are ``torando.tcpserver.TCPServer`` objects. Start the Scheduler, provide the listening port (defaults to 8786) and Tornado IOLoop (defaults to ``IOLoop.current()``) .. code-block:: python from distributed import Scheduler from tornado.ioloop import IOLoop from threading import Thread loop = IOLoop.current() t = Thread(target=loop.start, daemon=True) t.start() s = Scheduler(loop=loop) s.start(8786) On other nodes start worker processes that point to the IP address and port of the scheduler. .. code-block:: python from distributed import Worker from tornado.ioloop import IOLoop from threading import Thread loop = IOLoop.current() t = Thread(target=loop.start, daemon=True) t.start() w = Worker('127.0.0.1', 8786, loop=loop) w.start(0) # choose randomly assigned port Alternatively, replace ``Worker`` with ``Nanny`` if you want your workers to be managed in a separate process by a local nanny process. This allows workers to restart themselves in case of failure, provides some additional monitoring, and is useful when coordinating many workers that should live in different processes to avoid the GIL_. .. _GIL: https://docs.python.org/3/glossary.html#term-gil Using LocalCluster ------------------ You can do the work above easily using :doc:`LocalCluster`. .. code-block:: python from distributed import LocalCluster c = LocalCluster(nanny=False) A scheduler will be available under ``c.scheduler`` and a list of workers under ``c.workers``. There is an IOLoop running in a background thread. Using Amazon EC2 ---------------- See the :doc:`EC2 quickstart ` for information on the ``dask-ec2`` easy setup script to launch a canned cluster on EC2. Cluster Resource Managers ------------------------- Dask.distributed has been deployed on dozens of different cluster resource managers. This section contains links to some external projects, scripts, and instructions that may serve as useful starting points. Kubernetes ~~~~~~~~~~ * https://github.com/ogrisel/docker-distributed * https://github.com/hammerlab/dask-distributed-on-kubernetes/ Marathon ~~~~~~~~ * https://github.com/mrocklin/dask-marathon DRMAA (SGE, SLURM, Torque, etc..) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * https://github.com/mfouesneau/dasksge YARN ~~~~ * https://knit.readthedocs.io/en/latest/ Software Environment -------------------- The workers and clients should all share the same software environment. That means that they should all have access to the same libraries and that those libraries should be the same version. Dask generally assumes that it can call a function on any worker with the same outcome (unless explicitly told otherwise.) This is typically enforced through external means, such as by having a network file system (NFS) mount for libraries, by starting the ``dask-worker`` processes in equivalent Docker_ containers, using Conda_ environments, or through any of the other means typically employed by cluster administrators. .. _Docker: https://www.docker.com/ .. _Conda: http://conda.pydata.org/docs/ Windows ~~~~~~~ .. note:: - Running a ``dask-scheduler`` on Windows architectures is supported for only a limited number of workers (roughly 100). This is a detail of the underlying tcp server implementation and is discussed `here`__. - Running ``dask-worker`` processes on Windows is well supported, performant, and without limit. If you wish to run in a primarily Windows environment, it is recommneded to run a ``dask-scheduler`` on a linux or MacOSX environment, with ``dask-worker`` workers on the Windows boxes. This works because the scheduler environment is de-coupled from that of the workers. __ https://github.com/jfisteus/ztreamy/issues/26 distributed-1.14.3/docs/source/submitting-applications.rst000066400000000000000000000030651301211143400237470ustar00rootroot00000000000000Submitting Applications ======================= The ``dask-submit`` cli can be used to submit an application to the dask cluster running remotely. If your code depends on resources that can only be access from cluster running dask, ``dask-submit`` provides a mechanism to send the script to the cluster for execution from a different machine. For example, S3 buckets could not be visible from your local machine and hence any attempt to create a dask graph from local machine may not be work. Submitting dask Applications with `dask-submit` ----------------------------------------------- In order to remotely submit scripts to the cluster from a local machine or a CI/CD environment, we need to run a remote client on the same machine as the scheduler:: #scheduler machine dask-remote --port 8788 After making sure the `dask-remote` is running, you can submit a script by:: #local machine dask-submit : Some of the commonly used arguments are: - ``REMOTE_CLIENT_ADDRESS``: host name where dask-remote client is running - ``FILEPATH``: Local path to file containing dask application For example, given the following dask application saved in a file called ``script.py``: .. code-block:: python from distributed import Client def inc(x): return x + 1 if __name__=='__main__': client = Client('127.0.0.1:8786') x = client.submit(inc, 10) print(x.result()) We can submit this application from a local machine by running:: dask-submit : script.py distributed-1.14.3/docs/source/task-launch.rst000066400000000000000000000141551301211143400213120ustar00rootroot00000000000000Launch Tasks from Tasks ======================= Sometimes it is convenient to launch tasks from other tasks. For example you may not know what computations to run until you have the results of some initial computations. Motivating example ------------------ We want to download one piece of data and turn it into a list. Then we want to submit one task for every element of that list. We don't know how long the list will be until we have the data. So we send off our original ``download_and_convert_to_list`` function, which downloads the data and converts it to a list on one of our worker machines: .. code-block:: python future = e.submit(download_and_convert_to_list, uri) But now we need to submit new tasks for individual parts of this data. We have three options. 1. Gather the data back to the local process and then submit new jobs from the local process 2. Gather only enough information about the data back to the local process and submit jobs from the local process 3. Submit a task to the cluster that will submit other tasks directly from that worker Gather the data locally ----------------------- If the data is not large then we can bring it back to the client to perform the necessary logic on our local machine: .. code-block:: python >>> data = future.result() # gather data to local process >>> data # data is a list [...] >>> futures = e.map(process_element, data) # submit new tasks on data >>> analysis = e.submit(aggregate, futures) # submit final aggregation task This is straightforward and, if ``data`` is small then it is probably the simplest, and therefore correct choice. However, if ``data`` is large then we have to choose another option. Submit tasks from client ------------------------ We can run small functions on our remote data to determine enough to submit the right kinds of tasks. In the following example we compute the ``len`` function on ``data`` remotely and then break up data into its various elements. .. code-block:: python >>> n = e.submit(len, data) # compute number of elements >>> n = n.result() # gather n (small) locally >>> from operator import getitem >>> elements = [e.submit(getitem, data, i) for i in range(n)] # split data >>> futures = e.map(process_element, elements) >>> analysis = e.submit(aggregate, futures) We compute the length remotely, gather back this very small result, and then use it to submit more tasks to break up the data and process on the cluster. This is more complex because we had to go back and forth a couple of times between the cluster and the local process, but the data moved was very small, and so this only added a few milliseconds to our total processing time. Submit tasks from worker ------------------------ *Note: this interface is new and experimental. It may be changed without warning in future versions.* We can submit tasks from other tasks. This allows us to make decisions while on worker nodes. To submit new tasks from a worker that worker must first create a new client object that connects to the scheduler. There is a convenience function to do this for you so that you don't have to pass around connection information. However you must use this function ``local_client`` as a context manager to ensure proper cleanup on the worker. .. code-block:: python from distributed import local_client def process_all(data): with local_client() as e: elements = e.scatter(data) futures = e.map(process_element, elements) analysis = e.submit(aggregate, futures) result = analysis.result() return result analysis = e.submit(process_all, data) # spawns many tasks This approach is somewhat complex but very powerful. It allows you to spawn tasks that themselves act as potentially long-running clients, managing their own independent workloads. Extended Example ~~~~~~~~~~~~~~~~ This example computing the Fibonacci numbers creates tasks that submit tasks that submit tasks that submit other tasks, etc.. .. code-block:: python In [1]: from distributed import Client, local_client In [2]: client = Client() In [3]: def fib(n): ...: if n < 2: ...: return n ...: else: ...: with local_client() as c ...: a = c.submit(fib, n - 1) ...: b = c.submit(fib, n - 2) ...: a, b = c.gather([a, b]) ...: return a + b ...: In [4]: future = e.submit(fib, 100) In [5]: future Out[5]: In [6]: future.result() Out[6]: 354224848179261915075 This example is a bit extreme and spends most of its time establishing client connections from the worker rather than doing actual work, but does demonstrate that even pathological cases function robustly. Technical details ~~~~~~~~~~~~~~~~~ Tasks that invoke ``local_client`` are conservatively assumed to be *long running*. They can take a long time blocking, waiting for other tasks to finish, gathering results, etc.. In order to avoid having them take up processing slots the following actions occur whenever a task invokes ``local_client``. 1. The thread on the worker running this function *secedes* from the thread pool and goes off on its own. This allows the thread pool to populate that slot with a new thread and continue processing additional tasks without counting this long running task against its normal quota. 2. The Worker sends a message back to the scheduler temporarily increasing its allowed number of tasks by one. This likewise lets the scheduler allocate more tasks to this worker, not counting this long running task against it. Because of this behavior you can happily launch long running control tasks that manage worker-side clients happily, without fear of deadlocking the cluster. Establishing a connection to the scheduler takes on the order of 10-20 ms and so it is wise for computations that use this feature to be at least a few times longer in duration than this. distributed-1.14.3/docs/source/web.rst000066400000000000000000000142741301211143400176570ustar00rootroot00000000000000Web Interface ============= .. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/daskboard.gif :alt: Dask Bokeh Dashboard :width: 80% Information about the current state of the network helps to track progress, identify performance issues, and debug failures. Dask.distributed includes a web interface to help deliver this information over a normal web page in real time. This web interface is launched by default wherever the scheduler is launched if the scheduler machine has Bokeh_ installed (``conda install bokeh -c bokeh``). The web interface is normally available at ``http://scheduler-address:8787/status/`` and can be viewed any normal web browser. The available pages are ``http://scheduler-address:8787//`` where ```` is one of - ``status`` - ``tasks`` - ``workers`` .. _Bokeh: http://bokeh.pydata.org/en/latest/ Plots ----- Example Computation ~~~~~~~~~~~~~~~~~~~ The following plots show a trace of the following computation: .. code-block:: python from distributed import Client from time import sleep import random def inc(x): sleep(random.random() / 10) return x + 1 def dec(x): sleep(random.random() / 10) return x - 1 def add(x, y): sleep(random.random() / 10) return x + y client = Client('127.0.0.1:8786') incs = client.map(inc, range(100)) decs = client.map(dec, range(100)) adds = client.map(add, incs, decs) total = client.submit(sum, adds) del incs, decs, adds total.result() Progress ~~~~~~~~ The interface shows the progress of the various computations as well as the exact number completed. .. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/bokeh-progress.gif :alt: Resources view of Dask web interface Each bar is assigned a color according to the function being run. Each bar has a few components. On the left the lighter shade is the number of tasks that have both completed and have been released from memory. The darker shade to the right corresponds to the tasks that are completed and whose data still reside in memory. If errors occur then they appear as a black colored block to the right. Typical computations may involve dozens of kinds of functions. We handle this visually with the following approaches: 1. Functions are ordered by the number of total tasks 2. The colors are assigned in a round-robin fashion from a standard palette 3. The progress bars shrink horizontally to make space for more functions 4. Only the largest functions (in terms of number of tasks) are displayed .. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/bokeh-progress-large.gif :alt: Progress bar plot of Dask web interface Counts of tasks processing, waiting for dependencies, processing, etc.. are displayed in the title bar. Memory Use ~~~~~~~~~~ The interface shows the relative memory use of each function with a horizontal bar sorted by function name. .. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/bokeh-memory-use.gif :alt: Memory use plot of Dask web interface The title shows the number of total bytes in use. Hovering over any bar tells you the specific function and how many bytes its results are actively taking up in memory. This does not count data that has been released. Task Stream ~~~~~~~~~~~ The task stream plot shows when tasks complete on which workers. Worker cores are on the y-axis and time is on the x-axis. As a worker completes a task its start and end times are recorded and a rectangle is added to this plot accordingly. .. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/bokeh-task-stream.gif :alt: Task stream plot of Dask web interface If data transfer occurs between workers a *red* bar appears preceding the task bar showing the duration of the transfer. If an error occurs than a *black* bar replaces the normal color. This plot show the last 1000 tasks. It resets if there is a delay greater than 10 seconds. For a full history of the last 100,000 tasks see the ``tasks/`` page. Resources ~~~~~~~~~ The resources plot show the average CPU and Memory use over time as well as average network traffic. More detailed information on a per-worker basis is available in the ``workers/`` page. .. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/bokeh-resources.gif :alt: Resources view of Dask web interface Connecting to Web Interface --------------------------- Default ~~~~~~~ By default, ``dask-scheduler`` prints out the address of the web interface:: INFO - Bokeh UI at: http://10.129.39.91:8787/status ... INFO - Starting Bokeh server on port 8787 with applications at paths ['/status', '/tasks'] The machine hosting the scheduler runs an HTTP server serving at that address. Troubleshooting --------------- Some clusters restrict the ports that are visible to the outside world. These ports may include the default port for the web interface, ``8787``. There are a few ways to handle this: 1. Open port ``8787`` to the outside world. Often this involves asking your cluster administrator. 2. Use a different port that is publicly accessible using the ``--bokeh-port PORT`` option on the ``dask-scheduler`` command. 3. Use fancier techniques, like `Port Forwarding`_ .. _`Port Forwarding`: https://en.wikipedia.org/wiki/Port_forwarding Running distributed on a remote machine can cause issues with viewing the web UI -- this depends on the remote machines network configuration. Port Forwarding ~~~~~~~~~~~~~~~ If you have SSH access then one way to gain access to a blocked port is through SSH port forwarding. A typical use case looks like the following: .. code:: bash local$ ssh -L 8000:localhost:8787 user@remote remote$ dask-scheduler # now, the web UI is visible at localhost:8000 remote$ # continue to set up dask if needed -- add workers, etc It is then possible to go to ``localhost:8000`` and see Dask Web UI. This same approach is not specific to dask.distributed, but can be used by any service that operates over a network, such as Jupyter notebooks. For example, if we chose to do this we could forward port 8888 (the default Jupyter port) to port 8001 with ``ssh -L 8001:localhost:8888 user@remote``. distributed-1.14.3/docs/source/work-stealing.rst000066400000000000000000000103351301211143400216620ustar00rootroot00000000000000Work Stealing ============= Some tasks prefer to run on certain workers. This may be because that worker holds data dependencies of the task or because the user has expressed a loose desire that the task run in a particular place. Occasionally this results in a few very busy workers and several idle workers. In this situation the idle workers may choose to steal work from the busy workers, even if stealing work requires the costly movement of data. This is a performance optimization and not required for correctness. Work stealing provides robustness in many ad-hoc cases, but can also backfire when we steal the wrong tasks and reduce performance. Criteria for stealing -------------------------- Computation to Communication Ratio ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Stealing is profitable when the computation time for a task is much longer than the communication time of the task's dependencies. **Bad example** We do not want to steal tasks that require moving a large dependent piece of data across a wire from the victim to the thief if the computation is fast. We end up spending far more time in communication than just waiting a bit longer and giving up on parallelism. .. code-block:: python [data] = client.scatter([np.arange(1000000000)]) x = client.submit(np.sum, data) **Good example** We do want to steal task tasks that only need to move dependent pieces of data, especially when the computation time is expensive (here 100 seconds.) .. code-block:: python [data] = client.scatter([100]) x = client.submit(sleep, data) Fortunately we often know both the number of bytes of dependencies (as reported by calling ``sys.getsizeof`` on the workers) and the runtime cost of previously seen functions, which is maintained as an exponentially weighted moving average. Saturated Worker Burden ~~~~~~~~~~~~~~~~~~~~~~~ Stealing may be profitable even when the computation-time to communication-time ratio is poor. This occurs when the saturated workers have a very long backlog of tasks and there are a large number of idle workers. We determine if it acceptable to steal a task if the last task to be run by the saturated workers would finish more quickly if stolen or if it remains on the original/victim worker. The longer the backlog of stealable tasks, and the smaller the number of active workers we have both increase our willingness to steal. This is balanced against the compute-to-communicate cost ratio. Replicate Popular Data ~~~~~~~~~~~~~~~~~~~~~~ It is also good long term if stealing causes highly-sought-after data to be replicated on more workers. Steal from the Rich ~~~~~~~~~~~~~~~~~~~ We would like to steal tasks from particularly over-burdened workers rather than workers with just a few excess tasks. Restrictions ~~~~~~~~~~~~ If a task has been specifically restricted to run on particular workers (such as is the case when special hardware is required) then we do not steal. Choosing tasks to steal ----------------------- We maintain a list of sets of stealable tasks, ordered into bins by computation-to-communication time ratio. The first bin contains all tasks with a compute-to-communicate ratio greater than or equal to 8 (considered high enough to always steal), the next bin with a ratio of 4, the next bin with a ratio of 2, etc.., all the way down to a ratio of 1/256, which we will never steal. This data structure provides a somewhat-ordered view of all stealable tasks that we can add to and remove from in constant time, rather than ``log(n)`` as with more traditional data structures, like a heap. During any stage when we submit tasks to workers we check if there are both idle and saturated workers and if so we quickly run through this list of sets, selecting tasks from the best buckets first, working our way down to the buckets of less desirable stealable tasks. We stop either when there are no more stealable tasks, no more idle workers, or when the quality of the task-to-be-stolen is not high enough given the current backlog. This approach is fast, optimizes to steal the tasks with the best computation-to-communication cost ratio (up to a factor of two) and tends to steal from the workers that have the largest backlogs, just by nature that random selection tends to draw from the largest population. distributed-1.14.3/docs/source/worker.rst000066400000000000000000000137101301211143400204050ustar00rootroot00000000000000Worker ====== Workers provide two functions: 1. Workers compute tasks as directed by the scheduler 2. Workers hold and serve computed results, both for each other and for the clients Each worker contains a ThreadPool that it uses to evaluate tasks as requested by the scheduler. It stores the results of these tasks locally and serves them to the scheduler or to other workers on demand. If the worker is asked to evaluate a task for which it does not have all of the necessary data then it will reach out to its peer workers to gather the necessary dependencies. A typical conversation between a scheduler and two workers Alice and Bob may look like the following:: Scheduler -> Alice: Compute ``x <- add(1, 2)``! Alice -> Scheduler: I've computed x and am holding on to it! Scheduler -> Bob: Compute ``y <- add(x, 10)``! You will need x. Alice has x. Bob -> Alice: Please send me x. Alice -> Bob: Sure. x is 3! Bob -> Scheduler: I've computed y and am holding on to it! Storing Data ------------ Data is stored locally in a dictionary in the ``.data`` attribute that maps keys to the results of function calls. .. code-block:: python >>> worker.data {'x': 3, 'y': 13, ... '(df, 0)': pd.DataFrame(...), ... } Spill Excess Data to Disk ------------------------- Short version: To enable workers to spill excess data to disk start ``dask-worker`` with the ``--memory-limit`` option. Either giving ``auto`` to have it guess how many bytes to keep in memory or an integer, if you know the number of bytes it should use:: $ dask-worker scheduler:port --memory-limit=auto # 75% of available RAM $ dask-worker scheduler:port --memory-limit=2e9 # two gigabytes Some workloads may produce more data at one time than there is available RAM on the cluster. In these cases Workers may choose to write excess values to disk. This causes some performance degradation because writing to and reading from disk is generally slower than accessing memory, but is better than running out of memory entirely, which can cause the system to halt. If the ``dask-worker --memory-limit=NBYTES`` keyword is set during initialization then the worker will store at most NBYTES of data (as measured with ``sizeof``) in memory. After that it will start storing least recently used (LRU) data in a temporary directory. Workers serialize data for writing to disk with the same system used to write data on the wire, a combination of ``pickle`` and the default compressor. Now whenever new data comes in it will push out old data until at most NBYTES of data is in RAM. If an old value is requested it will be read from disk, possibly pushing other values down. It is still possible to run out of RAM on a worker. Here are a few possible issues: 1. The objects being stored take up more RAM than is stated with the `__sizeof__ protocol `_. If you use custom classes then we encourage adding a faithful ``__sizeof__`` method to your class that returns an accurate accounting of the bytes used. 2. Computations and communications may take up additional RAM not accounted for. It is wise to have a suitable buffer of memory that can handle your most expensive function RAM-wise running as many times as there are active threads on the machine. 3. It is possible to misjudge the amount of RAM on the machine. Using the ``--memory-limit=auto`` heuristic sets the value to 75% of the return value of ``psutil.virtual_memory().total``. Thread Pool ----------- Each worker sends computations to a thread in a `concurrent.futures.ThreadPoolExecutor `_ for computation. These computations occur in the same process as the Worker communication server so that they can access and share data efficiently between each other. For the purposes of data locality all threads within a worker are considered the same worker. If your computations are mostly numeric in nature (for example NumPy and Pandas computations) and release the GIL entirely then it is advisable to run ``dask-worker`` processes with many threads and one process. This reduces communication costs and generally simplifies deployment. If your computations are mostly Python code and don't release the GIL then it is advisable to run ``dask-worker`` processes with many processes and one thread per core:: $ dask-worker scheduler:8786 --nprocs 8 If your computations are external to Python and long-running and don't release the GIL then beware that while the computation is running the worker process will not be able to communicate to other workers or to the scheduler. This situation should be avoided. If you don't link in your own custom C/Fortran code then this topic probably doesn't apply to you. Command Line tool ----------------- Use the ``dask-worker`` command line tool to start an individual worker. Here are the available options:: $ dask-worker --help Usage: dask-worker [OPTIONS] SCHEDULER Options: --worker-port INTEGER Serving worker port, defaults to randomly assigned --http-port INTEGER Serving http port, defaults to randomly assigned --nanny-port INTEGER Serving nanny port, defaults to randomly assigned --port INTEGER Deprecated, see --nanny-port --host TEXT Serving host. Defaults to an ip address that can hopefully be visible from the scheduler network. --nthreads INTEGER Number of threads per process. Defaults to number of cores --nprocs INTEGER Number of worker processes. Defaults to one. --name TEXT Alias --memory-limit TEXT Number of bytes before spilling data to disk --no-nanny --help Show this message and exit. API Documentation ----------------- .. autoclass:: distributed.worker.Worker distributed-1.14.3/release-notes.md000066400000000000000000000116221301211143400172020ustar00rootroot000000000000001.14.3 - November 13th, 2016 ---------------------------- * Remove custom Bokeh export tool that implicitly relied on nodejs #655 * Clean up scheduler logging #657 1.14.2 - November 11th, 2016 ---------------------------- * Support more numpy dtypes in custom serialization, #627, #630, #636 * Update Bokeh plots #628 * Improve spill to disk heuristics #633 * Add Export tool to Task Stream plot * Reverse frame order in loads for very many frames #651 * Add timeout when waiting on write #653 1.14.0 - November 3rd, 2016 --------------------------- * Add ``Client.get_versions()`` function to return software and package information from the scheduler, workers, and client #595 * Improved windows support #577 #590 #583 #597 * Clean up rpc objects explicitly #584 * Normalize collections against known futures #587 * Add key= keyword to map to specify keynames #589 * Custom data serialization #606 * Refactor the web interface #608 #615 #621 * Allow user-supplied Executor in Worker #609 * Pass Worker kwargs through LocalCluster 1.13.3 - October 15th, 2016 --------------------------- * Schedulers can retire workers cleanly * Add Future.add_done_callback for concurrent.futures compatibility * Update web interface to be consistent with Bokeh 0.12.3 * Close streams explicitly, avoiding race conditions and supporting more robust restarts on Windows. * Improved shuffled performance for dask.dataframe * Add adaptive allocation cluster manager * Reduce administrative overhead when dealing with many workers * dask-ssh --log-directory . no longer errors * Microperformance tuning for the scheduler 1.13.2 ------ * Revert dask_worker to use fork rather than subprocess by default * Scatter retains type information * Bokeh always uses subprocess rather than spawn 1.13.1 ------ * Fix critical Windows error with dask_worker executable 1.13.0 ------ * Rename Executor to Client #492 * Add `--memory-limit` option to `dask-worker`, enabling spill-to-disk behavior when running out of memory #485 * Add `--pid-file` option to dask-worker and `--dask-scheduler` #496 * Add ``upload_environment`` function to distribute conda environments. This is experimental, undocumented, and may change without notice. # 494 * Add `workers=` keyword argument to `Client.compute` and `Client.persist`, supporting location-restricted workloads with Dask collections #484 * Add ``upload_environment`` function to distribute conda environments. This is experimental, undocumented, and may change without notice. # 494 * Add optional `dask_worker=` keyword to `client.run` functions that gets provided the worker or nanny object * Add `nanny=False` keyword to `Client.run`, allowing for the execution of arbitrary functions on the nannies as well as normal workers 1.12.2 ------ This release adds some new features and removes dead code * Publish and share datasets on the scheduler between many clients https://github.com/dask/distributed/pull/453 https://distributed.readthedocs.io/en/latest/publish.html * Launch tasks from other tasks (experimental) https://distributed.readthedocs.io/en/latest/task-launch.html https://github.com/dask/distributed/pull/471 * Remove unused code, notably the `Center` object and older client functions https://github.com/dask/distributed/pull/478 * Executor() and LocalCluster() is now robust to Bokeh's absence https://github.com/dask/distributed/pull/481 * Removed s3fs and boto3 from requirements. These have moved to Dask. 1.12.1 ------ This release is largely a bugfix release, recovering from the previous large refactor. * Fixes from previous refactor * Ensure idempotence across clients * Stress test losing scattered data permanently * IPython fixes * Add `start_ipython_scheduler` method to Executor * Add `%remote` magic for workers * Clean up code and tests * Pool connects to maintain reuse and reduce number of open file handles * Re-implement work stealing algorithm * Support cancellation of tuple keys, such as occur in dask.arrays * Start synchronizing against worker data that may be superfluous * Improve bokeh plots styling * Add memory plot tracking number of bytes * Make the progress bars more compact and align colors * Add workers/ page with workers table, stacks/processing plot, and memory * Add this release notes document 1.12.0 ------ This release was largely a refactoring release. Internals were changed significantly without many new features. * Major refactor of the scheduler to use transitions system * Tweak protocol to traverse down complex messages in search of large bytestrings * Add dask-submit and dask-remote * Refactor HDFS writing to align with changes in the dask library * Executor reconnects to scheduler on broken connection or failed scheduler * Support sklearn.external.joblib as well as normal joblib distributed-1.14.3/requirements.txt000066400000000000000000000002041301211143400173700ustar00rootroot00000000000000tornado >= 4.2 toolz >= 0.7.4 msgpack-python cloudpickle >= 0.2.1 dask >= 0.12.0 click >= 6.6 six tblib psutil locket zict >= 0.1.0 distributed-1.14.3/setup.py000077500000000000000000000033151301211143400156270ustar00rootroot00000000000000#!/usr/bin/env python import os from setuptools import setup import sys requires = open('requirements.txt').read().strip().split('\n') if sys.version_info[0] < 3: requires.append('futures') requires.append('locket') if sys.version_info < (3, 4): requires.append('singledispatch') setup(name='distributed', version='1.14.3', description='Distributed computing', url='https://distributed.readthedocs.io/en/latest/', maintainer='Matthew Rocklin', maintainer_email='mrocklin@gmail.com', license='BSD', package_data={ '': ['templates/index.html'], }, include_package_data=True, install_requires=requires, packages=['distributed', 'distributed.bokeh', 'distributed.bokeh.background', 'distributed.bokeh.status', 'distributed.bokeh.tasks', 'distributed.bokeh.workers', 'distributed.cli', 'distributed.deploy', 'distributed.diagnostics', 'distributed.protocol', 'distributed.http'], long_description=(open('README.md').read() if os.path.exists('README.md') else ''), entry_points=''' [console_scripts] dask-ssh=distributed.cli.dask_ssh:go dask-submit=distributed.cli.dask_submit:go dask-remote=distributed.cli.dask_remote:go dask-scheduler=distributed.cli.dask_scheduler:go dask-worker=distributed.cli.dask_worker:go dcenter=distributed.cli.dcenter:go dcluster=distributed.cli.dcluster:go dscheduler=distributed.cli.dscheduler:go dworker=distributed.cli.dworker:go ''', zip_safe=False)