How to use the distributed.utils.key_split function in distributed

To help you get started, we’ve selected a few distributed examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github dask / distributed / distributed / worker.py View on Github external
async def actor_execute(
        self, comm=None, actor=None, function=None, args=(), kwargs={}
    ):
        separate_thread = kwargs.pop("separate_thread", True)
        key = actor
        actor = self.actors[key]
        func = getattr(actor, function)
        name = key_split(key) + "." + function

        if iscoroutinefunction(func):
            result = await func(*args, **kwargs)
        elif separate_thread:
            result = await self.executor_submit(
                name,
                apply_function_actor,
                args=(
                    func,
                    args,
                    kwargs,
                    self.execution_state,
                    name,
                    self.active_threads,
                    self.active_threads_lock,
                ),
github dask / distributed / distributed / diagnostics / progress_stream.py View on Github external
def color_of_message(msg):
    if msg["status"] == "OK":
        split = key_split(msg["key"])
        return color_of(split)
    else:
        return "black"
github dask / distributed / distributed / http / scheduler.py View on Github external
def get(self):
        resp = {addr: [ensure_string(key_split(t)) for t in tasks]
                for addr, tasks in self.server.processing.items()}
        self.write(resp)
github dask / distributed / distributed / bokeh / status_monitor.py View on Github external
def task_stream_append(lists, msg, workers, palette=task_stream_palette):
    start, stop = msg['compute_start'], msg['compute_stop']
    lists['start'].append((start + stop) / 2 * 1000)
    lists['duration'].append(1000 * (stop - start))
    key = msg['key']
    name = key_split(key)
    if msg['status'] == 'OK':
        color = palette[incrementing_index(name) % len(palette)]
    else:
        color = 'black'
    lists['key'].append(key)
    lists['name'].append(name)
    lists['color'].append(color)
    lists['alpha'].append(1)
    lists['worker'].append(msg['worker'])

    worker_thread = '%s-%d' % (msg['worker'], msg['thread'])
    lists['worker_thread'].append(worker_thread)
    if worker_thread not in workers:
        workers[worker_thread] = len(workers)
    lists['y'].append(workers[worker_thread])
github dask / distributed / distributed / scheduler.py View on Github external
def get_task_duration(self, key, default=0.5):
        """
        Get the estimated computation cost of the given key
        (not including any communication cost).
        """
        ks = key_split(key)
        try:
            return self.task_duration[ks]
        except KeyError:
            self.unknown_durations[ks].add(key)
            return default
github dask / distributed / distributed / dashboard / components / worker.py View on Github external
def process_msg(self, msg):
        try:

            def func(k):
                return msg["keys"].get(k, 0)

            status_key = max(msg["keys"], key=func)
            typ = self.worker.types.get(status_key, object).__name__
            keyname = key_split(status_key)
            d = {
                "nbytes": msg["total"],
                "duration": msg["duration"],
                "bandwidth": msg["bandwidth"],
                "count": len(msg["keys"]),
                "type": typ,
                "type-color": color_of(typ),
                "key": keyname,
                "key-color": color_of(keyname),
                "start": msg["start"],
                "stop": msg["stop"],
            }
            return d
        except Exception as e:
            logger.exception(e)
            raise
github dask / distributed / distributed / bokeh / resource_monitor.py View on Github external
def func(scheduler):
            """ Get tasks running or waiting on each worker """
            workers = [k for k, v in sorted(scheduler.ncores.items(),
                                            key=lambda x: x[0], reverse=True)]
            processing = [list(map(key_split, scheduler.processing[w]))
                          for w in workers]
            nprocessing = list(map(len, processing))
            nstacks = [len(scheduler.stacks[w]) for w in workers]

            return {'host': [w.split(':')[0] for w in workers],
                    'processing': processing,
                    'nprocessing': nprocessing,
                    'waiting': nstacks}
github dask / distributed / distributed / scheduler.py View on Github external
We stop the task from being stolen in the future, and change task
        duration accounting as if the task has stopped.
        """
        if 'stealing' in self.extensions:
            self.extensions['stealing'].remove_key_from_stealable(key)

        try:
            actual_worker = self.rprocessing[key]
        except KeyError:
            logger.debug("Received long-running signal from duplicate task. "
                         "Ignoring.")
            return

        if compute_duration:
            ks = key_split(key)
            old_duration = self.task_duration.get(ks, 0)
            new_duration = compute_duration
            if not old_duration:
                avg_duration = new_duration
            else:
                avg_duration = (0.5 * old_duration
                              + 0.5 * new_duration)

            self.task_duration[ks] = avg_duration

        worker = self.rprocessing[key]
        self.occupancy[actual_worker] -= self.processing[actual_worker][key]
        self.processing[actual_worker][key] = 0
github dask / distributed / distributed / diagnostics / websocket.py View on Github external
finish: string
            Final state of the transition.
        *args, **kwargs: More options passed when transitioning
            This may include worker ID, compute time, etc.
        """
        if key not in self.scheduler.tasks:
            return
        kwargs["key"] = key
        startstops = kwargs.get("startstops", [])
        for startstop in startstops:
            color = colors[startstop["action"]]
            if type(color) is not str:
                color = color(kwargs)
            data = {
                "key": key,
                "name": key_split(key),
                "color": color,
                **kwargs,
                **startstop,
            }
            self.socket.send("transition", data)
github dask / distributed / distributed / diagnostics / scheduler.py View on Github external
'cpu': 0.0,
                   'last-seen': 0.003068,
                   'latency': 0.01584628690034151,
                   'addresses': ['tcp://127.0.0.1:54871', 'tcp://127.0.0.1:50943'],
                   'processing': {'inc': 2, 'add': 1},
                   'disk-read': 1234,
                   'disk-write': 1234,
                   'network-send': 1234,
                   'network-recv': 1234,
                   'memory': 16701911040,
                   'memory_percent': 85}}
    """
    hosts = {host: d['addresses']
             for host, d in s.host_info.items()}

    processing = {host: countby(key_split, concat(s.processing[w] for w in addrs))
                  for host, addrs in hosts.items()}

    now = time()

    result = {}
    for host, info in s.host_info.items():
        info = info.copy()
        # info = dissoc(info, 'heartbeat', 'heartbeat-port')
        info['processing'] = processing[host]
        result[host] = info
        info['addresses'] = sorted(info['addresses'])
        if 'last-seen' in info:
            info['last-seen'] = (now - info['last-seen'])

    return result