2020-03-21T17:51:18.6288296Z ================================== FAILURES ===================================
2020-03-21T17:51:18.6288604Z _________________________ test_recompute_released_key _________________________
2020-03-21T17:51:18.6288900Z
2020-03-21T17:51:18.6290201Z def test_func():
2020-03-21T17:51:18.6290435Z result = None
2020-03-21T17:51:18.6290735Z workers = []
2020-03-21T17:51:18.6291000Z with clean(timeout=active_rpc_timeout, **clean_kwargs) as loop:
2020-03-21T17:51:18.6292016Z
2020-03-21T17:51:18.6292270Z async def coro():
2020-03-21T17:51:18.6292538Z with dask.config.set(config):
2020-03-21T17:51:18.6292793Z s = False
2020-03-21T17:51:18.6293043Z for i in range(5):
2020-03-21T17:51:18.6293357Z try:
2020-03-21T17:51:18.6293623Z s, ws = await start_cluster(
2020-03-21T17:51:18.6293899Z nthreads,
2020-03-21T17:51:18.6294154Z scheduler,
2020-03-21T17:51:18.6294457Z loop,
2020-03-21T17:51:18.6294726Z security=security,
2020-03-21T17:51:18.6295085Z Worker=Worker,
2020-03-21T17:51:18.6295315Z scheduler_kwargs=scheduler_kwargs,
2020-03-21T17:51:18.6295639Z worker_kwargs=worker_kwargs,
2020-03-21T17:51:18.6295904Z )
2020-03-21T17:51:18.6296162Z except Exception as e:
2020-03-21T17:51:18.6296415Z logger.error(
2020-03-21T17:51:18.6296727Z "Failed to start gen_cluster, retrying",
2020-03-21T17:51:18.6297002Z exc_info=True,
2020-03-21T17:51:18.6297252Z )
2020-03-21T17:51:18.6297503Z else:
2020-03-21T17:51:18.6297799Z workers[:] = ws
2020-03-21T17:51:18.6298055Z args = [s] + workers
2020-03-21T17:51:18.6298310Z break
2020-03-21T17:51:18.6298577Z if s is False:
2020-03-21T17:51:18.6298923Z raise Exception("Could not start cluster")
2020-03-21T17:51:18.6299188Z if client:
2020-03-21T17:51:18.6299465Z c = await Client(
2020-03-21T17:51:18.6299760Z s.address,
2020-03-21T17:51:18.6300011Z loop=loop,
2020-03-21T17:51:18.6300220Z security=security,
2020-03-21T17:51:18.6300518Z asynchronous=True,
2020-03-21T17:51:18.6300774Z **client_kwargs
2020-03-21T17:51:18.6301022Z )
2020-03-21T17:51:18.6301311Z args = [c] + args
2020-03-21T17:51:18.6301564Z try:
2020-03-21T17:51:18.6301818Z future = func(*args)
2020-03-21T17:51:18.6302244Z if timeout:
2020-03-21T17:51:18.6302544Z future = asyncio.wait_for(future, timeout)
2020-03-21T17:51:18.6302844Z result = await future
2020-03-21T17:51:18.6303098Z if s.validate:
2020-03-21T17:51:18.6303387Z s.validate_state()
2020-03-21T17:51:18.6303643Z finally:
2020-03-21T17:51:18.6303934Z if client and c.status not in ("closing", "closed"):
2020-03-21T17:51:18.6304235Z await c._close(fast=s.status == "closed")
2020-03-21T17:51:18.6304541Z await end_cluster(s, workers)
2020-03-21T17:51:18.6305038Z await asyncio.wait_for(cleanup_global_workers(), 1)
2020-03-21T17:51:18.6305325Z
2020-03-21T17:51:18.6305587Z try:
2020-03-21T17:51:18.6305864Z c = await default_client()
2020-03-21T17:51:18.6306160Z except ValueError:
2020-03-21T17:51:18.6306444Z pass
2020-03-21T17:51:18.6306726Z else:
2020-03-21T17:51:18.6307011Z await c._close(fast=True)
2020-03-21T17:51:18.6307265Z
2020-03-21T17:51:18.6307536Z for i in range(5):
2020-03-21T17:51:18.6307913Z if all(c.closed() for c in Comm._instances):
2020-03-21T17:51:18.6308225Z break
2020-03-21T17:51:18.6308515Z else:
2020-03-21T17:51:18.6308773Z await asyncio.sleep(0.05)
2020-03-21T17:51:18.6309057Z else:
2020-03-21T17:51:18.6309340Z L = [c for c in Comm._instances if not c.closed()]
2020-03-21T17:51:18.6309632Z Comm._instances.clear()
2020-03-21T17:51:18.6309923Z # raise ValueError("Unclosed Comms", L)
2020-03-21T17:51:18.6310930Z print("Unclosed Comms", L)
2020-03-21T17:51:18.6311324Z
2020-03-21T17:51:18.6311819Z return result
2020-03-21T17:51:18.6312101Z
2020-03-21T17:51:18.6312301Z result = loop.run_sync(
2020-03-21T17:51:18.6312572Z > coro, timeout=timeout * 2 if timeout else timeout
2020-03-21T17:51:18.6312846Z )
2020-03-21T17:51:18.6313087Z
2020-03-21T17:51:18.6313334Z distributed\utils_test.py:957:
2020-03-21T17:51:18.6313626Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2020-03-21T17:51:18.6313955Z C:\Miniconda3\envs\testenv\lib\site-packages\tornado\ioloop.py:576: in run_sync
2020-03-21T17:51:18.6314247Z return future_cell[0].result()
2020-03-21T17:51:18.6314719Z distributed\utils_test.py:927: in coro
2020-03-21T17:51:18.6315016Z result = await future
2020-03-21T17:51:18.6315552Z C:\Miniconda3\envs\testenv\lib\asyncio\tasks.py:442: in wait_for
2020-03-21T17:51:18.6315846Z return fut.result()
2020-03-21T17:51:18.6316411Z C:\Miniconda3\envs\testenv\lib\site-packages\tornado\gen.py:1147: in run
2020-03-21T17:51:18.6316716Z yielded = self.gen.send(value)
2020-03-21T17:51:18.6317391Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2020-03-21T17:51:18.6317634Z
2020-03-21T17:51:18.6318067Z c = <Client: not connected>
2020-03-21T17:51:18.6318580Z s = <Scheduler: "tcp://127.0.0.1:58695" processes: 0 cores: 0>
2020-03-21T17:51:18.6319448Z a = <Worker: 'tcp://127.0.0.1:58696', 0, closed, stored: 0, running: 0/1, ready: 0, comm: 0, waiting: 0>
2020-03-21T17:51:18.6320032Z b = <Worker: 'tcp://127.0.0.1:58697', 1, closed, stored: 0, running: 0/2, ready: 0, comm: 0, waiting: 0>
2020-03-21T17:51:18.6320321Z
2020-03-21T17:51:18.6320547Z @gen_cluster(timeout=1000, client=True)
2020-03-21T17:51:18.6320921Z def test_recompute_released_key(c, s, a, b):
2020-03-21T17:51:18.6321204Z x = c.submit(inc, 100)
2020-03-21T17:51:18.6321491Z result1 = yield x
2020-03-21T17:51:18.6321771Z xkey = x.key
2020-03-21T17:51:18.6322025Z del x
2020-03-21T17:51:18.6322309Z import gc
2020-03-21T17:51:18.6322563Z
2020-03-21T17:51:18.6322800Z gc.collect()
2020-03-21T17:51:18.6323084Z yield gen.moment
2020-03-21T17:51:18.6323342Z > assert c.refcount[xkey] == 0
2020-03-21T17:51:18.6323643Z E assert 1 == 0
2020-03-21T17:51:18.6323810Z
2020-03-21T17:51:18.6324087Z distributed\tests\test_client.py:769: AssertionError
2020-03-21T17:51:18.6360931Z ============================== warnings summary ===============================
Traceback:
xref https://github.com/dask/distributed/pull/3611/checks?check_run_id=524283460#step:7:107