Traceback (most recent call last):


 "/home/kpostlet/.local/share/virtualenvs/message_replay-Dxv_qimG/lib64/python3.6/site-packages/aiohttp/web_protocol.py", line 418, in start

resp = await task


 "/home/kpostlet/.local/share/virtualenvs/message_replay-Dxv_qimG/lib64/python3.6/site-packages/aiohttp/web_app.py", line 458, in _handle

resp = await handler(request)


 "/home/kpostlet/.local/share/virtualenvs/message_replay-Dxv_qimG/lib/python3.6/site-packages/faust/web/drivers/aiohttp.py", line 242, in _dispatch

return await handler(request)


 "/home/kpostlet/.local/share/virtualenvs/message_replay-Dxv_qimG/lib/python3.6/site-packages/faust/web/views.py", line 77, in __call__

return await self.dispatch(request)


 "/home/kpostlet/.local/share/virtualenvs/message_replay-Dxv_qimG/lib/python3.6/site-packages/faust/web/views.py", line 93, in dispatch

response = await method(cast(Request, request), **kwargs)


 "/home/kpostlet/kpostoffice/message_replay/consumer.py", line 46, in seek_offset

copy[(topic.get_topic_name, p)].delta(delta_seconds)


 "/home/kpostlet/.local/share/virtualenvs/message_replay-Dxv_qimG/lib/python3.6/site-packages/faust/tables/wrappers.py", line 237, in delta

return table._windowed_delta(self.key, d, event or self.event)


 "/home/kpostlet/.local/share/virtualenvs/message_replay-Dxv_qimG/lib/python3.6/site-packages/faust/tables/base.py", line 550, in _windowed_delta

window.delta(self._relative_event(event), d)),


 "/home/kpostlet/.local/share/virtualenvs/message_replay-Dxv_qimG/lib/python3.6/site-packages/faust/tables/base.py", line 516, in _relative_event

raise RuntimeError('Operation outside of stream iteration')
RuntimeError: Operation outside of stream iteration

Traceback (most recent call last):


 "/usr/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap

fn(i, *args)


 "/home/jsaric/lib/my-detectron2/detectron2/engine/launch.py", line 84, in _distributed_worker

main_func(*args)


 "/home/jsaric/lib/my-detectron2/tools/train_net.py", line 162, in main

return trainer.train()


 "/home/jsaric/lib/my-detectron2/detectron2/engine/defaults.py", line 373, in train

super().train(self.start_iter, self.max_iter)


 "/home/jsaric/lib/my-detectron2/detectron2/engine/train_loop.py", line 133, in train

self.after_step()


 "/home/jsaric/lib/my-detectron2/detectron2/engine/train_loop.py", line 151, in after_step

h.after_step()


 "/home/jsaric/lib/my-detectron2/detectron2/engine/hooks.py", line 323, in after_step

results = self._func()


 "/home/jsaric/lib/my-detectron2/detectron2/engine/defaults.py", line 324, in test_and_save_results

self._last_eval_results = self.test(self.cfg, self.model)


 "/home/jsaric/lib/my-detectron2/detectron2/engine/defaults.py", line 484, in test

results_i = inference_on_dataset(model, data_loader, evaluator)


 "/home/jsaric/lib/my-detectron2/detectron2/evaluation/evaluator.py", line 121, in inference_on_dataset

outputs = model(inputs)


 "/usr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 541, in __call__

result = self.forward(*input, **kwargs)


 "/usr/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 442, in forward

output = self.module(*inputs[0], **kwargs[0])


 "/usr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 541, in __call__

result = self.forward(*input, **kwargs)


 "/home/jsaric/lib/my-detectron2/detectron2/modeling/meta_arch/rcnn.py", line 109, in forward

return self.inference(batched_inputs)


 "/home/jsaric/lib/my-detectron2/detectron2/modeling/meta_arch/rcnn.py", line 183, in inference

r = detector_postprocess(results_per_image, height, width)


 "/home/jsaric/lib/my-detectron2/detectron2/modeling/postprocessing.py", line 41, in detector_postprocess

results.pred_masks = paste_masks_in_image(


 "/home/jsaric/lib/my-detectron2/detectron2/layers/mask_ops.py", line 117, in paste_masks_in_image

masks_chunk, spatial_inds = _do_paste_mask(


 "/home/jsaric/lib/my-detectron2/detectron2/layers/mask_ops.py", line 57, in _do_paste_mask

grid = torch.stack([gx, gy], dim=3)
RuntimeError: CUDA out of memory. Tried to allocate 2.02 GiB (GPU 0; 10.76 GiB total capacity; 4.19 GiB already allocated; 1.62 GiB free; 4.09 GiB cached)
