def my_worker_splitter(urls):
    'Split urls per worker\n    Selects a subset of urls based on Torch get_worker_info.\n    Used as a shard selection function in Dataset.\n    replaces wds.split_by_worker'
    urls = [url for url in urls]
    assert isinstance(urls, list)
    worker_info = torch.utils.data.get_worker_info()
    if (worker_info is not None):
        wid = worker_info.id
        num_workers = worker_info.num_workers
        return urls[wid::num_workers]
    else:
        return urls
Split urls per worker
Selects a subset of urls based on Torch get_worker_info.
Used as a shard selection function in Dataset.
replaces wds.split_by_worker
def my_worker_splitter(urls):
    'Split urls per worker\n    Selects a subset of urls based on Torch get_worker_info.\n    Used as a shard selection function in Dataset.\n    replaces wds.split_by_worker'
    urls = [url for url in urls]
    assert isinstance(urls, list)
    worker_info = torch.utils.data.get_worker_info()
    if (worker_info is not None):
        wid = worker_info.id
        num_workers = worker_info.num_workers
        return urls[wid::num_workers]
    else:
        return urls