We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[BUG-REPORT] Vaex cannot open more than 816 parquet files on s3
**Description**

I try to open many parquet partitions. Vaex opens first 816, then fails with the following error:

> ERROR:MainThread:vaex:error opening 's3://my/path/part-00817-db9aee77-5b44-4571-857b-ea2879609450.c000.snappy.parquet'
> Traceback (most recent call last):
>   File "/home/myuser/p/VS_Code_Projects/SkillExtractor/test_vaex_multiprocessing.py", line 32, in <module>
>     df = vaex.concat([df, vaex.open(parquet_file)])
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/vaex/__init__.py", line 211, in open
>     ds = vaex.dataset.open(path, fs_options=fs_options, *args, **kwargs)
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/vaex/dataset.py", line 42, in open
>     return opener.open(path, *args, **kwargs)
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/vaex/arrow/opener.py", line 29, in open
>     return open_parquet(path, *args, **kwargs)
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/vaex/arrow/dataset.py", line 175, in open_parquet
>     arrow_ds = pyarrow.dataset.dataset(path, filesystem=file_system)
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/pyarrow/dataset.py", line 671, in dataset
>     return _filesystem_dataset(source, **kwargs)
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/pyarrow/dataset.py", line 428, in _filesystem_dataset
>     fs, paths_or_selector = _ensure_single_source(source, filesystem)
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/pyarrow/dataset.py", line 395, in _ensure_single_source
>     file_info = filesystem.get_file_info([path])[0]
>   File "pyarrow/_fs.pyx", line 434, in pyarrow._fs.FileSystem.get_file_info
>   File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
>   File "pyarrow/error.pxi", line 99, in pyarrow.lib.check_status
> OSError: When getting information for key 'demand/positions-20211123/place_country_code=US/part-00817-db9aee77-5b44-4571-857b-ea2879609450.c000.snappy.parquet' in bucket 'tn-datafeed-dev': AWS Error [code 99]: curlCode: 6, Couldn't resolve host name

Tried to open in two different ways, in both cases the same error.

Way 1:

`df = vaex.open("s3://some/path/*.parquet")`

Way 2:

```
s3 = boto3.resource('s3')
my_bucket = s3.Bucket('my_bucket_name')

parquet_files = []
for my_bucket_object in my_bucket.objects.filter(Prefix='some/prefix'):
    if my_bucket_object.key.endswith(".parquet"):
        parquet_files.append("s3://" + my_bucket._name + "/" + my_bucket_object.key)

df = None
for parquet_file in parquet_files:
    if not df:
        df = vaex.open(parquet_file)
    else:
        try:
            df = vaex.concat([df, vaex.open(parquet_file)])
        except Exception as e:
            print(f"WARNING: The file {parquet_file} could not be read!")
            print(e)
```

Everything is fine until it reaches 816 files open. I tried to start with parquet # 817 and it works until another certain number of files is open.

**Software information**
 - Vaex version (`import vaex; vaex.__version__)`: {'vaex-core': '4.0.0a5'}
 - Vaex was installed via: pip
 - OS: AWS Linux v2




--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `packages/vaex-core/vaex/file/s3arrow.py`
Content:
```
1 import os
2 import io
3 from .s3 import patch_profile
4 
5 import pyarrow as pa
6 import pyarrow.fs
7 
8 from . import split_options, FileProxy, split_scheme
9 from .cache import FileSystemHandlerCached
10 
11 
12 def glob(path, fs_options={}):
13     from .s3 import glob
14     return glob(path, fs_options)
15 
16 
17 def parse(path, fs_options, for_arrow=False):
18     path, fs_options = split_options(path, fs_options)
19     path = path.replace('arrow+s3://', 's3://')
20     scheme, _ = split_scheme(path)
21     assert scheme == 's3'
22     # anon is for backwards compatibility
23     fs_options['anonymous'] = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])
24     fs_options = patch_profile(fs_options)
25     use_cache = fs_options.pop('cache', 'true') in [True, 'true', 'True', '1']
26     if 'region' not in fs_options:
27         # we use this to get the default region
28         file_system, path = pa.fs.FileSystem.from_uri(path)
29         # Remove this line for testing purposes to fake not having s3 support
30         # raise pyarrow.lib.ArrowNotImplementedError('FOR TESTING')
31         fs_options['region'] = file_system.region
32     fs = pa.fs.S3FileSystem(**fs_options)
33     if use_cache:
34         fs = FileSystemHandlerCached(fs, scheme='s3', for_arrow=for_arrow)
35         if for_arrow:
36             fs = pyarrow.fs.PyFileSystem(fs)
37     return fs, path
38
```

Path: `packages/vaex-core/vaex/file/s3fs.py`
Content:
```
1 import vaex.utils
2 import pyarrow.fs
3 s3fs = vaex.utils.optional_import('s3fs')
4 
5 import vaex.file.cache
6 from . import split_options, split_scheme
7 from .cache import FileSystemHandlerCached
8 from .s3 import patch_profile
9 
10 
11 def translate_options(fs_options):
12     # translate options of arrow to s3fs
13     fs_options = fs_options.copy()
14     not_supported = {
15         'role_arn', 'session_name', 'external_id', 'load_frequency', 'scheme', 'background_writes', 'profile', 'profile_name'
16     }
17     for key in not_supported:
18         if key in fs_options:
19              warnings.warn(f'The option {key} is not supported using s3fs instead of arrow, so it will be ignored')
20              fs_options.pop(key)
21 
22     # top level
23     mapping = {
24         'anonymous': 'anon',
25     }
26     for key in list(fs_options):
27         if key in mapping:
28             fs_options[mapping[key]] = fs_options.pop(key)
29 
30     # client kwargs
31     mapping = {
32         'access_key': 'aws_access_key_id',
33         'secret_key': 'aws_secret_access_key',
34         'session_token': 'aws_session_token',
35         'region': 'region_name',
36         'endpoint_override': 'endpoint_url',
37     }
38     fs_options['client_kwargs'] = fs_options.get('client_kwargs', {})
39     for key in list(fs_options):
40         if key in mapping:
41             fs_options['client_kwargs'][mapping[key]] = fs_options.pop(key)
42     return fs_options
43 
44 
45 def glob(path, fs_options={}):
46     if '?' in path:
47         __, query = path[:path.index('?')], path[path.index('?'):]
48     else:
49         query = ''
50     scheme, _ = split_scheme(path)
51     path = path.replace('s3fs://', 's3://')
52     path, fs_options = split_options(path, fs_options)
53     # anon is for backwards compatibility
54     if 'cache' in fs_options:
55         del fs_options['cache']
56     # standardize value, and make bool
57     anon = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])
58     fs_options = patch_profile(fs_options)
59     fs_options = translate_options(fs_options)
60     s3 = s3fs.S3FileSystem(anon=anon, **fs_options)
61     return [f'{scheme}://' + k + query for k in s3.glob(path)]
62 
63 
64 def parse(path, fs_options, for_arrow=False):
65     path = path.replace('fsspec+s3://', 's3://')
66     path, fs_options = split_options(path, fs_options)
67     scheme, path = split_scheme(path)
68     use_cache = fs_options.pop('cache', 'true') in [True, 'true', 'True', '1']
69     # standardize value, and make bool
70     anon = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])
71     fs_options = patch_profile(fs_options)
72     fs_options = translate_options(fs_options)
73     s3 = s3fs.S3FileSystem(anon=anon, default_fill_cache=False, **fs_options)
74     fs = pyarrow.fs.FSSpecHandler(s3)
75     if use_cache:
76         fs = FileSystemHandlerCached(fs, scheme='s3', for_arrow=for_arrow)
77     if for_arrow:
78         fs = pyarrow.fs.PyFileSystem(fs)
79     return fs, path
80
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/packages/vaex-core/vaex/file/s3arrow.py b/packages/vaex-core/vaex/file/s3arrow.py
--- a/packages/vaex-core/vaex/file/s3arrow.py
+++ b/packages/vaex-core/vaex/file/s3arrow.py
@@ -7,6 +7,11 @@
 
 from . import split_options, FileProxy, split_scheme
 from .cache import FileSystemHandlerCached
+from ..cache import fingerprint
+
+
+region_cache = {}
+fs_arrow_cache = {}
 
 
 def glob(path, fs_options={}):
@@ -15,21 +20,36 @@
 
 
 def parse(path, fs_options, for_arrow=False):
+    # Remove this line for testing purposes to fake not having s3 support
+    # raise pyarrow.lib.ArrowNotImplementedError('FOR TESTING')
     path, fs_options = split_options(path, fs_options)
     path = path.replace('arrow+s3://', 's3://')
-    scheme, _ = split_scheme(path)
+    fullpath = path
+    scheme, path = split_scheme(path)
     assert scheme == 's3'
+
     # anon is for backwards compatibility
     fs_options['anonymous'] = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])
     fs_options = patch_profile(fs_options)
     use_cache = fs_options.pop('cache', 'true') in [True, 'true', 'True', '1']
+    bucket = path.split('/')[0]
     if 'region' not in fs_options:
-        # we use this to get the default region
-        file_system, path = pa.fs.FileSystem.from_uri(path)
-        # Remove this line for testing purposes to fake not having s3 support
-        # raise pyarrow.lib.ArrowNotImplementedError('FOR TESTING')
-        fs_options['region'] = file_system.region
-    fs = pa.fs.S3FileSystem(**fs_options)
+        # cache region
+        if bucket not in region_cache:
+            # we use this to get the default region
+            file_system, _ = pa.fs.FileSystem.from_uri(fullpath)
+            region = file_system.region
+            region_cache[bucket] = region
+        else:
+            region = region_cache[bucket]
+        fs_options['region'] = region
+    # bucket and options make up a unique key
+    key = fingerprint(bucket, fs_options)
+    if key not in fs_arrow_cache:
+        fs = pa.fs.S3FileSystem(**fs_options)
+        fs_arrow_cache[key] = fs
+    else:
+        fs = fs_arrow_cache[key]
     if use_cache:
         fs = FileSystemHandlerCached(fs, scheme='s3', for_arrow=for_arrow)
         if for_arrow:
diff --git a/packages/vaex-core/vaex/file/s3fs.py b/packages/vaex-core/vaex/file/s3fs.py
--- a/packages/vaex-core/vaex/file/s3fs.py
+++ b/packages/vaex-core/vaex/file/s3fs.py
@@ -6,6 +6,10 @@
 from . import split_options, split_scheme
 from .cache import FileSystemHandlerCached
 from .s3 import patch_profile
+from ..cache import fingerprint
+
+
+fs_cache = {}
 
 
 def translate_options(fs_options):
@@ -70,7 +74,14 @@
     anon = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])
     fs_options = patch_profile(fs_options)
     fs_options = translate_options(fs_options)
-    s3 = s3fs.S3FileSystem(anon=anon, default_fill_cache=False, **fs_options)
+
+    bucket = path.split('/')[0]
+    key = fingerprint(bucket, fs_options)
+    if key not in fs_cache:
+        s3 = s3fs.S3FileSystem(anon=anon, default_fill_cache=False, **fs_options)
+        fs_cache[key] = s3
+    else:
+        s3 = fs_cache[key]
     fs = pyarrow.fs.FSSpecHandler(s3)
     if use_cache:
         fs = FileSystemHandlerCached(fs, scheme='s3', for_arrow=for_arrow)

{"golden_diff": "diff --git a/packages/vaex-core/vaex/file/s3arrow.py b/packages/vaex-core/vaex/file/s3arrow.py\n--- a/packages/vaex-core/vaex/file/s3arrow.py\n+++ b/packages/vaex-core/vaex/file/s3arrow.py\n@@ -7,6 +7,11 @@\n \n from . import split_options, FileProxy, split_scheme\n from .cache import FileSystemHandlerCached\n+from ..cache import fingerprint\n+\n+\n+region_cache = {}\n+fs_arrow_cache = {}\n \n \n def glob(path, fs_options={}):\n@@ -15,21 +20,36 @@\n \n \n def parse(path, fs_options, for_arrow=False):\n+    # Remove this line for testing purposes to fake not having s3 support\n+    # raise pyarrow.lib.ArrowNotImplementedError('FOR TESTING')\n     path, fs_options = split_options(path, fs_options)\n     path = path.replace('arrow+s3://', 's3://')\n-    scheme, _ = split_scheme(path)\n+    fullpath = path\n+    scheme, path = split_scheme(path)\n     assert scheme == 's3'\n+\n     # anon is for backwards compatibility\n     fs_options['anonymous'] = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])\n     fs_options = patch_profile(fs_options)\n     use_cache = fs_options.pop('cache', 'true') in [True, 'true', 'True', '1']\n+    bucket = path.split('/')[0]\n     if 'region' not in fs_options:\n-        # we use this to get the default region\n-        file_system, path = pa.fs.FileSystem.from_uri(path)\n-        # Remove this line for testing purposes to fake not having s3 support\n-        # raise pyarrow.lib.ArrowNotImplementedError('FOR TESTING')\n-        fs_options['region'] = file_system.region\n-    fs = pa.fs.S3FileSystem(**fs_options)\n+        # cache region\n+        if bucket not in region_cache:\n+            # we use this to get the default region\n+            file_system, _ = pa.fs.FileSystem.from_uri(fullpath)\n+            region = file_system.region\n+            region_cache[bucket] = region\n+        else:\n+            region = region_cache[bucket]\n+        fs_options['region'] = region\n+    # bucket and options make up a unique key\n+    key = fingerprint(bucket, fs_options)\n+    if key not in fs_arrow_cache:\n+        fs = pa.fs.S3FileSystem(**fs_options)\n+        fs_arrow_cache[key] = fs\n+    else:\n+        fs = fs_arrow_cache[key]\n     if use_cache:\n         fs = FileSystemHandlerCached(fs, scheme='s3', for_arrow=for_arrow)\n         if for_arrow:\ndiff --git a/packages/vaex-core/vaex/file/s3fs.py b/packages/vaex-core/vaex/file/s3fs.py\n--- a/packages/vaex-core/vaex/file/s3fs.py\n+++ b/packages/vaex-core/vaex/file/s3fs.py\n@@ -6,6 +6,10 @@\n from . import split_options, split_scheme\n from .cache import FileSystemHandlerCached\n from .s3 import patch_profile\n+from ..cache import fingerprint\n+\n+\n+fs_cache = {}\n \n \n def translate_options(fs_options):\n@@ -70,7 +74,14 @@\n     anon = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])\n     fs_options = patch_profile(fs_options)\n     fs_options = translate_options(fs_options)\n-    s3 = s3fs.S3FileSystem(anon=anon, default_fill_cache=False, **fs_options)\n+\n+    bucket = path.split('/')[0]\n+    key = fingerprint(bucket, fs_options)\n+    if key not in fs_cache:\n+        s3 = s3fs.S3FileSystem(anon=anon, default_fill_cache=False, **fs_options)\n+        fs_cache[key] = s3\n+    else:\n+        s3 = fs_cache[key]\n     fs = pyarrow.fs.FSSpecHandler(s3)\n     if use_cache:\n         fs = FileSystemHandlerCached(fs, scheme='s3', for_arrow=for_arrow)\n", "issue": "[BUG-REPORT] Vaex cannot open more than 816 parquet files on s3\n**Description**\r\n\r\nI try to open many parquet partitions. Vaex opens first 816, then fails with the following error:\r\n\r\n> ERROR:MainThread:vaex:error opening 's3://my/path/part-00817-db9aee77-5b44-4571-857b-ea2879609450.c000.snappy.parquet'\r\n> Traceback (most recent call last):\r\n>   File \"/home/myuser/p/VS_Code_Projects/SkillExtractor/test_vaex_multiprocessing.py\", line 32, in <module>\r\n>     df = vaex.concat([df, vaex.open(parquet_file)])\r\n>   File \"/home/myuser/anaconda3/lib/python3.8/site-packages/vaex/__init__.py\", line 211, in open\r\n>     ds = vaex.dataset.open(path, fs_options=fs_options, *args, **kwargs)\r\n>   File \"/home/myuser/anaconda3/lib/python3.8/site-packages/vaex/dataset.py\", line 42, in open\r\n>     return opener.open(path, *args, **kwargs)\r\n>   File \"/home/myuser/anaconda3/lib/python3.8/site-packages/vaex/arrow/opener.py\", line 29, in open\r\n>     return open_parquet(path, *args, **kwargs)\r\n>   File \"/home/myuser/anaconda3/lib/python3.8/site-packages/vaex/arrow/dataset.py\", line 175, in open_parquet\r\n>     arrow_ds = pyarrow.dataset.dataset(path, filesystem=file_system)\r\n>   File \"/home/myuser/anaconda3/lib/python3.8/site-packages/pyarrow/dataset.py\", line 671, in dataset\r\n>     return _filesystem_dataset(source, **kwargs)\r\n>   File \"/home/myuser/anaconda3/lib/python3.8/site-packages/pyarrow/dataset.py\", line 428, in _filesystem_dataset\r\n>     fs, paths_or_selector = _ensure_single_source(source, filesystem)\r\n>   File \"/home/myuser/anaconda3/lib/python3.8/site-packages/pyarrow/dataset.py\", line 395, in _ensure_single_source\r\n>     file_info = filesystem.get_file_info([path])[0]\r\n>   File \"pyarrow/_fs.pyx\", line 434, in pyarrow._fs.FileSystem.get_file_info\r\n>   File \"pyarrow/error.pxi\", line 122, in pyarrow.lib.pyarrow_internal_check_status\r\n>   File \"pyarrow/error.pxi\", line 99, in pyarrow.lib.check_status\r\n> OSError: When getting information for key 'demand/positions-20211123/place_country_code=US/part-00817-db9aee77-5b44-4571-857b-ea2879609450.c000.snappy.parquet' in bucket 'tn-datafeed-dev': AWS Error [code 99]: curlCode: 6, Couldn't resolve host name\r\n\r\nTried to open in two different ways, in both cases the same error.\r\n\r\nWay 1:\r\n\r\n`df = vaex.open(\"s3://some/path/*.parquet\")`\r\n\r\nWay 2:\r\n\r\n```\r\ns3 = boto3.resource('s3')\r\nmy_bucket = s3.Bucket('my_bucket_name')\r\n\r\nparquet_files = []\r\nfor my_bucket_object in my_bucket.objects.filter(Prefix='some/prefix'):\r\n    if my_bucket_object.key.endswith(\".parquet\"):\r\n        parquet_files.append(\"s3://\" + my_bucket._name + \"/\" + my_bucket_object.key)\r\n\r\ndf = None\r\nfor parquet_file in parquet_files:\r\n    if not df:\r\n        df = vaex.open(parquet_file)\r\n    else:\r\n        try:\r\n            df = vaex.concat([df, vaex.open(parquet_file)])\r\n        except Exception as e:\r\n            print(f\"WARNING: The file {parquet_file} could not be read!\")\r\n            print(e)\r\n```\r\n\r\nEverything is fine until it reaches 816 files open. I tried to start with parquet # 817 and it works until another certain number of files is open.\r\n\r\n**Software information**\r\n - Vaex version (`import vaex; vaex.__version__)`: {'vaex-core': '4.0.0a5'}\r\n - Vaex was installed via: pip\r\n - OS: AWS Linux v2\r\n\r\n\r\n\n", "before_files": [{"content": "import os\nimport io\nfrom .s3 import patch_profile\n\nimport pyarrow as pa\nimport pyarrow.fs\n\nfrom . import split_options, FileProxy, split_scheme\nfrom .cache import FileSystemHandlerCached\n\n\ndef glob(path, fs_options={}):\n    from .s3 import glob\n    return glob(path, fs_options)\n\n\ndef parse(path, fs_options, for_arrow=False):\n    path, fs_options = split_options(path, fs_options)\n    path = path.replace('arrow+s3://', 's3://')\n    scheme, _ = split_scheme(path)\n    assert scheme == 's3'\n    # anon is for backwards compatibility\n    fs_options['anonymous'] = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])\n    fs_options = patch_profile(fs_options)\n    use_cache = fs_options.pop('cache', 'true') in [True, 'true', 'True', '1']\n    if 'region' not in fs_options:\n        # we use this to get the default region\n        file_system, path = pa.fs.FileSystem.from_uri(path)\n        # Remove this line for testing purposes to fake not having s3 support\n        # raise pyarrow.lib.ArrowNotImplementedError('FOR TESTING')\n        fs_options['region'] = file_system.region\n    fs = pa.fs.S3FileSystem(**fs_options)\n    if use_cache:\n        fs = FileSystemHandlerCached(fs, scheme='s3', for_arrow=for_arrow)\n        if for_arrow:\n            fs = pyarrow.fs.PyFileSystem(fs)\n    return fs, path\n", "path": "packages/vaex-core/vaex/file/s3arrow.py"}, {"content": "import vaex.utils\nimport pyarrow.fs\ns3fs = vaex.utils.optional_import('s3fs')\n\nimport vaex.file.cache\nfrom . import split_options, split_scheme\nfrom .cache import FileSystemHandlerCached\nfrom .s3 import patch_profile\n\n\ndef translate_options(fs_options):\n    # translate options of arrow to s3fs\n    fs_options = fs_options.copy()\n    not_supported = {\n        'role_arn', 'session_name', 'external_id', 'load_frequency', 'scheme', 'background_writes', 'profile', 'profile_name'\n    }\n    for key in not_supported:\n        if key in fs_options:\n             warnings.warn(f'The option {key} is not supported using s3fs instead of arrow, so it will be ignored')\n             fs_options.pop(key)\n\n    # top level\n    mapping = {\n        'anonymous': 'anon',\n    }\n    for key in list(fs_options):\n        if key in mapping:\n            fs_options[mapping[key]] = fs_options.pop(key)\n\n    # client kwargs\n    mapping = {\n        'access_key': 'aws_access_key_id',\n        'secret_key': 'aws_secret_access_key',\n        'session_token': 'aws_session_token',\n        'region': 'region_name',\n        'endpoint_override': 'endpoint_url',\n    }\n    fs_options['client_kwargs'] = fs_options.get('client_kwargs', {})\n    for key in list(fs_options):\n        if key in mapping:\n            fs_options['client_kwargs'][mapping[key]] = fs_options.pop(key)\n    return fs_options\n\n\ndef glob(path, fs_options={}):\n    if '?' in path:\n        __, query = path[:path.index('?')], path[path.index('?'):]\n    else:\n        query = ''\n    scheme, _ = split_scheme(path)\n    path = path.replace('s3fs://', 's3://')\n    path, fs_options = split_options(path, fs_options)\n    # anon is for backwards compatibility\n    if 'cache' in fs_options:\n        del fs_options['cache']\n    # standardize value, and make bool\n    anon = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])\n    fs_options = patch_profile(fs_options)\n    fs_options = translate_options(fs_options)\n    s3 = s3fs.S3FileSystem(anon=anon, **fs_options)\n    return [f'{scheme}://' + k + query for k in s3.glob(path)]\n\n\ndef parse(path, fs_options, for_arrow=False):\n    path = path.replace('fsspec+s3://', 's3://')\n    path, fs_options = split_options(path, fs_options)\n    scheme, path = split_scheme(path)\n    use_cache = fs_options.pop('cache', 'true') in [True, 'true', 'True', '1']\n    # standardize value, and make bool\n    anon = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])\n    fs_options = patch_profile(fs_options)\n    fs_options = translate_options(fs_options)\n    s3 = s3fs.S3FileSystem(anon=anon, default_fill_cache=False, **fs_options)\n    fs = pyarrow.fs.FSSpecHandler(s3)\n    if use_cache:\n        fs = FileSystemHandlerCached(fs, scheme='s3', for_arrow=for_arrow)\n    if for_arrow:\n        fs = pyarrow.fs.PyFileSystem(fs)\n    return fs, path\n", "path": "packages/vaex-core/vaex/file/s3fs.py"}], "after_files": [{"content": "import os\nimport io\nfrom .s3 import patch_profile\n\nimport pyarrow as pa\nimport pyarrow.fs\n\nfrom . import split_options, FileProxy, split_scheme\nfrom .cache import FileSystemHandlerCached\nfrom ..cache import fingerprint\n\n\nregion_cache = {}\nfs_arrow_cache = {}\n\n\ndef glob(path, fs_options={}):\n    from .s3 import glob\n    return glob(path, fs_options)\n\n\ndef parse(path, fs_options, for_arrow=False):\n    # Remove this line for testing purposes to fake not having s3 support\n    # raise pyarrow.lib.ArrowNotImplementedError('FOR TESTING')\n    path, fs_options = split_options(path, fs_options)\n    path = path.replace('arrow+s3://', 's3://')\n    fullpath = path\n    scheme, path = split_scheme(path)\n    assert scheme == 's3'\n\n    # anon is for backwards compatibility\n    fs_options['anonymous'] = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])\n    fs_options = patch_profile(fs_options)\n    use_cache = fs_options.pop('cache', 'true') in [True, 'true', 'True', '1']\n    bucket = path.split('/')[0]\n    if 'region' not in fs_options:\n        # cache region\n        if bucket not in region_cache:\n            # we use this to get the default region\n            file_system, _ = pa.fs.FileSystem.from_uri(fullpath)\n            region = file_system.region\n            region_cache[bucket] = region\n        else:\n            region = region_cache[bucket]\n        fs_options['region'] = region\n    # bucket and options make up a unique key\n    key = fingerprint(bucket, fs_options)\n    if key not in fs_arrow_cache:\n        fs = pa.fs.S3FileSystem(**fs_options)\n        fs_arrow_cache[key] = fs\n    else:\n        fs = fs_arrow_cache[key]\n    if use_cache:\n        fs = FileSystemHandlerCached(fs, scheme='s3', for_arrow=for_arrow)\n        if for_arrow:\n            fs = pyarrow.fs.PyFileSystem(fs)\n    return fs, path\n", "path": "packages/vaex-core/vaex/file/s3arrow.py"}, {"content": "import vaex.utils\nimport pyarrow.fs\ns3fs = vaex.utils.optional_import('s3fs')\n\nimport vaex.file.cache\nfrom . import split_options, split_scheme\nfrom .cache import FileSystemHandlerCached\nfrom .s3 import patch_profile\nfrom ..cache import fingerprint\n\n\nfs_cache = {}\n\n\ndef translate_options(fs_options):\n    # translate options of arrow to s3fs\n    fs_options = fs_options.copy()\n    not_supported = {\n        'role_arn', 'session_name', 'external_id', 'load_frequency', 'scheme', 'background_writes', 'profile', 'profile_name'\n    }\n    for key in not_supported:\n        if key in fs_options:\n             warnings.warn(f'The option {key} is not supported using s3fs instead of arrow, so it will be ignored')\n             fs_options.pop(key)\n\n    # top level\n    mapping = {\n        'anonymous': 'anon',\n    }\n    for key in list(fs_options):\n        if key in mapping:\n            fs_options[mapping[key]] = fs_options.pop(key)\n\n    # client kwargs\n    mapping = {\n        'access_key': 'aws_access_key_id',\n        'secret_key': 'aws_secret_access_key',\n        'session_token': 'aws_session_token',\n        'region': 'region_name',\n        'endpoint_override': 'endpoint_url',\n    }\n    fs_options['client_kwargs'] = fs_options.get('client_kwargs', {})\n    for key in list(fs_options):\n        if key in mapping:\n            fs_options['client_kwargs'][mapping[key]] = fs_options.pop(key)\n    return fs_options\n\n\ndef glob(path, fs_options={}):\n    if '?' in path:\n        __, query = path[:path.index('?')], path[path.index('?'):]\n    else:\n        query = ''\n    scheme, _ = split_scheme(path)\n    path = path.replace('s3fs://', 's3://')\n    path, fs_options = split_options(path, fs_options)\n    # anon is for backwards compatibility\n    if 'cache' in fs_options:\n        del fs_options['cache']\n    # standardize value, and make bool\n    anon = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])\n    fs_options = patch_profile(fs_options)\n    fs_options = translate_options(fs_options)\n    s3 = s3fs.S3FileSystem(anon=anon, **fs_options)\n    return [f'{scheme}://' + k + query for k in s3.glob(path)]\n\n\ndef parse(path, fs_options, for_arrow=False):\n    path = path.replace('fsspec+s3://', 's3://')\n    path, fs_options = split_options(path, fs_options)\n    scheme, path = split_scheme(path)\n    use_cache = fs_options.pop('cache', 'true') in [True, 'true', 'True', '1']\n    # standardize value, and make bool\n    anon = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])\n    fs_options = patch_profile(fs_options)\n    fs_options = translate_options(fs_options)\n\n    bucket = path.split('/')[0]\n    key = fingerprint(bucket, fs_options)\n    if key not in fs_cache:\n        s3 = s3fs.S3FileSystem(anon=anon, default_fill_cache=False, **fs_options)\n        fs_cache[key] = s3\n    else:\n        s3 = fs_cache[key]\n    fs = pyarrow.fs.FSSpecHandler(s3)\n    if use_cache:\n        fs = FileSystemHandlerCached(fs, scheme='s3', for_arrow=for_arrow)\n    if for_arrow:\n        fs = pyarrow.fs.PyFileSystem(fs)\n    return fs, path\n", "path": "packages/vaex-core/vaex/file/s3fs.py"}]}