We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Auto-archive script not running on crawled dataset?
All OSF or Zenodo datasets are not being archived when running the script `auto_archive.py` with option `--all`.

The following information systematically shows:
```
SKIPPED: archive not needed for projects/<dataset_name>
```
 but the dataset size is definitely below 20GB.

List of affected datasets:

- CFMM_7T__MP2RAGE_T1_mapping_version-2020-10-02T12:55:03.086599.tar.gz
- Comparing_Perturbation_Modes_for_Evaluating_Instabilities_in_Neuroimaging__Processed_NKI_RS_Subset__08_2019__version-None.tar.gz
- Intracellular_Recordings_of_Murine_Neocortical_Neurons_version-2020-04-18T19:34:10.458118.tar.gz
- Learning_Naturalistic_Structure__Processed_fMRI_dataset_version-0.1.0.tar.gz
- MRI_and_unbiased_averages_of_wild_muskrats__Ondatra_zibethicus__and_red_squirrels__Tamiasciurus_hudsonicus__version-1.0.tar.gz:
- Multi-model_functionalization_of_disease-associated_PTEN_missense_mutations_version-1.0.tar.gz
- Quantifying_Neural_Cognitive_Relationships_Across_the_Brain_version-2020-11-11T02:30:59.429493.tar.gz
- Synthetic_Animated_Mouse__SAM___University_of_British_Columbia__Datasets_and_3D_models_version-2021-04-09T14:12:32.548954.tar.gz:
- Systematic_phenomics_analysis_of_autism-associated_genes_version-3.2.tar.gz
- VFA_T1_mapping___RTHawk__open__vs_Siemens__commercial__version-v1.0.tar.gz



Auto-archive script not running on crawled dataset?
All OSF or Zenodo datasets are not being archived when running the script `auto_archive.py` with option `--all`.

The following information systematically shows:
```
SKIPPED: archive not needed for projects/<dataset_name>
```
 but the dataset size is definitely below 20GB.

List of affected datasets:

- CFMM_7T__MP2RAGE_T1_mapping_version-2020-10-02T12:55:03.086599.tar.gz
- Comparing_Perturbation_Modes_for_Evaluating_Instabilities_in_Neuroimaging__Processed_NKI_RS_Subset__08_2019__version-None.tar.gz
- Intracellular_Recordings_of_Murine_Neocortical_Neurons_version-2020-04-18T19:34:10.458118.tar.gz
- Learning_Naturalistic_Structure__Processed_fMRI_dataset_version-0.1.0.tar.gz
- MRI_and_unbiased_averages_of_wild_muskrats__Ondatra_zibethicus__and_red_squirrels__Tamiasciurus_hudsonicus__version-1.0.tar.gz:
- Multi-model_functionalization_of_disease-associated_PTEN_missense_mutations_version-1.0.tar.gz
- Quantifying_Neural_Cognitive_Relationships_Across_the_Brain_version-2020-11-11T02:30:59.429493.tar.gz
- Synthetic_Animated_Mouse__SAM___University_of_British_Columbia__Datasets_and_3D_models_version-2021-04-09T14:12:32.548954.tar.gz:
- Systematic_phenomics_analysis_of_autism-associated_genes_version-3.2.tar.gz
- VFA_T1_mapping___RTHawk__open__vs_Siemens__commercial__version-v1.0.tar.gz




--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/auto_archive.py`
Content:
```
1 from __future__ import annotations
2 
3 import argparse
4 import json
5 import logging
6 import os
7 from datetime import datetime
8 from datetime import timedelta
9 
10 import git
11 import humanfriendly
12 from datalad.plugin import export_archive
13 from github import Github
14 
15 from scripts.datalad_utils import get_dataset
16 from scripts.datalad_utils import install_dataset
17 from scripts.log import get_logger
18 from tests.functions import get_proper_submodules
19 
20 
21 logger = get_logger(
22     "CONP-Archive", filename="conp-archive.log", file_level=logging.DEBUG
23 )
24 
25 
26 class ArchiveFailed(Exception):
27     pass
28 
29 
30 def parse_args():
31     example_text = """Example:
32     PYTHONPATH=$PWD python scripts/auto_archive.py <out_dir>
33     """
34 
35     parser = argparse.ArgumentParser(
36         description="Archiver for the CONP-datasets.",
37         epilog=example_text,
38         formatter_class=argparse.RawDescriptionHelpFormatter,
39     )
40 
41     parser.add_argument(
42         "--out_dir", "-o", type=str, help="Path to store the archived datasets."
43     )
44     parser.add_argument(
45         "--max-size",
46         type=float,
47         default=20.0,
48         help="Maximum size of dataset to archive in GB.",
49     )
50     group = parser.add_mutually_exclusive_group()
51     group.add_argument(
52         "--all",
53         action="store_true",
54         help="Archive all the datasets rather than those modified since the last time.",
55     )
56     group.add_argument(
57         "--dataset",
58         "-d",
59         type=str,
60         nargs="+",
61         help="Restrict the archive to the specified dataset paths.",
62     )
63 
64     return parser.parse_args()
65 
66 
67 def get_all_datasets():
68     return {os.path.basename(submodule.path) for submodule in git.Repo().submodules}
69 
70 
71 def get_modified_datasets(
72     *,
73     since: datetime | None = None,
74     until: datetime | None = None,
75 ) -> set[str]:
76     """Retrieve the modified datasets.
77 
78     Requires to set GITHUB_ACCESS_TOKEN as an environment variable.
79 
80     Parameters
81     ----------
82     since : Optional[datetime], optional
83         Start date from which commits are retrieved, by default date of the previous crawl, if never crawled set to
84         one week ago.
85     until : Optional[datetime], optional
86         Latest date at which commit are retrieved, by default `now`
87 
88     Returns
89     -------
90     set[str]
91         Path of the dataset folders.
92     """
93     now = datetime.now().astimezone()
94 
95     if since is None:
96         if os.path.exists(".conp-archive"):
97             with open(".conp-archive") as fin:
98                 since = datetime.fromisoformat(fin.read())
99         else:
100             since = now - timedelta(weeks=1)
101 
102     if until is None:
103         until = now
104 
105     try:
106         gh_access_token = os.environ.get("GITHUB_ACCESS_TOKEN", None)
107         if gh_access_token is None:
108             raise OSError("GITHUB_ACCESS_TOKEN is not defined.")
109 
110     except OSError as e:
111         # The program is not stopped since GitHub allows 60 query per hours with
112         # authentication. However the program will most likely fail.
113         logger.critical(e)
114 
115     logger.info(f"Retrieving modified datasets since {since}")
116     repo = Github(gh_access_token).get_repo("CONP-PCNO/conp-dataset")
117     commits = repo.get_commits(since=since, until=until)
118 
119     with open(".conp-archive", "w") as fout:
120         fout.write(now.isoformat())
121 
122     modified_datasets: set[str] = {
123         os.path.basename(file_.filename)
124         for commit in commits
125         for file_ in commit.files
126         if file_.filename.startswith("projects/")
127     }
128 
129     return modified_datasets
130 
131 
132 def archive_dataset(dataset_path: str, out_dir: str, version: str) -> None:
133     os.makedirs(os.path.dirname(out_dir), mode=0o755, exist_ok=True)
134     out_filename = f"{out_dir}_version-{version}.tar.gz"
135     logger.info(f"Archiving dataset: {dataset_path} to {out_filename}")
136 
137     cwd = os.getcwd()
138     try:
139         datalad_archiver = export_archive.ExportArchive()
140         dataset_repo = git.Repo(dataset_path)
141 
142         with open(os.path.join(dataset_path, ".git.log"), "w") as fout:
143             fout.write(dataset_repo.git.log(pretty="format:%H %s"))
144 
145         # Export is performed from the dataset root.
146         # This is to avoid failure when a submodule is not downloaded; e.g. for parent
147         # dataset in dataset derivative.
148         os.chdir(os.path.join(cwd, dataset_path))
149         datalad_archiver(".", filename=out_filename)
150 
151     except Exception as e:
152         raise ArchiveFailed(
153             f"FAILURE: could not archive dataset: {dataset_path} to {out_filename}\n{e}"
154         )
155     finally:
156         os.chdir(cwd)
157 
158 
159 if __name__ == "__main__":
160     args = parse_args()
161 
162     # Only archive the datasets available locally.
163     datasets = get_all_datasets()
164     if args.dataset:
165         target_datasets = {os.path.basename(os.path.normpath(d)) for d in args.dataset}
166         logger.warning(
167             f"The following dataset were not found locally: {target_datasets - datasets}"
168         )
169         datasets &= target_datasets
170 
171     elif not args.all:
172         modified_datasets = get_modified_datasets()
173         logger.warning(
174             f"The following dataset were not found locally: {modified_datasets - datasets}"
175         )
176         datasets &= modified_datasets
177 
178     for dataset_name in datasets:
179         dataset = "projects/" + dataset_name
180 
181         try:
182             logger.info(f"Installing dataset: {dataset}")
183             install_dataset(dataset)
184 
185             is_public = False
186             version = ""
187             dataset_size = 0.0
188 
189             with open(os.path.join(dataset, "DATS.json")) as fin:
190                 metadata = json.load(fin)
191 
192                 is_public = metadata.get("privacy") == "open"
193                 version = metadata.get("version")
194 
195                 for distribution in metadata.get("distributions", list()):
196                     dataset_size += humanfriendly.parse_size(
197                         f"{distribution['size']} {distribution['unit']['value']}",
198                     )
199                     dataset_size //= 1024 ** 3  # Convert to GB
200 
201             # Only archive public dataset less than 20GB
202             if dataset_size <= args.max_size and is_public:
203                 logger.info(f"Downloading dataset: {dataset}")
204                 get_dataset(dataset)
205                 for submodule in get_proper_submodules(dataset):
206                     get_dataset(submodule)
207 
208                 archive_dataset(
209                     dataset,
210                     out_dir=os.path.join(args.out_dir, dataset_name),
211                     version=version,
212                 )
213                 logger.info(f"SUCCESS: archive created for {dataset}")
214 
215             else:
216                 logger.info(f"SKIPPED: archive not needed for {dataset}")
217 
218         except Exception as e:
219             # TODO implement notification system.
220             # This will alert when a dataset fails the archiving process.
221             logger.exception(
222                 f"FAILURE: could not archive dataset: {dataset} to {args.out_dir}.tar.gz\n{e}"
223             )
224 
225     logger.info("Done archiving the datasets.")
226
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/auto_archive.py b/scripts/auto_archive.py
--- a/scripts/auto_archive.py
+++ b/scripts/auto_archive.py
@@ -64,8 +64,11 @@
     return parser.parse_args()
 
 
-def get_all_datasets():
-    return {os.path.basename(submodule.path) for submodule in git.Repo().submodules}
+def get_datasets_path():
+    return {
+        os.path.basename(submodule.path): submodule.path
+        for submodule in git.Repo().submodules
+    }
 
 
 def get_modified_datasets(
@@ -129,9 +132,11 @@
     return modified_datasets
 
 
-def archive_dataset(dataset_path: str, out_dir: str, version: str) -> None:
+def archive_dataset(
+    dataset_path: str, out_dir: str, archive_name: str, version: str
+) -> None:
     os.makedirs(os.path.dirname(out_dir), mode=0o755, exist_ok=True)
-    out_filename = f"{out_dir}_version-{version}.tar.gz"
+    out_filename = f"{archive_name}_version-{version}.tar.gz"
     logger.info(f"Archiving dataset: {dataset_path} to {out_filename}")
 
     cwd = os.getcwd()
@@ -160,7 +165,8 @@
     args = parse_args()
 
     # Only archive the datasets available locally.
-    datasets = get_all_datasets()
+    datasets_path = get_datasets_path()
+    datasets = datasets_path.keys()
     if args.dataset:
         target_datasets = {os.path.basename(os.path.normpath(d)) for d in args.dataset}
         logger.warning(
@@ -176,7 +182,7 @@
         datasets &= modified_datasets
 
     for dataset_name in datasets:
-        dataset = "projects/" + dataset_name
+        dataset = datasets_path[dataset_name]
 
         try:
             logger.info(f"Installing dataset: {dataset}")
@@ -189,7 +195,13 @@
             with open(os.path.join(dataset, "DATS.json")) as fin:
                 metadata = json.load(fin)
 
-                is_public = metadata.get("privacy") == "open"
+                is_public = (
+                    metadata.get("distributions", [{}])[0]
+                    .get("access", {})
+                    .get("authorizations", [{}])[0]
+                    .get("value")
+                    == "public"
+                )
                 version = metadata.get("version")
 
                 for distribution in metadata.get("distributions", list()):
@@ -205,9 +217,13 @@
                 for submodule in get_proper_submodules(dataset):
                     get_dataset(submodule)
 
+                archive_name = "__".join(
+                    os.path.relpath(dataset, "projects").split("/")
+                )
                 archive_dataset(
                     dataset,
                     out_dir=os.path.join(args.out_dir, dataset_name),
+                    archive_name=archive_name,
                     version=version,
                 )
                 logger.info(f"SUCCESS: archive created for {dataset}")

{"golden_diff": "diff --git a/scripts/auto_archive.py b/scripts/auto_archive.py\n--- a/scripts/auto_archive.py\n+++ b/scripts/auto_archive.py\n@@ -64,8 +64,11 @@\n     return parser.parse_args()\n \n \n-def get_all_datasets():\n-    return {os.path.basename(submodule.path) for submodule in git.Repo().submodules}\n+def get_datasets_path():\n+    return {\n+        os.path.basename(submodule.path): submodule.path\n+        for submodule in git.Repo().submodules\n+    }\n \n \n def get_modified_datasets(\n@@ -129,9 +132,11 @@\n     return modified_datasets\n \n \n-def archive_dataset(dataset_path: str, out_dir: str, version: str) -> None:\n+def archive_dataset(\n+    dataset_path: str, out_dir: str, archive_name: str, version: str\n+) -> None:\n     os.makedirs(os.path.dirname(out_dir), mode=0o755, exist_ok=True)\n-    out_filename = f\"{out_dir}_version-{version}.tar.gz\"\n+    out_filename = f\"{archive_name}_version-{version}.tar.gz\"\n     logger.info(f\"Archiving dataset: {dataset_path} to {out_filename}\")\n \n     cwd = os.getcwd()\n@@ -160,7 +165,8 @@\n     args = parse_args()\n \n     # Only archive the datasets available locally.\n-    datasets = get_all_datasets()\n+    datasets_path = get_datasets_path()\n+    datasets = datasets_path.keys()\n     if args.dataset:\n         target_datasets = {os.path.basename(os.path.normpath(d)) for d in args.dataset}\n         logger.warning(\n@@ -176,7 +182,7 @@\n         datasets &= modified_datasets\n \n     for dataset_name in datasets:\n-        dataset = \"projects/\" + dataset_name\n+        dataset = datasets_path[dataset_name]\n \n         try:\n             logger.info(f\"Installing dataset: {dataset}\")\n@@ -189,7 +195,13 @@\n             with open(os.path.join(dataset, \"DATS.json\")) as fin:\n                 metadata = json.load(fin)\n \n-                is_public = metadata.get(\"privacy\") == \"open\"\n+                is_public = (\n+                    metadata.get(\"distributions\", [{}])[0]\n+                    .get(\"access\", {})\n+                    .get(\"authorizations\", [{}])[0]\n+                    .get(\"value\")\n+                    == \"public\"\n+                )\n                 version = metadata.get(\"version\")\n \n                 for distribution in metadata.get(\"distributions\", list()):\n@@ -205,9 +217,13 @@\n                 for submodule in get_proper_submodules(dataset):\n                     get_dataset(submodule)\n \n+                archive_name = \"__\".join(\n+                    os.path.relpath(dataset, \"projects\").split(\"/\")\n+                )\n                 archive_dataset(\n                     dataset,\n                     out_dir=os.path.join(args.out_dir, dataset_name),\n+                    archive_name=archive_name,\n                     version=version,\n                 )\n                 logger.info(f\"SUCCESS: archive created for {dataset}\")\n", "issue": "Auto-archive script not running on crawled dataset?\nAll OSF or Zenodo datasets are not being archived when running the script `auto_archive.py` with option `--all`.\r\n\r\nThe following information systematically shows:\r\n```\r\nSKIPPED: archive not needed for projects/<dataset_name>\r\n```\r\n but the dataset size is definitely below 20GB.\r\n\r\nList of affected datasets:\r\n\r\n- CFMM_7T__MP2RAGE_T1_mapping_version-2020-10-02T12:55:03.086599.tar.gz\r\n- Comparing_Perturbation_Modes_for_Evaluating_Instabilities_in_Neuroimaging__Processed_NKI_RS_Subset__08_2019__version-None.tar.gz\r\n- Intracellular_Recordings_of_Murine_Neocortical_Neurons_version-2020-04-18T19:34:10.458118.tar.gz\r\n- Learning_Naturalistic_Structure__Processed_fMRI_dataset_version-0.1.0.tar.gz\r\n- MRI_and_unbiased_averages_of_wild_muskrats__Ondatra_zibethicus__and_red_squirrels__Tamiasciurus_hudsonicus__version-1.0.tar.gz:\r\n- Multi-model_functionalization_of_disease-associated_PTEN_missense_mutations_version-1.0.tar.gz\r\n- Quantifying_Neural_Cognitive_Relationships_Across_the_Brain_version-2020-11-11T02:30:59.429493.tar.gz\r\n- Synthetic_Animated_Mouse__SAM___University_of_British_Columbia__Datasets_and_3D_models_version-2021-04-09T14:12:32.548954.tar.gz:\r\n- Systematic_phenomics_analysis_of_autism-associated_genes_version-3.2.tar.gz\r\n- VFA_T1_mapping___RTHawk__open__vs_Siemens__commercial__version-v1.0.tar.gz\r\n\r\n\r\n\nAuto-archive script not running on crawled dataset?\nAll OSF or Zenodo datasets are not being archived when running the script `auto_archive.py` with option `--all`.\r\n\r\nThe following information systematically shows:\r\n```\r\nSKIPPED: archive not needed for projects/<dataset_name>\r\n```\r\n but the dataset size is definitely below 20GB.\r\n\r\nList of affected datasets:\r\n\r\n- CFMM_7T__MP2RAGE_T1_mapping_version-2020-10-02T12:55:03.086599.tar.gz\r\n- Comparing_Perturbation_Modes_for_Evaluating_Instabilities_in_Neuroimaging__Processed_NKI_RS_Subset__08_2019__version-None.tar.gz\r\n- Intracellular_Recordings_of_Murine_Neocortical_Neurons_version-2020-04-18T19:34:10.458118.tar.gz\r\n- Learning_Naturalistic_Structure__Processed_fMRI_dataset_version-0.1.0.tar.gz\r\n- MRI_and_unbiased_averages_of_wild_muskrats__Ondatra_zibethicus__and_red_squirrels__Tamiasciurus_hudsonicus__version-1.0.tar.gz:\r\n- Multi-model_functionalization_of_disease-associated_PTEN_missense_mutations_version-1.0.tar.gz\r\n- Quantifying_Neural_Cognitive_Relationships_Across_the_Brain_version-2020-11-11T02:30:59.429493.tar.gz\r\n- Synthetic_Animated_Mouse__SAM___University_of_British_Columbia__Datasets_and_3D_models_version-2021-04-09T14:12:32.548954.tar.gz:\r\n- Systematic_phenomics_analysis_of_autism-associated_genes_version-3.2.tar.gz\r\n- VFA_T1_mapping___RTHawk__open__vs_Siemens__commercial__version-v1.0.tar.gz\r\n\r\n\r\n\n", "before_files": [{"content": "from __future__ import annotations\n\nimport argparse\nimport json\nimport logging\nimport os\nfrom datetime import datetime\nfrom datetime import timedelta\n\nimport git\nimport humanfriendly\nfrom datalad.plugin import export_archive\nfrom github import Github\n\nfrom scripts.datalad_utils import get_dataset\nfrom scripts.datalad_utils import install_dataset\nfrom scripts.log import get_logger\nfrom tests.functions import get_proper_submodules\n\n\nlogger = get_logger(\n    \"CONP-Archive\", filename=\"conp-archive.log\", file_level=logging.DEBUG\n)\n\n\nclass ArchiveFailed(Exception):\n    pass\n\n\ndef parse_args():\n    example_text = \"\"\"Example:\n    PYTHONPATH=$PWD python scripts/auto_archive.py <out_dir>\n    \"\"\"\n\n    parser = argparse.ArgumentParser(\n        description=\"Archiver for the CONP-datasets.\",\n        epilog=example_text,\n        formatter_class=argparse.RawDescriptionHelpFormatter,\n    )\n\n    parser.add_argument(\n        \"--out_dir\", \"-o\", type=str, help=\"Path to store the archived datasets.\"\n    )\n    parser.add_argument(\n        \"--max-size\",\n        type=float,\n        default=20.0,\n        help=\"Maximum size of dataset to archive in GB.\",\n    )\n    group = parser.add_mutually_exclusive_group()\n    group.add_argument(\n        \"--all\",\n        action=\"store_true\",\n        help=\"Archive all the datasets rather than those modified since the last time.\",\n    )\n    group.add_argument(\n        \"--dataset\",\n        \"-d\",\n        type=str,\n        nargs=\"+\",\n        help=\"Restrict the archive to the specified dataset paths.\",\n    )\n\n    return parser.parse_args()\n\n\ndef get_all_datasets():\n    return {os.path.basename(submodule.path) for submodule in git.Repo().submodules}\n\n\ndef get_modified_datasets(\n    *,\n    since: datetime | None = None,\n    until: datetime | None = None,\n) -> set[str]:\n    \"\"\"Retrieve the modified datasets.\n\n    Requires to set GITHUB_ACCESS_TOKEN as an environment variable.\n\n    Parameters\n    ----------\n    since : Optional[datetime], optional\n        Start date from which commits are retrieved, by default date of the previous crawl, if never crawled set to\n        one week ago.\n    until : Optional[datetime], optional\n        Latest date at which commit are retrieved, by default `now`\n\n    Returns\n    -------\n    set[str]\n        Path of the dataset folders.\n    \"\"\"\n    now = datetime.now().astimezone()\n\n    if since is None:\n        if os.path.exists(\".conp-archive\"):\n            with open(\".conp-archive\") as fin:\n                since = datetime.fromisoformat(fin.read())\n        else:\n            since = now - timedelta(weeks=1)\n\n    if until is None:\n        until = now\n\n    try:\n        gh_access_token = os.environ.get(\"GITHUB_ACCESS_TOKEN\", None)\n        if gh_access_token is None:\n            raise OSError(\"GITHUB_ACCESS_TOKEN is not defined.\")\n\n    except OSError as e:\n        # The program is not stopped since GitHub allows 60 query per hours with\n        # authentication. However the program will most likely fail.\n        logger.critical(e)\n\n    logger.info(f\"Retrieving modified datasets since {since}\")\n    repo = Github(gh_access_token).get_repo(\"CONP-PCNO/conp-dataset\")\n    commits = repo.get_commits(since=since, until=until)\n\n    with open(\".conp-archive\", \"w\") as fout:\n        fout.write(now.isoformat())\n\n    modified_datasets: set[str] = {\n        os.path.basename(file_.filename)\n        for commit in commits\n        for file_ in commit.files\n        if file_.filename.startswith(\"projects/\")\n    }\n\n    return modified_datasets\n\n\ndef archive_dataset(dataset_path: str, out_dir: str, version: str) -> None:\n    os.makedirs(os.path.dirname(out_dir), mode=0o755, exist_ok=True)\n    out_filename = f\"{out_dir}_version-{version}.tar.gz\"\n    logger.info(f\"Archiving dataset: {dataset_path} to {out_filename}\")\n\n    cwd = os.getcwd()\n    try:\n        datalad_archiver = export_archive.ExportArchive()\n        dataset_repo = git.Repo(dataset_path)\n\n        with open(os.path.join(dataset_path, \".git.log\"), \"w\") as fout:\n            fout.write(dataset_repo.git.log(pretty=\"format:%H %s\"))\n\n        # Export is performed from the dataset root.\n        # This is to avoid failure when a submodule is not downloaded; e.g. for parent\n        # dataset in dataset derivative.\n        os.chdir(os.path.join(cwd, dataset_path))\n        datalad_archiver(\".\", filename=out_filename)\n\n    except Exception as e:\n        raise ArchiveFailed(\n            f\"FAILURE: could not archive dataset: {dataset_path} to {out_filename}\\n{e}\"\n        )\n    finally:\n        os.chdir(cwd)\n\n\nif __name__ == \"__main__\":\n    args = parse_args()\n\n    # Only archive the datasets available locally.\n    datasets = get_all_datasets()\n    if args.dataset:\n        target_datasets = {os.path.basename(os.path.normpath(d)) for d in args.dataset}\n        logger.warning(\n            f\"The following dataset were not found locally: {target_datasets - datasets}\"\n        )\n        datasets &= target_datasets\n\n    elif not args.all:\n        modified_datasets = get_modified_datasets()\n        logger.warning(\n            f\"The following dataset were not found locally: {modified_datasets - datasets}\"\n        )\n        datasets &= modified_datasets\n\n    for dataset_name in datasets:\n        dataset = \"projects/\" + dataset_name\n\n        try:\n            logger.info(f\"Installing dataset: {dataset}\")\n            install_dataset(dataset)\n\n            is_public = False\n            version = \"\"\n            dataset_size = 0.0\n\n            with open(os.path.join(dataset, \"DATS.json\")) as fin:\n                metadata = json.load(fin)\n\n                is_public = metadata.get(\"privacy\") == \"open\"\n                version = metadata.get(\"version\")\n\n                for distribution in metadata.get(\"distributions\", list()):\n                    dataset_size += humanfriendly.parse_size(\n                        f\"{distribution['size']} {distribution['unit']['value']}\",\n                    )\n                    dataset_size //= 1024 ** 3  # Convert to GB\n\n            # Only archive public dataset less than 20GB\n            if dataset_size <= args.max_size and is_public:\n                logger.info(f\"Downloading dataset: {dataset}\")\n                get_dataset(dataset)\n                for submodule in get_proper_submodules(dataset):\n                    get_dataset(submodule)\n\n                archive_dataset(\n                    dataset,\n                    out_dir=os.path.join(args.out_dir, dataset_name),\n                    version=version,\n                )\n                logger.info(f\"SUCCESS: archive created for {dataset}\")\n\n            else:\n                logger.info(f\"SKIPPED: archive not needed for {dataset}\")\n\n        except Exception as e:\n            # TODO implement notification system.\n            # This will alert when a dataset fails the archiving process.\n            logger.exception(\n                f\"FAILURE: could not archive dataset: {dataset} to {args.out_dir}.tar.gz\\n{e}\"\n            )\n\n    logger.info(\"Done archiving the datasets.\")\n", "path": "scripts/auto_archive.py"}], "after_files": [{"content": "from __future__ import annotations\n\nimport argparse\nimport json\nimport logging\nimport os\nfrom datetime import datetime\nfrom datetime import timedelta\n\nimport git\nimport humanfriendly\nfrom datalad.plugin import export_archive\nfrom github import Github\n\nfrom scripts.datalad_utils import get_dataset\nfrom scripts.datalad_utils import install_dataset\nfrom scripts.log import get_logger\nfrom tests.functions import get_proper_submodules\n\n\nlogger = get_logger(\n    \"CONP-Archive\", filename=\"conp-archive.log\", file_level=logging.DEBUG\n)\n\n\nclass ArchiveFailed(Exception):\n    pass\n\n\ndef parse_args():\n    example_text = \"\"\"Example:\n    PYTHONPATH=$PWD python scripts/auto_archive.py <out_dir>\n    \"\"\"\n\n    parser = argparse.ArgumentParser(\n        description=\"Archiver for the CONP-datasets.\",\n        epilog=example_text,\n        formatter_class=argparse.RawDescriptionHelpFormatter,\n    )\n\n    parser.add_argument(\n        \"--out_dir\", \"-o\", type=str, help=\"Path to store the archived datasets.\"\n    )\n    parser.add_argument(\n        \"--max-size\",\n        type=float,\n        default=20.0,\n        help=\"Maximum size of dataset to archive in GB.\",\n    )\n    group = parser.add_mutually_exclusive_group()\n    group.add_argument(\n        \"--all\",\n        action=\"store_true\",\n        help=\"Archive all the datasets rather than those modified since the last time.\",\n    )\n    group.add_argument(\n        \"--dataset\",\n        \"-d\",\n        type=str,\n        nargs=\"+\",\n        help=\"Restrict the archive to the specified dataset paths.\",\n    )\n\n    return parser.parse_args()\n\n\ndef get_datasets_path():\n    return {\n        os.path.basename(submodule.path): submodule.path\n        for submodule in git.Repo().submodules\n    }\n\n\ndef get_modified_datasets(\n    *,\n    since: datetime | None = None,\n    until: datetime | None = None,\n) -> set[str]:\n    \"\"\"Retrieve the modified datasets.\n\n    Requires to set GITHUB_ACCESS_TOKEN as an environment variable.\n\n    Parameters\n    ----------\n    since : Optional[datetime], optional\n        Start date from which commits are retrieved, by default date of the previous crawl, if never crawled set to\n        one week ago.\n    until : Optional[datetime], optional\n        Latest date at which commit are retrieved, by default `now`\n\n    Returns\n    -------\n    set[str]\n        Path of the dataset folders.\n    \"\"\"\n    now = datetime.now().astimezone()\n\n    if since is None:\n        if os.path.exists(\".conp-archive\"):\n            with open(\".conp-archive\") as fin:\n                since = datetime.fromisoformat(fin.read())\n        else:\n            since = now - timedelta(weeks=1)\n\n    if until is None:\n        until = now\n\n    try:\n        gh_access_token = os.environ.get(\"GITHUB_ACCESS_TOKEN\", None)\n        if gh_access_token is None:\n            raise OSError(\"GITHUB_ACCESS_TOKEN is not defined.\")\n\n    except OSError as e:\n        # The program is not stopped since GitHub allows 60 query per hours with\n        # authentication. However the program will most likely fail.\n        logger.critical(e)\n\n    logger.info(f\"Retrieving modified datasets since {since}\")\n    repo = Github(gh_access_token).get_repo(\"CONP-PCNO/conp-dataset\")\n    commits = repo.get_commits(since=since, until=until)\n\n    with open(\".conp-archive\", \"w\") as fout:\n        fout.write(now.isoformat())\n\n    modified_datasets: set[str] = {\n        os.path.basename(file_.filename)\n        for commit in commits\n        for file_ in commit.files\n        if file_.filename.startswith(\"projects/\")\n    }\n\n    return modified_datasets\n\n\ndef archive_dataset(\n    dataset_path: str, out_dir: str, archive_name: str, version: str\n) -> None:\n    os.makedirs(os.path.dirname(out_dir), mode=0o755, exist_ok=True)\n    out_filename = f\"{archive_name}_version-{version}.tar.gz\"\n    logger.info(f\"Archiving dataset: {dataset_path} to {out_filename}\")\n\n    cwd = os.getcwd()\n    try:\n        datalad_archiver = export_archive.ExportArchive()\n        dataset_repo = git.Repo(dataset_path)\n\n        with open(os.path.join(dataset_path, \".git.log\"), \"w\") as fout:\n            fout.write(dataset_repo.git.log(pretty=\"format:%H %s\"))\n\n        # Export is performed from the dataset root.\n        # This is to avoid failure when a submodule is not downloaded; e.g. for parent\n        # dataset in dataset derivative.\n        os.chdir(os.path.join(cwd, dataset_path))\n        datalad_archiver(\".\", filename=out_filename)\n\n    except Exception as e:\n        raise ArchiveFailed(\n            f\"FAILURE: could not archive dataset: {dataset_path} to {out_filename}\\n{e}\"\n        )\n    finally:\n        os.chdir(cwd)\n\n\nif __name__ == \"__main__\":\n    args = parse_args()\n\n    # Only archive the datasets available locally.\n    datasets_path = get_datasets_path()\n    datasets = datasets_path.keys()\n    if args.dataset:\n        target_datasets = {os.path.basename(os.path.normpath(d)) for d in args.dataset}\n        logger.warning(\n            f\"The following dataset were not found locally: {target_datasets - datasets}\"\n        )\n        datasets &= target_datasets\n\n    elif not args.all:\n        modified_datasets = get_modified_datasets()\n        logger.warning(\n            f\"The following dataset were not found locally: {modified_datasets - datasets}\"\n        )\n        datasets &= modified_datasets\n\n    for dataset_name in datasets:\n        dataset = datasets_path[dataset_name]\n\n        try:\n            logger.info(f\"Installing dataset: {dataset}\")\n            install_dataset(dataset)\n\n            is_public = False\n            version = \"\"\n            dataset_size = 0.0\n\n            with open(os.path.join(dataset, \"DATS.json\")) as fin:\n                metadata = json.load(fin)\n\n                is_public = (\n                    metadata.get(\"distributions\", [{}])[0]\n                    .get(\"access\", {})\n                    .get(\"authorizations\", [{}])[0]\n                    .get(\"value\")\n                    == \"public\"\n                )\n                version = metadata.get(\"version\")\n\n                for distribution in metadata.get(\"distributions\", list()):\n                    dataset_size += humanfriendly.parse_size(\n                        f\"{distribution['size']} {distribution['unit']['value']}\",\n                    )\n                    dataset_size //= 1024 ** 3  # Convert to GB\n\n            # Only archive public dataset less than 20GB\n            if dataset_size <= args.max_size and is_public:\n                logger.info(f\"Downloading dataset: {dataset}\")\n                get_dataset(dataset)\n                for submodule in get_proper_submodules(dataset):\n                    get_dataset(submodule)\n\n                archive_name = \"__\".join(\n                    os.path.relpath(dataset, \"projects\").split(\"/\")\n                )\n                archive_dataset(\n                    dataset,\n                    out_dir=os.path.join(args.out_dir, dataset_name),\n                    archive_name=archive_name,\n                    version=version,\n                )\n                logger.info(f\"SUCCESS: archive created for {dataset}\")\n\n            else:\n                logger.info(f\"SKIPPED: archive not needed for {dataset}\")\n\n        except Exception as e:\n            # TODO implement notification system.\n            # This will alert when a dataset fails the archiving process.\n            logger.exception(\n                f\"FAILURE: could not archive dataset: {dataset} to {args.out_dir}.tar.gz\\n{e}\"\n            )\n\n    logger.info(\"Done archiving the datasets.\")\n", "path": "scripts/auto_archive.py"}]}