We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Add the option to apply augmentation while running `gandlf_preprocess`
**Is your feature request related to a problem? Please describe.**
`gandlf_preprocess` is currently applying all options in `data_preprocessing` key, but anything related to augmentations are not getting picked.

**Describe the solution you'd like**
Provide this as an option in CLI, so that users might save on this compute during training.

**Describe alternatives you've considered**
N.A.

**Additional context**
Useful for image-to-image generative tasks.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `GANDLF/cli/preprocess_and_save.py`
Content:
```
1 import os, sys, pickle
2 from pathlib import Path
3 import numpy as np
4 import SimpleITK as sitk
5 import pandas as pd
6 
7 from GANDLF.utils import (
8     get_filename_extension_sanitized,
9     parseTrainingCSV,
10     populate_header_in_parameters,
11 )
12 from GANDLF.parseConfig import parseConfig
13 from GANDLF.data.ImagesFromDataFrame import ImagesFromDataFrame
14 from torch.utils.data import DataLoader
15 from tqdm import tqdm
16 
17 import torchio
18 
19 
20 def preprocess_and_save(data_csv, config_file, output_dir, label_pad_mode="constant"):
21     """
22     This function performs preprocessing based on parameters provided and saves the output.
23 
24     Args:
25         data_csv (str): The CSV file of the training data.
26         config_file (str): The YAML file of the training configuration.
27         output_dir (str): The output directory.
28         label_pad_mode (str): The padding strategy for the label. Defaults to "constant".
29 
30     Raises:
31         ValueError: Parameter check from previous
32     """
33     Path(output_dir).mkdir(parents=True, exist_ok=True)
34 
35     # read the csv
36     # don't care if the dataframe gets shuffled or not
37     dataframe, headers = parseTrainingCSV(data_csv, train=False)
38     parameters = parseConfig(config_file)
39 
40     # save the parameters so that the same compute doesn't happen once again
41     parameter_file = os.path.join(output_dir, "parameters.pkl")
42     if os.path.exists(parameter_file):
43         parameters_prev = pickle.load(open(parameter_file, "rb"))
44         if parameters != parameters_prev:
45             raise ValueError(
46                 "The parameters are not the same as the ones stored in the previous run, please re-check."
47             )
48     else:
49         with open(parameter_file, "wb") as handle:
50             pickle.dump(parameters, handle, protocol=pickle.HIGHEST_PROTOCOL)
51 
52     parameters = populate_header_in_parameters(parameters, headers)
53 
54     data_for_processing = ImagesFromDataFrame(
55         dataframe, parameters, train=False, loader_type="full"
56     )
57 
58     dataloader_for_processing = DataLoader(
59         data_for_processing,
60         batch_size=1,
61         pin_memory=False,
62     )
63 
64     # initialize a new dict for the preprocessed data
65     base_df = pd.read_csv(data_csv)
66     # ensure csv only contains lower case columns
67     base_df.columns = base_df.columns.str.lower()
68     # only store the column names
69     output_columns_to_write = base_df.to_dict()
70     for key in output_columns_to_write.keys():
71         output_columns_to_write[key] = []
72 
73     # keep a record of the keys which contains only images
74     keys_with_images = parameters["headers"]["channelHeaders"]
75     keys_with_images = [str(x) for x in keys_with_images]
76 
77     ## to-do
78     # use dataloader_for_processing to loop through all images
79     # if padding is enabled, ensure that it gets applied to the images
80     # save the images to disk, but keep a record that these images are preprocessed.
81     # create new csv that contains new files.
82 
83     # give warning if label sampler is present but number of patches to extract is > 1
84     if (
85         (parameters["patch_sampler"] == "label")
86         or (isinstance(parameters["patch_sampler"], dict))
87     ) and parameters["q_samples_per_volume"] > 1:
88         print(
89             "[WARNING] Label sampling has been enabled but q_samples_per_volume > 1; this has been known to cause issues, so q_samples_per_volume will be hard-coded to 1 during preprocessing. Please contact GaNDLF developers for more information",
90             file=sys.stderr,
91             flush=True,
92         )
93 
94     for _, (subject) in enumerate(
95         tqdm(dataloader_for_processing, desc="Looping over data")
96     ):
97         # initialize the current_output_dir
98         current_output_dir = os.path.join(output_dir, str(subject["subject_id"][0]))
99         Path(current_output_dir).mkdir(parents=True, exist_ok=True)
100 
101         output_columns_to_write["subjectid"].append(subject["subject_id"][0])
102 
103         subject_dict_to_write, subject_process = {}, {}
104 
105         # start constructing the torchio.Subject object
106         for channel in parameters["headers"]["channelHeaders"]:
107             # the "squeeze" is needed because the dataloader automatically
108             # constructs 5D tensor considering the batch_size as first
109             # dimension, but the constructor needs 4D tensor.
110             subject_process[str(channel)] = torchio.Image(
111                 tensor=subject[str(channel)]["data"].squeeze(0),
112                 type=torchio.INTENSITY,
113                 path=subject[str(channel)]["path"],
114             )
115         if parameters["headers"]["labelHeader"] is not None:
116             subject_process["label"] = torchio.Image(
117                 tensor=subject["label"]["data"].squeeze(0),
118                 type=torchio.LABEL,
119                 path=subject["label"]["path"],
120             )
121         subject_dict_to_write = torchio.Subject(subject_process)
122 
123         # apply a different padding mode to image and label (so that label information is not duplicated)
124         if (parameters["patch_sampler"] == "label") or (
125             isinstance(parameters["patch_sampler"], dict)
126         ):
127             # get the padding size from the patch_size
128             psize_pad = list(
129                 np.asarray(np.ceil(np.divide(parameters["patch_size"], 2)), dtype=int)
130             )
131             # initialize the padder for images
132             padder = torchio.transforms.Pad(
133                 psize_pad, padding_mode="symmetric", include=keys_with_images
134             )
135             subject_dict_to_write = padder(subject_dict_to_write)
136 
137             if parameters["headers"]["labelHeader"] is not None:
138                 # initialize the padder for label
139                 padder_label = torchio.transforms.Pad(
140                     psize_pad, padding_mode=label_pad_mode, include="label"
141                 )
142                 subject_dict_to_write = padder_label(subject_dict_to_write)
143 
144                 sampler = torchio.data.LabelSampler(parameters["patch_size"])
145                 generator = sampler(subject_dict_to_write, num_patches=1)
146                 for patch in generator:
147                     for channel in parameters["headers"]["channelHeaders"]:
148                         subject_dict_to_write[str(channel)] = patch[str(channel)]
149 
150                     subject_dict_to_write["label"] = patch["label"]
151 
152         # write new images
153         common_ext = get_filename_extension_sanitized(subject["1"]["path"][0])
154         # in cases where the original image has a file format that does not support
155         # RGB floats, use the "vtk" format
156         if common_ext in [".png", ".jpg", ".jpeg", ".bmp", ".tiff", ".tif"]:
157             common_ext = ".vtk"
158 
159         if subject["1"]["path"][0] != "":
160             image_for_info_copy = sitk.ReadImage(subject["1"]["path"][0])
161         else:
162             image_for_info_copy = subject_dict_to_write["1"].as_sitk()
163         correct_spacing_for_info_copy = subject["spacing"][0].tolist()
164         for channel in parameters["headers"]["channelHeaders"]:
165             image_file = Path(
166                 os.path.join(
167                     current_output_dir,
168                     subject["subject_id"][0] + "_" + str(channel) + common_ext,
169                 )
170             ).as_posix()
171             output_columns_to_write["channel_" + str(channel - 1)].append(image_file)
172             image_to_write = subject_dict_to_write[str(channel)].as_sitk()
173             image_to_write.SetOrigin(image_for_info_copy.GetOrigin())
174             image_to_write.SetDirection(image_for_info_copy.GetDirection())
175             image_to_write.SetSpacing(correct_spacing_for_info_copy)
176             if not os.path.isfile(image_file):
177                 try:
178                     sitk.WriteImage(image_to_write, image_file)
179                 except IOError:
180                     IOError(
181                         "Could not write image file: {}. Make sure that the file is not open and try again.".format(
182                             image_file
183                         )
184                     )
185                     sys.exit(1)
186 
187         # now try to write the label
188         if "label" in subject_dict_to_write:
189             image_file = Path(
190                 os.path.join(
191                     current_output_dir, subject["subject_id"][0] + "_label" + common_ext
192                 )
193             ).as_posix()
194             output_columns_to_write["label"].append(image_file)
195             image_to_write = subject_dict_to_write["label"].as_sitk()
196             image_to_write.SetOrigin(image_for_info_copy.GetOrigin())
197             image_to_write.SetDirection(image_for_info_copy.GetDirection())
198             image_to_write.SetSpacing(correct_spacing_for_info_copy)
199             if not os.path.isfile(image_file):
200                 try:
201                     sitk.WriteImage(image_to_write, image_file)
202                 except IOError:
203                     IOError(
204                         "Could not write image file: {}. Make sure that the file is not open and try again.".format(
205                             image_file
206                         )
207                     )
208                     sys.exit(1)
209 
210         # ensure prediction headers are getting saved, as well
211         if len(parameters["headers"]["predictionHeaders"]) > 1:
212             for key in parameters["headers"]["predictionHeaders"]:
213                 output_columns_to_write["valuetopredict_" + str(key)].append(
214                     str(subject["value_" + str(key)].numpy()[0])
215                 )
216         elif len(parameters["headers"]["predictionHeaders"]) == 1:
217             output_columns_to_write["valuetopredict"].append(
218                 str(subject["value_0"].numpy()[0])
219             )
220 
221     path_for_csv = Path(os.path.join(output_dir, "data_processed.csv")).as_posix()
222     print("Writing final csv for subsequent training: ", path_for_csv)
223     pd.DataFrame.from_dict(data=output_columns_to_write).to_csv(
224         path_for_csv, header=True, index=False
225     )
226
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/GANDLF/cli/preprocess_and_save.py b/GANDLF/cli/preprocess_and_save.py
--- a/GANDLF/cli/preprocess_and_save.py
+++ b/GANDLF/cli/preprocess_and_save.py
@@ -17,7 +17,9 @@
 import torchio
 
 
-def preprocess_and_save(data_csv, config_file, output_dir, label_pad_mode="constant"):
+def preprocess_and_save(
+    data_csv, config_file, output_dir, label_pad_mode="constant", applyaugs=False
+):
     """
     This function performs preprocessing based on parameters provided and saves the output.
 
@@ -26,6 +28,7 @@
         config_file (str): The YAML file of the training configuration.
         output_dir (str): The output directory.
         label_pad_mode (str): The padding strategy for the label. Defaults to "constant".
+        applyaugs (bool): If data augmentation is to be applied before saving the image. Defaults to False.
 
     Raises:
         ValueError: Parameter check from previous
@@ -52,7 +55,7 @@
     parameters = populate_header_in_parameters(parameters, headers)
 
     data_for_processing = ImagesFromDataFrame(
-        dataframe, parameters, train=False, loader_type="full"
+        dataframe, parameters, train=applyaugs, loader_type="full"
     )
 
     dataloader_for_processing = DataLoader(

{"golden_diff": "diff --git a/GANDLF/cli/preprocess_and_save.py b/GANDLF/cli/preprocess_and_save.py\n--- a/GANDLF/cli/preprocess_and_save.py\n+++ b/GANDLF/cli/preprocess_and_save.py\n@@ -17,7 +17,9 @@\n import torchio\n \n \n-def preprocess_and_save(data_csv, config_file, output_dir, label_pad_mode=\"constant\"):\n+def preprocess_and_save(\n+    data_csv, config_file, output_dir, label_pad_mode=\"constant\", applyaugs=False\n+):\n     \"\"\"\n     This function performs preprocessing based on parameters provided and saves the output.\n \n@@ -26,6 +28,7 @@\n         config_file (str): The YAML file of the training configuration.\n         output_dir (str): The output directory.\n         label_pad_mode (str): The padding strategy for the label. Defaults to \"constant\".\n+        applyaugs (bool): If data augmentation is to be applied before saving the image. Defaults to False.\n \n     Raises:\n         ValueError: Parameter check from previous\n@@ -52,7 +55,7 @@\n     parameters = populate_header_in_parameters(parameters, headers)\n \n     data_for_processing = ImagesFromDataFrame(\n-        dataframe, parameters, train=False, loader_type=\"full\"\n+        dataframe, parameters, train=applyaugs, loader_type=\"full\"\n     )\n \n     dataloader_for_processing = DataLoader(\n", "issue": "Add the option to apply augmentation while running `gandlf_preprocess`\n**Is your feature request related to a problem? Please describe.**\r\n`gandlf_preprocess` is currently applying all options in `data_preprocessing` key, but anything related to augmentations are not getting picked.\r\n\r\n**Describe the solution you'd like**\r\nProvide this as an option in CLI, so that users might save on this compute during training.\r\n\r\n**Describe alternatives you've considered**\r\nN.A.\r\n\r\n**Additional context**\r\nUseful for image-to-image generative tasks.\r\n\n", "before_files": [{"content": "import os, sys, pickle\nfrom pathlib import Path\nimport numpy as np\nimport SimpleITK as sitk\nimport pandas as pd\n\nfrom GANDLF.utils import (\n    get_filename_extension_sanitized,\n    parseTrainingCSV,\n    populate_header_in_parameters,\n)\nfrom GANDLF.parseConfig import parseConfig\nfrom GANDLF.data.ImagesFromDataFrame import ImagesFromDataFrame\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\n\nimport torchio\n\n\ndef preprocess_and_save(data_csv, config_file, output_dir, label_pad_mode=\"constant\"):\n    \"\"\"\n    This function performs preprocessing based on parameters provided and saves the output.\n\n    Args:\n        data_csv (str): The CSV file of the training data.\n        config_file (str): The YAML file of the training configuration.\n        output_dir (str): The output directory.\n        label_pad_mode (str): The padding strategy for the label. Defaults to \"constant\".\n\n    Raises:\n        ValueError: Parameter check from previous\n    \"\"\"\n    Path(output_dir).mkdir(parents=True, exist_ok=True)\n\n    # read the csv\n    # don't care if the dataframe gets shuffled or not\n    dataframe, headers = parseTrainingCSV(data_csv, train=False)\n    parameters = parseConfig(config_file)\n\n    # save the parameters so that the same compute doesn't happen once again\n    parameter_file = os.path.join(output_dir, \"parameters.pkl\")\n    if os.path.exists(parameter_file):\n        parameters_prev = pickle.load(open(parameter_file, \"rb\"))\n        if parameters != parameters_prev:\n            raise ValueError(\n                \"The parameters are not the same as the ones stored in the previous run, please re-check.\"\n            )\n    else:\n        with open(parameter_file, \"wb\") as handle:\n            pickle.dump(parameters, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\n    parameters = populate_header_in_parameters(parameters, headers)\n\n    data_for_processing = ImagesFromDataFrame(\n        dataframe, parameters, train=False, loader_type=\"full\"\n    )\n\n    dataloader_for_processing = DataLoader(\n        data_for_processing,\n        batch_size=1,\n        pin_memory=False,\n    )\n\n    # initialize a new dict for the preprocessed data\n    base_df = pd.read_csv(data_csv)\n    # ensure csv only contains lower case columns\n    base_df.columns = base_df.columns.str.lower()\n    # only store the column names\n    output_columns_to_write = base_df.to_dict()\n    for key in output_columns_to_write.keys():\n        output_columns_to_write[key] = []\n\n    # keep a record of the keys which contains only images\n    keys_with_images = parameters[\"headers\"][\"channelHeaders\"]\n    keys_with_images = [str(x) for x in keys_with_images]\n\n    ## to-do\n    # use dataloader_for_processing to loop through all images\n    # if padding is enabled, ensure that it gets applied to the images\n    # save the images to disk, but keep a record that these images are preprocessed.\n    # create new csv that contains new files.\n\n    # give warning if label sampler is present but number of patches to extract is > 1\n    if (\n        (parameters[\"patch_sampler\"] == \"label\")\n        or (isinstance(parameters[\"patch_sampler\"], dict))\n    ) and parameters[\"q_samples_per_volume\"] > 1:\n        print(\n            \"[WARNING] Label sampling has been enabled but q_samples_per_volume > 1; this has been known to cause issues, so q_samples_per_volume will be hard-coded to 1 during preprocessing. Please contact GaNDLF developers for more information\",\n            file=sys.stderr,\n            flush=True,\n        )\n\n    for _, (subject) in enumerate(\n        tqdm(dataloader_for_processing, desc=\"Looping over data\")\n    ):\n        # initialize the current_output_dir\n        current_output_dir = os.path.join(output_dir, str(subject[\"subject_id\"][0]))\n        Path(current_output_dir).mkdir(parents=True, exist_ok=True)\n\n        output_columns_to_write[\"subjectid\"].append(subject[\"subject_id\"][0])\n\n        subject_dict_to_write, subject_process = {}, {}\n\n        # start constructing the torchio.Subject object\n        for channel in parameters[\"headers\"][\"channelHeaders\"]:\n            # the \"squeeze\" is needed because the dataloader automatically\n            # constructs 5D tensor considering the batch_size as first\n            # dimension, but the constructor needs 4D tensor.\n            subject_process[str(channel)] = torchio.Image(\n                tensor=subject[str(channel)][\"data\"].squeeze(0),\n                type=torchio.INTENSITY,\n                path=subject[str(channel)][\"path\"],\n            )\n        if parameters[\"headers\"][\"labelHeader\"] is not None:\n            subject_process[\"label\"] = torchio.Image(\n                tensor=subject[\"label\"][\"data\"].squeeze(0),\n                type=torchio.LABEL,\n                path=subject[\"label\"][\"path\"],\n            )\n        subject_dict_to_write = torchio.Subject(subject_process)\n\n        # apply a different padding mode to image and label (so that label information is not duplicated)\n        if (parameters[\"patch_sampler\"] == \"label\") or (\n            isinstance(parameters[\"patch_sampler\"], dict)\n        ):\n            # get the padding size from the patch_size\n            psize_pad = list(\n                np.asarray(np.ceil(np.divide(parameters[\"patch_size\"], 2)), dtype=int)\n            )\n            # initialize the padder for images\n            padder = torchio.transforms.Pad(\n                psize_pad, padding_mode=\"symmetric\", include=keys_with_images\n            )\n            subject_dict_to_write = padder(subject_dict_to_write)\n\n            if parameters[\"headers\"][\"labelHeader\"] is not None:\n                # initialize the padder for label\n                padder_label = torchio.transforms.Pad(\n                    psize_pad, padding_mode=label_pad_mode, include=\"label\"\n                )\n                subject_dict_to_write = padder_label(subject_dict_to_write)\n\n                sampler = torchio.data.LabelSampler(parameters[\"patch_size\"])\n                generator = sampler(subject_dict_to_write, num_patches=1)\n                for patch in generator:\n                    for channel in parameters[\"headers\"][\"channelHeaders\"]:\n                        subject_dict_to_write[str(channel)] = patch[str(channel)]\n\n                    subject_dict_to_write[\"label\"] = patch[\"label\"]\n\n        # write new images\n        common_ext = get_filename_extension_sanitized(subject[\"1\"][\"path\"][0])\n        # in cases where the original image has a file format that does not support\n        # RGB floats, use the \"vtk\" format\n        if common_ext in [\".png\", \".jpg\", \".jpeg\", \".bmp\", \".tiff\", \".tif\"]:\n            common_ext = \".vtk\"\n\n        if subject[\"1\"][\"path\"][0] != \"\":\n            image_for_info_copy = sitk.ReadImage(subject[\"1\"][\"path\"][0])\n        else:\n            image_for_info_copy = subject_dict_to_write[\"1\"].as_sitk()\n        correct_spacing_for_info_copy = subject[\"spacing\"][0].tolist()\n        for channel in parameters[\"headers\"][\"channelHeaders\"]:\n            image_file = Path(\n                os.path.join(\n                    current_output_dir,\n                    subject[\"subject_id\"][0] + \"_\" + str(channel) + common_ext,\n                )\n            ).as_posix()\n            output_columns_to_write[\"channel_\" + str(channel - 1)].append(image_file)\n            image_to_write = subject_dict_to_write[str(channel)].as_sitk()\n            image_to_write.SetOrigin(image_for_info_copy.GetOrigin())\n            image_to_write.SetDirection(image_for_info_copy.GetDirection())\n            image_to_write.SetSpacing(correct_spacing_for_info_copy)\n            if not os.path.isfile(image_file):\n                try:\n                    sitk.WriteImage(image_to_write, image_file)\n                except IOError:\n                    IOError(\n                        \"Could not write image file: {}. Make sure that the file is not open and try again.\".format(\n                            image_file\n                        )\n                    )\n                    sys.exit(1)\n\n        # now try to write the label\n        if \"label\" in subject_dict_to_write:\n            image_file = Path(\n                os.path.join(\n                    current_output_dir, subject[\"subject_id\"][0] + \"_label\" + common_ext\n                )\n            ).as_posix()\n            output_columns_to_write[\"label\"].append(image_file)\n            image_to_write = subject_dict_to_write[\"label\"].as_sitk()\n            image_to_write.SetOrigin(image_for_info_copy.GetOrigin())\n            image_to_write.SetDirection(image_for_info_copy.GetDirection())\n            image_to_write.SetSpacing(correct_spacing_for_info_copy)\n            if not os.path.isfile(image_file):\n                try:\n                    sitk.WriteImage(image_to_write, image_file)\n                except IOError:\n                    IOError(\n                        \"Could not write image file: {}. Make sure that the file is not open and try again.\".format(\n                            image_file\n                        )\n                    )\n                    sys.exit(1)\n\n        # ensure prediction headers are getting saved, as well\n        if len(parameters[\"headers\"][\"predictionHeaders\"]) > 1:\n            for key in parameters[\"headers\"][\"predictionHeaders\"]:\n                output_columns_to_write[\"valuetopredict_\" + str(key)].append(\n                    str(subject[\"value_\" + str(key)].numpy()[0])\n                )\n        elif len(parameters[\"headers\"][\"predictionHeaders\"]) == 1:\n            output_columns_to_write[\"valuetopredict\"].append(\n                str(subject[\"value_0\"].numpy()[0])\n            )\n\n    path_for_csv = Path(os.path.join(output_dir, \"data_processed.csv\")).as_posix()\n    print(\"Writing final csv for subsequent training: \", path_for_csv)\n    pd.DataFrame.from_dict(data=output_columns_to_write).to_csv(\n        path_for_csv, header=True, index=False\n    )\n", "path": "GANDLF/cli/preprocess_and_save.py"}], "after_files": [{"content": "import os, sys, pickle\nfrom pathlib import Path\nimport numpy as np\nimport SimpleITK as sitk\nimport pandas as pd\n\nfrom GANDLF.utils import (\n    get_filename_extension_sanitized,\n    parseTrainingCSV,\n    populate_header_in_parameters,\n)\nfrom GANDLF.parseConfig import parseConfig\nfrom GANDLF.data.ImagesFromDataFrame import ImagesFromDataFrame\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\n\nimport torchio\n\n\ndef preprocess_and_save(\n    data_csv, config_file, output_dir, label_pad_mode=\"constant\", applyaugs=False\n):\n    \"\"\"\n    This function performs preprocessing based on parameters provided and saves the output.\n\n    Args:\n        data_csv (str): The CSV file of the training data.\n        config_file (str): The YAML file of the training configuration.\n        output_dir (str): The output directory.\n        label_pad_mode (str): The padding strategy for the label. Defaults to \"constant\".\n        applyaugs (bool): If data augmentation is to be applied before saving the image. Defaults to False.\n\n    Raises:\n        ValueError: Parameter check from previous\n    \"\"\"\n    Path(output_dir).mkdir(parents=True, exist_ok=True)\n\n    # read the csv\n    # don't care if the dataframe gets shuffled or not\n    dataframe, headers = parseTrainingCSV(data_csv, train=False)\n    parameters = parseConfig(config_file)\n\n    # save the parameters so that the same compute doesn't happen once again\n    parameter_file = os.path.join(output_dir, \"parameters.pkl\")\n    if os.path.exists(parameter_file):\n        parameters_prev = pickle.load(open(parameter_file, \"rb\"))\n        if parameters != parameters_prev:\n            raise ValueError(\n                \"The parameters are not the same as the ones stored in the previous run, please re-check.\"\n            )\n    else:\n        with open(parameter_file, \"wb\") as handle:\n            pickle.dump(parameters, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\n    parameters = populate_header_in_parameters(parameters, headers)\n\n    data_for_processing = ImagesFromDataFrame(\n        dataframe, parameters, train=applyaugs, loader_type=\"full\"\n    )\n\n    dataloader_for_processing = DataLoader(\n        data_for_processing,\n        batch_size=1,\n        pin_memory=False,\n    )\n\n    # initialize a new dict for the preprocessed data\n    base_df = pd.read_csv(data_csv)\n    # ensure csv only contains lower case columns\n    base_df.columns = base_df.columns.str.lower()\n    # only store the column names\n    output_columns_to_write = base_df.to_dict()\n    for key in output_columns_to_write.keys():\n        output_columns_to_write[key] = []\n\n    # keep a record of the keys which contains only images\n    keys_with_images = parameters[\"headers\"][\"channelHeaders\"]\n    keys_with_images = [str(x) for x in keys_with_images]\n\n    ## to-do\n    # use dataloader_for_processing to loop through all images\n    # if padding is enabled, ensure that it gets applied to the images\n    # save the images to disk, but keep a record that these images are preprocessed.\n    # create new csv that contains new files.\n\n    # give warning if label sampler is present but number of patches to extract is > 1\n    if (\n        (parameters[\"patch_sampler\"] == \"label\")\n        or (isinstance(parameters[\"patch_sampler\"], dict))\n    ) and parameters[\"q_samples_per_volume\"] > 1:\n        print(\n            \"[WARNING] Label sampling has been enabled but q_samples_per_volume > 1; this has been known to cause issues, so q_samples_per_volume will be hard-coded to 1 during preprocessing. Please contact GaNDLF developers for more information\",\n            file=sys.stderr,\n            flush=True,\n        )\n\n    for _, (subject) in enumerate(\n        tqdm(dataloader_for_processing, desc=\"Looping over data\")\n    ):\n        # initialize the current_output_dir\n        current_output_dir = os.path.join(output_dir, str(subject[\"subject_id\"][0]))\n        Path(current_output_dir).mkdir(parents=True, exist_ok=True)\n\n        output_columns_to_write[\"subjectid\"].append(subject[\"subject_id\"][0])\n\n        subject_dict_to_write, subject_process = {}, {}\n\n        # start constructing the torchio.Subject object\n        for channel in parameters[\"headers\"][\"channelHeaders\"]:\n            # the \"squeeze\" is needed because the dataloader automatically\n            # constructs 5D tensor considering the batch_size as first\n            # dimension, but the constructor needs 4D tensor.\n            subject_process[str(channel)] = torchio.Image(\n                tensor=subject[str(channel)][\"data\"].squeeze(0),\n                type=torchio.INTENSITY,\n                path=subject[str(channel)][\"path\"],\n            )\n        if parameters[\"headers\"][\"labelHeader\"] is not None:\n            subject_process[\"label\"] = torchio.Image(\n                tensor=subject[\"label\"][\"data\"].squeeze(0),\n                type=torchio.LABEL,\n                path=subject[\"label\"][\"path\"],\n            )\n        subject_dict_to_write = torchio.Subject(subject_process)\n\n        # apply a different padding mode to image and label (so that label information is not duplicated)\n        if (parameters[\"patch_sampler\"] == \"label\") or (\n            isinstance(parameters[\"patch_sampler\"], dict)\n        ):\n            # get the padding size from the patch_size\n            psize_pad = list(\n                np.asarray(np.ceil(np.divide(parameters[\"patch_size\"], 2)), dtype=int)\n            )\n            # initialize the padder for images\n            padder = torchio.transforms.Pad(\n                psize_pad, padding_mode=\"symmetric\", include=keys_with_images\n            )\n            subject_dict_to_write = padder(subject_dict_to_write)\n\n            if parameters[\"headers\"][\"labelHeader\"] is not None:\n                # initialize the padder for label\n                padder_label = torchio.transforms.Pad(\n                    psize_pad, padding_mode=label_pad_mode, include=\"label\"\n                )\n                subject_dict_to_write = padder_label(subject_dict_to_write)\n\n                sampler = torchio.data.LabelSampler(parameters[\"patch_size\"])\n                generator = sampler(subject_dict_to_write, num_patches=1)\n                for patch in generator:\n                    for channel in parameters[\"headers\"][\"channelHeaders\"]:\n                        subject_dict_to_write[str(channel)] = patch[str(channel)]\n\n                    subject_dict_to_write[\"label\"] = patch[\"label\"]\n\n        # write new images\n        common_ext = get_filename_extension_sanitized(subject[\"1\"][\"path\"][0])\n        # in cases where the original image has a file format that does not support\n        # RGB floats, use the \"vtk\" format\n        if common_ext in [\".png\", \".jpg\", \".jpeg\", \".bmp\", \".tiff\", \".tif\"]:\n            common_ext = \".vtk\"\n\n        if subject[\"1\"][\"path\"][0] != \"\":\n            image_for_info_copy = sitk.ReadImage(subject[\"1\"][\"path\"][0])\n        else:\n            image_for_info_copy = subject_dict_to_write[\"1\"].as_sitk()\n        correct_spacing_for_info_copy = subject[\"spacing\"][0].tolist()\n        for channel in parameters[\"headers\"][\"channelHeaders\"]:\n            image_file = Path(\n                os.path.join(\n                    current_output_dir,\n                    subject[\"subject_id\"][0] + \"_\" + str(channel) + common_ext,\n                )\n            ).as_posix()\n            output_columns_to_write[\"channel_\" + str(channel - 1)].append(image_file)\n            image_to_write = subject_dict_to_write[str(channel)].as_sitk()\n            image_to_write.SetOrigin(image_for_info_copy.GetOrigin())\n            image_to_write.SetDirection(image_for_info_copy.GetDirection())\n            image_to_write.SetSpacing(correct_spacing_for_info_copy)\n            if not os.path.isfile(image_file):\n                try:\n                    sitk.WriteImage(image_to_write, image_file)\n                except IOError:\n                    IOError(\n                        \"Could not write image file: {}. Make sure that the file is not open and try again.\".format(\n                            image_file\n                        )\n                    )\n                    sys.exit(1)\n\n        # now try to write the label\n        if \"label\" in subject_dict_to_write:\n            image_file = Path(\n                os.path.join(\n                    current_output_dir, subject[\"subject_id\"][0] + \"_label\" + common_ext\n                )\n            ).as_posix()\n            output_columns_to_write[\"label\"].append(image_file)\n            image_to_write = subject_dict_to_write[\"label\"].as_sitk()\n            image_to_write.SetOrigin(image_for_info_copy.GetOrigin())\n            image_to_write.SetDirection(image_for_info_copy.GetDirection())\n            image_to_write.SetSpacing(correct_spacing_for_info_copy)\n            if not os.path.isfile(image_file):\n                try:\n                    sitk.WriteImage(image_to_write, image_file)\n                except IOError:\n                    IOError(\n                        \"Could not write image file: {}. Make sure that the file is not open and try again.\".format(\n                            image_file\n                        )\n                    )\n                    sys.exit(1)\n\n        # ensure prediction headers are getting saved, as well\n        if len(parameters[\"headers\"][\"predictionHeaders\"]) > 1:\n            for key in parameters[\"headers\"][\"predictionHeaders\"]:\n                output_columns_to_write[\"valuetopredict_\" + str(key)].append(\n                    str(subject[\"value_\" + str(key)].numpy()[0])\n                )\n        elif len(parameters[\"headers\"][\"predictionHeaders\"]) == 1:\n            output_columns_to_write[\"valuetopredict\"].append(\n                str(subject[\"value_0\"].numpy()[0])\n            )\n\n    path_for_csv = Path(os.path.join(output_dir, \"data_processed.csv\")).as_posix()\n    print(\"Writing final csv for subsequent training: \", path_for_csv)\n    pd.DataFrame.from_dict(data=output_columns_to_write).to_csv(\n        path_for_csv, header=True, index=False\n    )\n", "path": "GANDLF/cli/preprocess_and_save.py"}]}