We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
cesm2_alpha10d doesn't attempt to download missing input files
I recently ported both alpha10c and alpha10d on my macbook. While alpha10c successfully downloads all missing input files, alpha10d doesn't appear to attempt to download missing input files. I've encountered this behavior both with A and C compsets.

Here is a snippet of ./case.submit standart output for alpha10c:
```
Loading input file list: 'Buildconf/cpl.input_data_list'
  Model cpl missing file ocn2atm_fmapname = '/Users/altuntas/projects/cesm-inputdata/cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc'
Trying to download file: 'cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc' to path '/Users/altuntas/projects/cesm-inputdata/cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc'
SUCCESS
```

And here is the corresponding ./case.submit standart output for alpha10d:
```
Loading input file list: 'Buildconf/cpl.input_data_list'
  Model cpl missing file ocn2atm_fmapname = '/Users/altuntas/projects/cesm-inputdata/cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc'
  Model cpl missing file ocn2atm_smapname = '/Users/altuntas/projects/cesm-inputdata/cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc'
  Model cpl missing file ice2atm_fmapname = '/Users/altuntas/projects/cesm-inputdata/cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc'
  Model cpl missing file ice2atm_smapname = '/Users/altuntas/projects/cesm-inputdata/cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc'
```

While alpha10c runs sucessfully on my macbook (high sierra), alpha10d eventually fails due to missing input files.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/lib/CIME/case/check_input_data.py`
Content:
```
1 """
2 API for checking input for testcase
3 """
4 from CIME.XML.standard_module_setup import *
5 from CIME.utils import SharedArea, find_files, safe_copy, expect
6 from CIME.XML.inputdata import Inputdata
7 import CIME.Servers
8 
9 import glob
10 
11 logger = logging.getLogger(__name__)
12 
13 def _download_if_in_repo(server, input_data_root, rel_path, isdirectory=False):
14     """
15     Return True if successfully downloaded
16     """
17     if not server.fileexists(rel_path):
18         return False
19 
20     full_path = os.path.join(input_data_root, rel_path)
21     logging.info("Trying to download file: '{}' to path '{}'".format(rel_path, full_path))
22     # Make sure local path exists, create if it does not
23     if isdirectory or full_path.endswith(os.sep):
24         if not os.path.exists(full_path):
25             logger.info("Creating directory {}".format(full_path))
26             os.makedirs(full_path)
27         isdirectory = True
28     elif not os.path.exists(os.path.dirname(full_path)):
29         os.makedirs(os.path.dirname(full_path))
30 
31     # Use umask to make sure files are group read/writable. As long as parent directories
32     # have +s, then everything should work.
33     with SharedArea():
34         if isdirectory:
35             return server.getdirectory(rel_path, full_path)
36         else:
37             return server.getfile(rel_path, full_path)
38 
39 ###############################################################################
40 def check_all_input_data(self, protocal=None, address=None, input_data_root=None, data_list_dir="Buildconf", download=True):
41 ###############################################################################
42     success = False
43     if protocal is not None and address is not None:
44         success = self.check_input_data(protocal=protocal, address=address, download=download,
45                                    input_data_root=input_data_root, data_list_dir=data_list_dir)
46     else:
47         success = self.check_input_data(protocal=protocal, address=address, download=False,
48                                    input_data_root=input_data_root, data_list_dir=data_list_dir)
49         if download and not success:
50             success = _downloadfromserver(self, input_data_root, data_list_dir)
51 
52     self.stage_refcase(input_data_root=input_data_root, data_list_dir=data_list_dir)
53     return success
54 
55 def _downloadfromserver(case, input_data_root, data_list_dir):
56     # needs to be downloaded
57     success = False
58     protocal = 'svn'
59     inputdata = Inputdata()
60     while not success and protocal is not None:
61         protocal, address = inputdata.get_next_server()
62         logger.info("Checking server {} with protocal {}".format(address, protocal))
63         success = case.check_input_data(protocal=protocal, address=address, download=True,
64                                         input_data_root=input_data_root, data_list_dir=data_list_dir)
65     return success
66 
67 def stage_refcase(self, input_data_root=None, data_list_dir=None):
68     get_refcase  = self.get_value("GET_REFCASE")
69     run_type     = self.get_value("RUN_TYPE")
70     continue_run = self.get_value("CONTINUE_RUN")
71 
72     # We do not fully populate the inputdata directory on every
73     # machine and do not expect every user to download the 3TB+ of
74     # data in our inputdata repository. This code checks for the
75     # existence of inputdata in the local inputdata directory and
76     # attempts to download data from the server if it's needed and
77     # missing.
78     if get_refcase and run_type != "startup" and not continue_run:
79         din_loc_root = self.get_value("DIN_LOC_ROOT")
80         run_refdate  = self.get_value("RUN_REFDATE")
81         run_refcase  = self.get_value("RUN_REFCASE")
82         run_refdir   = self.get_value("RUN_REFDIR")
83         rundir       = self.get_value("RUNDIR")
84 
85         refdir = os.path.join(din_loc_root, run_refdir, run_refcase, run_refdate)
86         if not os.path.isdir(refdir):
87             logger.warning("Refcase not found in {}, will attempt to download from inputdata".format(refdir))
88             with open(os.path.join("Buildconf","refcase.input_data_list"),"w") as fd:
89                 fd.write("refdir = {}{}".format(refdir, os.sep))
90             if input_data_root is None:
91                 input_data_root = din_loc_root
92             if data_list_dir is None:
93                 data_list_dir = "Buildconf"
94             success = _downloadfromserver(self, input_data_root=input_data_root, data_list_dir=data_list_dir)
95             expect(success, "Could not download refcase from any server")
96 
97         logger.info(" - Prestaging REFCASE ({}) to {}".format(refdir, rundir))
98 
99         # prestage the reference case's files.
100 
101         if (not os.path.exists(rundir)):
102             logger.debug("Creating run directory: {}".format(rundir))
103             os.makedirs(rundir)
104 
105         # copy the refcases' rpointer files to the run directory
106         for rpointerfile in glob.iglob(os.path.join("{}","*rpointer*").format(refdir)):
107             logger.info("Copy rpointer {}".format(rpointerfile))
108             safe_copy(rpointerfile, rundir)
109 
110         # link everything else
111 
112         for rcfile in glob.iglob(os.path.join(refdir,"*")):
113             rcbaseline = os.path.basename(rcfile)
114             if not os.path.exists("{}/{}".format(rundir, rcbaseline)):
115                 logger.info("Staging file {}".format(rcfile))
116                 os.symlink(rcfile, "{}/{}".format(rundir, rcbaseline))
117         # Backward compatibility, some old refcases have cam2 in the name
118         # link to local cam file.
119         for cam2file in  glob.iglob(os.path.join("{}","*.cam2.*").format(rundir)):
120             camfile = cam2file.replace("cam2", "cam")
121             os.symlink(cam2file, camfile)
122 
123     return True
124 
125 def check_input_data(case, protocal="svn", address=None, input_data_root=None, data_list_dir="Buildconf", download=False):
126     """
127     Return True if no files missing
128     """
129     case.load_env(reset=True)
130     # Fill in defaults as needed
131     input_data_root = case.get_value("DIN_LOC_ROOT") if input_data_root is None else input_data_root
132 
133     expect(os.path.isdir(input_data_root), "Invalid input_data_root directory: '{}'".format(input_data_root))
134     expect(os.path.isdir(data_list_dir), "Invalid data_list_dir directory: '{}'".format(data_list_dir))
135 
136     data_list_files = find_files(data_list_dir, "*.input_data_list")
137     expect(data_list_files, "No .input_data_list files found in dir '{}'".format(data_list_dir))
138 
139     no_files_missing = True
140 
141     if download:
142         if protocal not in vars(CIME.Servers):
143             logger.warning("Client protocal {} not enabled".format(protocal))
144             return False
145 
146         if protocal == "svn":
147             server = CIME.Servers.SVN(address)
148         elif protocal == "gftp":
149             server = CIME.Servers.GridFTP(address)
150         elif protocal == "ftp":
151             server = CIME.Servers.FTP(address)
152         elif protocal == "wget":
153             server = CIME.Servers.WGET(address)
154         else:
155             expect(False, "Unsupported inputdata protocal: {}".format(protocal))
156 
157 
158 
159     for data_list_file in data_list_files:
160         logging.info("Loading input file list: '{}'".format(data_list_file))
161         with open(data_list_file, "r") as fd:
162             lines = fd.readlines()
163 
164         for line in lines:
165             line = line.strip()
166             if (line and not line.startswith("#")):
167                 tokens = line.split('=')
168                 description, full_path = tokens[0].strip(), tokens[1].strip()
169                 if(full_path):
170                     # expand xml variables
171                     full_path = case.get_resolved_value(full_path)
172                     rel_path  = full_path.replace(input_data_root, "")
173                     model = os.path.basename(data_list_file).split('.')[0]
174 
175                     if ("/" in rel_path and rel_path == full_path):
176                         # User pointing to a file outside of input_data_root, we cannot determine
177                         # rel_path, and so cannot download the file. If it already exists, we can
178                         # proceed
179                         if not os.path.exists(full_path):
180                             logging.warning("  Model {} missing file {} = '{}'".format(model, description, full_path))
181                             if download:
182                                 logging.warning("    Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root))
183                             no_files_missing = False
184                         else:
185                             logging.debug("  Found input file: '{}'".format(full_path))
186 
187                     else:
188                         # There are some special values of rel_path that
189                         # we need to ignore - some of the component models
190                         # set things like 'NULL' or 'same_as_TS' -
191                         # basically if rel_path does not contain '/' (a
192                         # directory tree) you can assume it's a special
193                         # value and ignore it (perhaps with a warning)
194                         if ("/" in rel_path and not os.path.exists(full_path)):
195                             logging.warning("  Model {} missing file {} = '{}'".format(model, description, full_path))
196 
197                             if (download):
198                                 success = _download_if_in_repo(server, input_data_root, rel_path.strip(os.sep),
199                                                                isdirectory=rel_path.endswith(os.sep))
200                                 if not success:
201                                     no_files_missing = False
202                         else:
203                             logging.debug("  Already had input file: '{}'".format(full_path))
204 
205                 else:
206                     model = os.path.basename(data_list_file).split('.')[0]
207                     logging.warning("Model {} no file specified for {}".format(model, description))
208 
209     return no_files_missing
210
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/lib/CIME/case/check_input_data.py b/scripts/lib/CIME/case/check_input_data.py
--- a/scripts/lib/CIME/case/check_input_data.py
+++ b/scripts/lib/CIME/case/check_input_data.py
@@ -177,7 +177,7 @@
                         # rel_path, and so cannot download the file. If it already exists, we can
                         # proceed
                         if not os.path.exists(full_path):
-                            logging.warning("  Model {} missing file {} = '{}'".format(model, description, full_path))
+                            logging.warning("Model {} missing file {} = '{}'".format(model, description, full_path))
                             if download:
                                 logging.warning("    Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root))
                             no_files_missing = False
@@ -193,12 +193,11 @@
                         # value and ignore it (perhaps with a warning)
                         if ("/" in rel_path and not os.path.exists(full_path)):
                             logging.warning("  Model {} missing file {} = '{}'".format(model, description, full_path))
+                            no_files_missing = False
 
                             if (download):
-                                success = _download_if_in_repo(server, input_data_root, rel_path.strip(os.sep),
+                                no_files_missing = _download_if_in_repo(server, input_data_root, rel_path.strip(os.sep),
                                                                isdirectory=rel_path.endswith(os.sep))
-                                if not success:
-                                    no_files_missing = False
                         else:
                             logging.debug("  Already had input file: '{}'".format(full_path))

{"golden_diff": "diff --git a/scripts/lib/CIME/case/check_input_data.py b/scripts/lib/CIME/case/check_input_data.py\n--- a/scripts/lib/CIME/case/check_input_data.py\n+++ b/scripts/lib/CIME/case/check_input_data.py\n@@ -177,7 +177,7 @@\n                         # rel_path, and so cannot download the file. If it already exists, we can\n                         # proceed\n                         if not os.path.exists(full_path):\n-                            logging.warning(\"  Model {} missing file {} = '{}'\".format(model, description, full_path))\n+                            logging.warning(\"Model {} missing file {} = '{}'\".format(model, description, full_path))\n                             if download:\n                                 logging.warning(\"    Cannot download file since it lives outside of the input_data_root '{}'\".format(input_data_root))\n                             no_files_missing = False\n@@ -193,12 +193,11 @@\n                         # value and ignore it (perhaps with a warning)\n                         if (\"/\" in rel_path and not os.path.exists(full_path)):\n                             logging.warning(\"  Model {} missing file {} = '{}'\".format(model, description, full_path))\n+                            no_files_missing = False\n \n                             if (download):\n-                                success = _download_if_in_repo(server, input_data_root, rel_path.strip(os.sep),\n+                                no_files_missing = _download_if_in_repo(server, input_data_root, rel_path.strip(os.sep),\n                                                                isdirectory=rel_path.endswith(os.sep))\n-                                if not success:\n-                                    no_files_missing = False\n                         else:\n                             logging.debug(\"  Already had input file: '{}'\".format(full_path))\n", "issue": "cesm2_alpha10d doesn't attempt to download missing input files\nI recently ported both alpha10c and alpha10d on my macbook. While alpha10c successfully downloads all missing input files, alpha10d doesn't appear to attempt to download missing input files. I've encountered this behavior both with A and C compsets.\r\n\r\nHere is a snippet of ./case.submit standart output for alpha10c:\r\n```\r\nLoading input file list: 'Buildconf/cpl.input_data_list'\r\n  Model cpl missing file ocn2atm_fmapname = '/Users/altuntas/projects/cesm-inputdata/cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc'\r\nTrying to download file: 'cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc' to path '/Users/altuntas/projects/cesm-inputdata/cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc'\r\nSUCCESS\r\n```\r\n\r\nAnd here is the corresponding ./case.submit standart output for alpha10d:\r\n```\r\nLoading input file list: 'Buildconf/cpl.input_data_list'\r\n  Model cpl missing file ocn2atm_fmapname = '/Users/altuntas/projects/cesm-inputdata/cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc'\r\n  Model cpl missing file ocn2atm_smapname = '/Users/altuntas/projects/cesm-inputdata/cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc'\r\n  Model cpl missing file ice2atm_fmapname = '/Users/altuntas/projects/cesm-inputdata/cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc'\r\n  Model cpl missing file ice2atm_smapname = '/Users/altuntas/projects/cesm-inputdata/cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc'\r\n```\r\n\r\nWhile alpha10c runs sucessfully on my macbook (high sierra), alpha10d eventually fails due to missing input files.\r\n\n", "before_files": [{"content": "\"\"\"\nAPI for checking input for testcase\n\"\"\"\nfrom CIME.XML.standard_module_setup import *\nfrom CIME.utils import SharedArea, find_files, safe_copy, expect\nfrom CIME.XML.inputdata import Inputdata\nimport CIME.Servers\n\nimport glob\n\nlogger = logging.getLogger(__name__)\n\ndef _download_if_in_repo(server, input_data_root, rel_path, isdirectory=False):\n    \"\"\"\n    Return True if successfully downloaded\n    \"\"\"\n    if not server.fileexists(rel_path):\n        return False\n\n    full_path = os.path.join(input_data_root, rel_path)\n    logging.info(\"Trying to download file: '{}' to path '{}'\".format(rel_path, full_path))\n    # Make sure local path exists, create if it does not\n    if isdirectory or full_path.endswith(os.sep):\n        if not os.path.exists(full_path):\n            logger.info(\"Creating directory {}\".format(full_path))\n            os.makedirs(full_path)\n        isdirectory = True\n    elif not os.path.exists(os.path.dirname(full_path)):\n        os.makedirs(os.path.dirname(full_path))\n\n    # Use umask to make sure files are group read/writable. As long as parent directories\n    # have +s, then everything should work.\n    with SharedArea():\n        if isdirectory:\n            return server.getdirectory(rel_path, full_path)\n        else:\n            return server.getfile(rel_path, full_path)\n\n###############################################################################\ndef check_all_input_data(self, protocal=None, address=None, input_data_root=None, data_list_dir=\"Buildconf\", download=True):\n###############################################################################\n    success = False\n    if protocal is not None and address is not None:\n        success = self.check_input_data(protocal=protocal, address=address, download=download,\n                                   input_data_root=input_data_root, data_list_dir=data_list_dir)\n    else:\n        success = self.check_input_data(protocal=protocal, address=address, download=False,\n                                   input_data_root=input_data_root, data_list_dir=data_list_dir)\n        if download and not success:\n            success = _downloadfromserver(self, input_data_root, data_list_dir)\n\n    self.stage_refcase(input_data_root=input_data_root, data_list_dir=data_list_dir)\n    return success\n\ndef _downloadfromserver(case, input_data_root, data_list_dir):\n    # needs to be downloaded\n    success = False\n    protocal = 'svn'\n    inputdata = Inputdata()\n    while not success and protocal is not None:\n        protocal, address = inputdata.get_next_server()\n        logger.info(\"Checking server {} with protocal {}\".format(address, protocal))\n        success = case.check_input_data(protocal=protocal, address=address, download=True,\n                                        input_data_root=input_data_root, data_list_dir=data_list_dir)\n    return success\n\ndef stage_refcase(self, input_data_root=None, data_list_dir=None):\n    get_refcase  = self.get_value(\"GET_REFCASE\")\n    run_type     = self.get_value(\"RUN_TYPE\")\n    continue_run = self.get_value(\"CONTINUE_RUN\")\n\n    # We do not fully populate the inputdata directory on every\n    # machine and do not expect every user to download the 3TB+ of\n    # data in our inputdata repository. This code checks for the\n    # existence of inputdata in the local inputdata directory and\n    # attempts to download data from the server if it's needed and\n    # missing.\n    if get_refcase and run_type != \"startup\" and not continue_run:\n        din_loc_root = self.get_value(\"DIN_LOC_ROOT\")\n        run_refdate  = self.get_value(\"RUN_REFDATE\")\n        run_refcase  = self.get_value(\"RUN_REFCASE\")\n        run_refdir   = self.get_value(\"RUN_REFDIR\")\n        rundir       = self.get_value(\"RUNDIR\")\n\n        refdir = os.path.join(din_loc_root, run_refdir, run_refcase, run_refdate)\n        if not os.path.isdir(refdir):\n            logger.warning(\"Refcase not found in {}, will attempt to download from inputdata\".format(refdir))\n            with open(os.path.join(\"Buildconf\",\"refcase.input_data_list\"),\"w\") as fd:\n                fd.write(\"refdir = {}{}\".format(refdir, os.sep))\n            if input_data_root is None:\n                input_data_root = din_loc_root\n            if data_list_dir is None:\n                data_list_dir = \"Buildconf\"\n            success = _downloadfromserver(self, input_data_root=input_data_root, data_list_dir=data_list_dir)\n            expect(success, \"Could not download refcase from any server\")\n\n        logger.info(\" - Prestaging REFCASE ({}) to {}\".format(refdir, rundir))\n\n        # prestage the reference case's files.\n\n        if (not os.path.exists(rundir)):\n            logger.debug(\"Creating run directory: {}\".format(rundir))\n            os.makedirs(rundir)\n\n        # copy the refcases' rpointer files to the run directory\n        for rpointerfile in glob.iglob(os.path.join(\"{}\",\"*rpointer*\").format(refdir)):\n            logger.info(\"Copy rpointer {}\".format(rpointerfile))\n            safe_copy(rpointerfile, rundir)\n\n        # link everything else\n\n        for rcfile in glob.iglob(os.path.join(refdir,\"*\")):\n            rcbaseline = os.path.basename(rcfile)\n            if not os.path.exists(\"{}/{}\".format(rundir, rcbaseline)):\n                logger.info(\"Staging file {}\".format(rcfile))\n                os.symlink(rcfile, \"{}/{}\".format(rundir, rcbaseline))\n        # Backward compatibility, some old refcases have cam2 in the name\n        # link to local cam file.\n        for cam2file in  glob.iglob(os.path.join(\"{}\",\"*.cam2.*\").format(rundir)):\n            camfile = cam2file.replace(\"cam2\", \"cam\")\n            os.symlink(cam2file, camfile)\n\n    return True\n\ndef check_input_data(case, protocal=\"svn\", address=None, input_data_root=None, data_list_dir=\"Buildconf\", download=False):\n    \"\"\"\n    Return True if no files missing\n    \"\"\"\n    case.load_env(reset=True)\n    # Fill in defaults as needed\n    input_data_root = case.get_value(\"DIN_LOC_ROOT\") if input_data_root is None else input_data_root\n\n    expect(os.path.isdir(input_data_root), \"Invalid input_data_root directory: '{}'\".format(input_data_root))\n    expect(os.path.isdir(data_list_dir), \"Invalid data_list_dir directory: '{}'\".format(data_list_dir))\n\n    data_list_files = find_files(data_list_dir, \"*.input_data_list\")\n    expect(data_list_files, \"No .input_data_list files found in dir '{}'\".format(data_list_dir))\n\n    no_files_missing = True\n\n    if download:\n        if protocal not in vars(CIME.Servers):\n            logger.warning(\"Client protocal {} not enabled\".format(protocal))\n            return False\n\n        if protocal == \"svn\":\n            server = CIME.Servers.SVN(address)\n        elif protocal == \"gftp\":\n            server = CIME.Servers.GridFTP(address)\n        elif protocal == \"ftp\":\n            server = CIME.Servers.FTP(address)\n        elif protocal == \"wget\":\n            server = CIME.Servers.WGET(address)\n        else:\n            expect(False, \"Unsupported inputdata protocal: {}\".format(protocal))\n\n\n\n    for data_list_file in data_list_files:\n        logging.info(\"Loading input file list: '{}'\".format(data_list_file))\n        with open(data_list_file, \"r\") as fd:\n            lines = fd.readlines()\n\n        for line in lines:\n            line = line.strip()\n            if (line and not line.startswith(\"#\")):\n                tokens = line.split('=')\n                description, full_path = tokens[0].strip(), tokens[1].strip()\n                if(full_path):\n                    # expand xml variables\n                    full_path = case.get_resolved_value(full_path)\n                    rel_path  = full_path.replace(input_data_root, \"\")\n                    model = os.path.basename(data_list_file).split('.')[0]\n\n                    if (\"/\" in rel_path and rel_path == full_path):\n                        # User pointing to a file outside of input_data_root, we cannot determine\n                        # rel_path, and so cannot download the file. If it already exists, we can\n                        # proceed\n                        if not os.path.exists(full_path):\n                            logging.warning(\"  Model {} missing file {} = '{}'\".format(model, description, full_path))\n                            if download:\n                                logging.warning(\"    Cannot download file since it lives outside of the input_data_root '{}'\".format(input_data_root))\n                            no_files_missing = False\n                        else:\n                            logging.debug(\"  Found input file: '{}'\".format(full_path))\n\n                    else:\n                        # There are some special values of rel_path that\n                        # we need to ignore - some of the component models\n                        # set things like 'NULL' or 'same_as_TS' -\n                        # basically if rel_path does not contain '/' (a\n                        # directory tree) you can assume it's a special\n                        # value and ignore it (perhaps with a warning)\n                        if (\"/\" in rel_path and not os.path.exists(full_path)):\n                            logging.warning(\"  Model {} missing file {} = '{}'\".format(model, description, full_path))\n\n                            if (download):\n                                success = _download_if_in_repo(server, input_data_root, rel_path.strip(os.sep),\n                                                               isdirectory=rel_path.endswith(os.sep))\n                                if not success:\n                                    no_files_missing = False\n                        else:\n                            logging.debug(\"  Already had input file: '{}'\".format(full_path))\n\n                else:\n                    model = os.path.basename(data_list_file).split('.')[0]\n                    logging.warning(\"Model {} no file specified for {}\".format(model, description))\n\n    return no_files_missing\n", "path": "scripts/lib/CIME/case/check_input_data.py"}], "after_files": [{"content": "\"\"\"\nAPI for checking input for testcase\n\"\"\"\nfrom CIME.XML.standard_module_setup import *\nfrom CIME.utils import SharedArea, find_files, safe_copy, expect\nfrom CIME.XML.inputdata import Inputdata\nimport CIME.Servers\n\nimport glob\n\nlogger = logging.getLogger(__name__)\n\ndef _download_if_in_repo(server, input_data_root, rel_path, isdirectory=False):\n    \"\"\"\n    Return True if successfully downloaded\n    \"\"\"\n    if not server.fileexists(rel_path):\n        return False\n\n    full_path = os.path.join(input_data_root, rel_path)\n    logging.info(\"Trying to download file: '{}' to path '{}'\".format(rel_path, full_path))\n    # Make sure local path exists, create if it does not\n    if isdirectory or full_path.endswith(os.sep):\n        if not os.path.exists(full_path):\n            logger.info(\"Creating directory {}\".format(full_path))\n            os.makedirs(full_path)\n        isdirectory = True\n    elif not os.path.exists(os.path.dirname(full_path)):\n        os.makedirs(os.path.dirname(full_path))\n\n    # Use umask to make sure files are group read/writable. As long as parent directories\n    # have +s, then everything should work.\n    with SharedArea():\n        if isdirectory:\n            return server.getdirectory(rel_path, full_path)\n        else:\n            return server.getfile(rel_path, full_path)\n\n###############################################################################\ndef check_all_input_data(self, protocal=None, address=None, input_data_root=None, data_list_dir=\"Buildconf\", download=True):\n###############################################################################\n    success = False\n    if protocal is not None and address is not None:\n        success = self.check_input_data(protocal=protocal, address=address, download=download,\n                                   input_data_root=input_data_root, data_list_dir=data_list_dir)\n    else:\n        success = self.check_input_data(protocal=protocal, address=address, download=False,\n                                   input_data_root=input_data_root, data_list_dir=data_list_dir)\n        if download and not success:\n            success = _downloadfromserver(self, input_data_root, data_list_dir)\n\n    self.stage_refcase(input_data_root=input_data_root, data_list_dir=data_list_dir)\n    return success\n\ndef _downloadfromserver(case, input_data_root, data_list_dir):\n    # needs to be downloaded\n    success = False\n    protocal = 'svn'\n    inputdata = Inputdata()\n    while not success and protocal is not None:\n        protocal, address = inputdata.get_next_server()\n        logger.info(\"Checking server {} with protocal {}\".format(address, protocal))\n        success = case.check_input_data(protocal=protocal, address=address, download=True,\n                                        input_data_root=input_data_root, data_list_dir=data_list_dir)\n    return success\n\ndef stage_refcase(self, input_data_root=None, data_list_dir=None):\n    get_refcase  = self.get_value(\"GET_REFCASE\")\n    run_type     = self.get_value(\"RUN_TYPE\")\n    continue_run = self.get_value(\"CONTINUE_RUN\")\n\n    # We do not fully populate the inputdata directory on every\n    # machine and do not expect every user to download the 3TB+ of\n    # data in our inputdata repository. This code checks for the\n    # existence of inputdata in the local inputdata directory and\n    # attempts to download data from the server if it's needed and\n    # missing.\n    if get_refcase and run_type != \"startup\" and not continue_run:\n        din_loc_root = self.get_value(\"DIN_LOC_ROOT\")\n        run_refdate  = self.get_value(\"RUN_REFDATE\")\n        run_refcase  = self.get_value(\"RUN_REFCASE\")\n        run_refdir   = self.get_value(\"RUN_REFDIR\")\n        rundir       = self.get_value(\"RUNDIR\")\n\n        refdir = os.path.join(din_loc_root, run_refdir, run_refcase, run_refdate)\n        if not os.path.isdir(refdir):\n            logger.warning(\"Refcase not found in {}, will attempt to download from inputdata\".format(refdir))\n            with open(os.path.join(\"Buildconf\",\"refcase.input_data_list\"),\"w\") as fd:\n                fd.write(\"refdir = {}{}\".format(refdir, os.sep))\n            if input_data_root is None:\n                input_data_root = din_loc_root\n            if data_list_dir is None:\n                data_list_dir = \"Buildconf\"\n            success = _downloadfromserver(self, input_data_root=input_data_root, data_list_dir=data_list_dir)\n            expect(success, \"Could not download refcase from any server\")\n\n        logger.info(\" - Prestaging REFCASE ({}) to {}\".format(refdir, rundir))\n\n        # prestage the reference case's files.\n\n        if (not os.path.exists(rundir)):\n            logger.debug(\"Creating run directory: {}\".format(rundir))\n            os.makedirs(rundir)\n\n        # copy the refcases' rpointer files to the run directory\n        for rpointerfile in glob.iglob(os.path.join(\"{}\",\"*rpointer*\").format(refdir)):\n            logger.info(\"Copy rpointer {}\".format(rpointerfile))\n            safe_copy(rpointerfile, rundir)\n\n        # link everything else\n\n        for rcfile in glob.iglob(os.path.join(refdir,\"*\")):\n            rcbaseline = os.path.basename(rcfile)\n            if not os.path.exists(\"{}/{}\".format(rundir, rcbaseline)):\n                logger.info(\"Staging file {}\".format(rcfile))\n                os.symlink(rcfile, \"{}/{}\".format(rundir, rcbaseline))\n        # Backward compatibility, some old refcases have cam2 in the name\n        # link to local cam file.\n        for cam2file in  glob.iglob(os.path.join(\"{}\",\"*.cam2.*\").format(rundir)):\n            camfile = cam2file.replace(\"cam2\", \"cam\")\n            os.symlink(cam2file, camfile)\n\n    return True\n\ndef check_input_data(case, protocal=\"svn\", address=None, input_data_root=None, data_list_dir=\"Buildconf\", download=False):\n    \"\"\"\n    Return True if no files missing\n    \"\"\"\n    case.load_env(reset=True)\n    # Fill in defaults as needed\n    input_data_root = case.get_value(\"DIN_LOC_ROOT\") if input_data_root is None else input_data_root\n\n    expect(os.path.isdir(input_data_root), \"Invalid input_data_root directory: '{}'\".format(input_data_root))\n    expect(os.path.isdir(data_list_dir), \"Invalid data_list_dir directory: '{}'\".format(data_list_dir))\n\n    data_list_files = find_files(data_list_dir, \"*.input_data_list\")\n    expect(data_list_files, \"No .input_data_list files found in dir '{}'\".format(data_list_dir))\n\n    no_files_missing = True\n\n    if download:\n        if protocal not in vars(CIME.Servers):\n            logger.warning(\"Client protocal {} not enabled\".format(protocal))\n            return False\n\n        if protocal == \"svn\":\n            server = CIME.Servers.SVN(address)\n        elif protocal == \"gftp\":\n            server = CIME.Servers.GridFTP(address)\n        elif protocal == \"ftp\":\n            server = CIME.Servers.FTP(address)\n        elif protocal == \"wget\":\n            server = CIME.Servers.WGET(address)\n        else:\n            expect(False, \"Unsupported inputdata protocal: {}\".format(protocal))\n\n\n\n    for data_list_file in data_list_files:\n        logging.info(\"Loading input file list: '{}'\".format(data_list_file))\n        with open(data_list_file, \"r\") as fd:\n            lines = fd.readlines()\n\n        for line in lines:\n            line = line.strip()\n            if (line and not line.startswith(\"#\")):\n                tokens = line.split('=')\n                description, full_path = tokens[0].strip(), tokens[1].strip()\n                if(full_path):\n                    # expand xml variables\n                    full_path = case.get_resolved_value(full_path)\n                    rel_path  = full_path.replace(input_data_root, \"\")\n                    model = os.path.basename(data_list_file).split('.')[0]\n\n                    if (\"/\" in rel_path and rel_path == full_path):\n                        # User pointing to a file outside of input_data_root, we cannot determine\n                        # rel_path, and so cannot download the file. If it already exists, we can\n                        # proceed\n                        if not os.path.exists(full_path):\n                            logging.warning(\"Model {} missing file {} = '{}'\".format(model, description, full_path))\n                            if download:\n                                logging.warning(\"    Cannot download file since it lives outside of the input_data_root '{}'\".format(input_data_root))\n                            no_files_missing = False\n                        else:\n                            logging.debug(\"  Found input file: '{}'\".format(full_path))\n\n                    else:\n                        # There are some special values of rel_path that\n                        # we need to ignore - some of the component models\n                        # set things like 'NULL' or 'same_as_TS' -\n                        # basically if rel_path does not contain '/' (a\n                        # directory tree) you can assume it's a special\n                        # value and ignore it (perhaps with a warning)\n                        if (\"/\" in rel_path and not os.path.exists(full_path)):\n                            logging.warning(\"  Model {} missing file {} = '{}'\".format(model, description, full_path))\n                            no_files_missing = False\n\n                            if (download):\n                                no_files_missing = _download_if_in_repo(server, input_data_root, rel_path.strip(os.sep),\n                                                               isdirectory=rel_path.endswith(os.sep))\n                        else:\n                            logging.debug(\"  Already had input file: '{}'\".format(full_path))\n\n                else:\n                    model = os.path.basename(data_list_file).split('.')[0]\n                    logging.warning(\"Model {} no file specified for {}\".format(model, description))\n\n    return no_files_missing\n", "path": "scripts/lib/CIME/case/check_input_data.py"}]}