We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
multi driver restart failing in cime 
ERROR: CONTINUE_RUN is true but this case does not appear to have restart files staged in /glade/scratch/joemci/f.c63074.FXHIST.f09_f09_mg17.Heelis_Multi_NoMag_LowKp.001/run rpointer.cpl


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `CIME/case/case_submit.py`
Content:
```
1 #!/usr/bin/env python3
2 
3 """
4 case.submit - Submit a cesm workflow to the queueing system or run it
5 if there is no queueing system.  A cesm workflow may include multiple
6 jobs.
7 submit, check_case and check_da_settings are members of class Case in file case.py
8 """
9 import configparser
10 from CIME.XML.standard_module_setup import *
11 from CIME.utils import expect, run_and_log_case_status, CIMEError
12 from CIME.locked_files import unlock_file, lock_file
13 from CIME.test_status import *
14 
15 import socket
16 
17 logger = logging.getLogger(__name__)
18 
19 
20 def _build_prereq_str(case, prev_job_ids):
21     delimiter = case.get_value("depend_separator")
22     prereq_str = ""
23     for job_id in prev_job_ids.values():
24         prereq_str += str(job_id) + delimiter
25     return prereq_str[:-1]
26 
27 
28 def _submit(
29     case,
30     job=None,
31     no_batch=False,
32     prereq=None,
33     allow_fail=False,
34     resubmit=False,
35     resubmit_immediate=False,
36     skip_pnl=False,
37     mail_user=None,
38     mail_type=None,
39     batch_args=None,
40     workflow=True,
41     chksum=False,
42 ):
43     if job is None:
44         job = case.get_first_job()
45     caseroot = case.get_value("CASEROOT")
46     # Check mediator
47     hasMediator = True
48     comp_classes = case.get_values("COMP_CLASSES")
49     if "CPL" not in comp_classes:
50         hasMediator = False
51 
52     # Check if CONTINUE_RUN value makes sense
53     # if submitted with a prereq don't do this check
54     if (
55         job != "case.test"
56         and case.get_value("CONTINUE_RUN")
57         and hasMediator
58         and not prereq
59     ):
60         rundir = case.get_value("RUNDIR")
61         expect(
62             os.path.isdir(rundir),
63             "CONTINUE_RUN is true but RUNDIR {} does not exist".format(rundir),
64         )
65         # only checks for the first instance in a multidriver case
66         if case.get_value("COMP_INTERFACE") == "nuopc":
67             rpointer = "rpointer.cpl"
68         elif case.get_value("MULTI_DRIVER"):
69             rpointer = "rpointer.drv_0001"
70         else:
71             rpointer = "rpointer.drv"
72         expect(
73             os.path.exists(os.path.join(rundir, rpointer)),
74             "CONTINUE_RUN is true but this case does not appear to have restart files staged in {} {}".format(
75                 rundir, rpointer
76             ),
77         )
78         # Finally we open the rpointer file and check that it's correct
79         casename = case.get_value("CASE")
80         with open(os.path.join(rundir, rpointer), "r") as fd:
81             ncfile = fd.readline().strip()
82             expect(
83                 ncfile.startswith(casename)
84                 and os.path.exists(os.path.join(rundir, ncfile)),
85                 "File {ncfile} not present or does not match case {casename}".format(
86                     ncfile=os.path.join(rundir, ncfile), casename=casename
87                 ),
88             )
89 
90     # if case.submit is called with the no_batch flag then we assume that this
91     # flag will stay in effect for the duration of the RESUBMITs
92     env_batch = case.get_env("batch")
93     external_workflow = case.get_value("EXTERNAL_WORKFLOW")
94     if env_batch.get_batch_system_type() == "none" or resubmit and external_workflow:
95         no_batch = True
96 
97     if no_batch:
98         batch_system = "none"
99     else:
100         batch_system = env_batch.get_batch_system_type()
101     unlock_file(os.path.basename(env_batch.filename), caseroot=caseroot)
102     case.set_value("BATCH_SYSTEM", batch_system)
103 
104     env_batch_has_changed = False
105     if not external_workflow:
106         try:
107             case.check_lockedfile(
108                 os.path.basename(env_batch.filename), caseroot=caseroot
109             )
110         except:
111             env_batch_has_changed = True
112 
113     if batch_system != "none" and env_batch_has_changed and not external_workflow:
114         # May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc)
115         logger.warning(
116             """
117 env_batch.xml appears to have changed, regenerating batch scripts
118 manual edits to these file will be lost!
119 """
120         )
121         env_batch.make_all_batch_files(case)
122     case.flush()
123     lock_file(os.path.basename(env_batch.filename), caseroot=caseroot)
124 
125     if resubmit:
126         # This is a resubmission, do not reinitialize test values
127         if job == "case.test":
128             case.set_value("IS_FIRST_RUN", False)
129 
130         resub = case.get_value("RESUBMIT")
131         logger.info("Submitting job '{}', resubmit={:d}".format(job, resub))
132         case.set_value("RESUBMIT", resub - 1)
133         if case.get_value("RESUBMIT_SETS_CONTINUE_RUN"):
134             case.set_value("CONTINUE_RUN", True)
135 
136     else:
137         if job == "case.test":
138             case.set_value("IS_FIRST_RUN", True)
139 
140         if no_batch:
141             batch_system = "none"
142         else:
143             batch_system = env_batch.get_batch_system_type()
144 
145         case.set_value("BATCH_SYSTEM", batch_system)
146 
147         env_batch_has_changed = False
148         try:
149             case.check_lockedfile(os.path.basename(env_batch.filename))
150         except CIMEError:
151             env_batch_has_changed = True
152 
153         if env_batch.get_batch_system_type() != "none" and env_batch_has_changed:
154             # May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc)
155             logger.warning(
156                 """
157 env_batch.xml appears to have changed, regenerating batch scripts
158 manual edits to these file will be lost!
159 """
160             )
161             env_batch.make_all_batch_files(case)
162 
163         unlock_file(os.path.basename(env_batch.filename), caseroot=caseroot)
164         lock_file(os.path.basename(env_batch.filename), caseroot=caseroot)
165 
166         case.check_case(skip_pnl=skip_pnl, chksum=chksum)
167         if job == case.get_primary_job():
168             case.check_DA_settings()
169             if case.get_value("MACH") == "mira":
170                 with open(".original_host", "w") as fd:
171                     fd.write(socket.gethostname())
172 
173     # Load Modules
174     case.load_env()
175 
176     case.flush()
177 
178     logger.warning("submit_jobs {}".format(job))
179     job_ids = case.submit_jobs(
180         no_batch=no_batch,
181         job=job,
182         prereq=prereq,
183         skip_pnl=skip_pnl,
184         resubmit_immediate=resubmit_immediate,
185         allow_fail=allow_fail,
186         mail_user=mail_user,
187         mail_type=mail_type,
188         batch_args=batch_args,
189         workflow=workflow,
190     )
191 
192     xml_jobids = []
193     for jobname, jobid in job_ids.items():
194         logger.info("Submitted job {} with id {}".format(jobname, jobid))
195         if jobid:
196             xml_jobids.append("{}:{}".format(jobname, jobid))
197 
198     xml_jobid_text = ", ".join(xml_jobids)
199     if xml_jobid_text:
200         case.set_value("JOB_IDS", xml_jobid_text)
201 
202     return xml_jobid_text
203 
204 
205 def submit(
206     self,
207     job=None,
208     no_batch=False,
209     prereq=None,
210     allow_fail=False,
211     resubmit=False,
212     resubmit_immediate=False,
213     skip_pnl=False,
214     mail_user=None,
215     mail_type=None,
216     batch_args=None,
217     workflow=True,
218     chksum=False,
219 ):
220     if resubmit_immediate and self.get_value("MACH") in ["mira", "cetus"]:
221         logger.warning(
222             "resubmit_immediate does not work on Mira/Cetus, submitting normally"
223         )
224         resubmit_immediate = False
225 
226     caseroot = self.get_value("CASEROOT")
227     if self.get_value("TEST"):
228         casebaseid = self.get_value("CASEBASEID")
229         # This should take care of the race condition where the submitted job
230         # begins immediately and tries to set RUN phase. We proactively assume
231         # a passed SUBMIT phase. If this state is already PASS, don't set it again
232         # because then we'll lose RUN phase info if it's there. This info is important
233         # for system_tests_common to know if it needs to reinitialize the test or not.
234         with TestStatus(test_dir=caseroot, test_name=casebaseid) as ts:
235             phase_status = ts.get_status(SUBMIT_PHASE)
236             if phase_status != TEST_PASS_STATUS:
237                 ts.set_status(SUBMIT_PHASE, TEST_PASS_STATUS)
238 
239     # If this is a resubmit check the hidden file .submit_options for
240     # any submit options used on the original submit and use them again
241     submit_options = os.path.join(caseroot, ".submit_options")
242     if resubmit and os.path.exists(submit_options):
243         config = configparser.RawConfigParser()
244         config.read(submit_options)
245         if not skip_pnl and config.has_option("SubmitOptions", "skip_pnl"):
246             skip_pnl = config.getboolean("SubmitOptions", "skip_pnl")
247         if mail_user is None and config.has_option("SubmitOptions", "mail_user"):
248             mail_user = config.get("SubmitOptions", "mail_user")
249         if mail_type is None and config.has_option("SubmitOptions", "mail_type"):
250             mail_type = str(config.get("SubmitOptions", "mail_type")).split(",")
251         if batch_args is None and config.has_option("SubmitOptions", "batch_args"):
252             batch_args = config.get("SubmitOptions", "batch_args")
253 
254     is_batch = self.get_value("BATCH_SYSTEM") is not None
255 
256     try:
257         functor = lambda: _submit(
258             self,
259             job=job,
260             no_batch=no_batch,
261             prereq=prereq,
262             allow_fail=allow_fail,
263             resubmit=resubmit,
264             resubmit_immediate=resubmit_immediate,
265             skip_pnl=skip_pnl,
266             mail_user=mail_user,
267             mail_type=mail_type,
268             batch_args=batch_args,
269             workflow=workflow,
270             chksum=chksum,
271         )
272         run_and_log_case_status(
273             functor,
274             "case.submit",
275             caseroot=caseroot,
276             custom_success_msg_functor=lambda x: x.split(":")[-1],
277             is_batch=is_batch,
278         )
279     except BaseException:  # Want to catch KeyboardInterrupt too
280         # If something failed in the batch system, make sure to mark
281         # the test as failed if we are running a test.
282         if self.get_value("TEST"):
283             with TestStatus(test_dir=caseroot, test_name=casebaseid) as ts:
284                 ts.set_status(SUBMIT_PHASE, TEST_FAIL_STATUS)
285 
286         raise
287 
288 
289 def check_case(self, skip_pnl=False, chksum=False):
290     self.check_lockedfiles()
291     if not skip_pnl:
292         self.create_namelists()  # Must be called before check_all_input_data
293     logger.info("Checking that inputdata is available as part of case submission")
294     self.check_all_input_data(chksum=chksum)
295 
296     if self.get_value("COMP_WAV") == "ww":
297         # the ww3 buildnml has dependencies on inputdata so we must run it again
298         self.create_namelists(component="WAV")
299 
300     expect(
301         self.get_value("BUILD_COMPLETE"),
302         "Build complete is " "not True please rebuild the model by calling case.build",
303     )
304     logger.info("Check case OK")
305 
306 
307 def check_DA_settings(self):
308     script = self.get_value("DATA_ASSIMILATION_SCRIPT")
309     cycles = self.get_value("DATA_ASSIMILATION_CYCLES")
310     if len(script) > 0 and os.path.isfile(script) and cycles > 0:
311         logger.info(
312             "Data Assimilation enabled using script {} with {:d} cycles".format(
313                 script, cycles
314             )
315         )
316
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/CIME/case/case_submit.py b/CIME/case/case_submit.py
--- a/CIME/case/case_submit.py
+++ b/CIME/case/case_submit.py
@@ -65,10 +65,10 @@
         # only checks for the first instance in a multidriver case
         if case.get_value("COMP_INTERFACE") == "nuopc":
             rpointer = "rpointer.cpl"
-        elif case.get_value("MULTI_DRIVER"):
-            rpointer = "rpointer.drv_0001"
         else:
             rpointer = "rpointer.drv"
+        if case.get_value("MULTI_DRIVER"):
+            rpointer = rpointer + "_0001"
         expect(
             os.path.exists(os.path.join(rundir, rpointer)),
             "CONTINUE_RUN is true but this case does not appear to have restart files staged in {} {}".format(

{"golden_diff": "diff --git a/CIME/case/case_submit.py b/CIME/case/case_submit.py\n--- a/CIME/case/case_submit.py\n+++ b/CIME/case/case_submit.py\n@@ -65,10 +65,10 @@\n         # only checks for the first instance in a multidriver case\n         if case.get_value(\"COMP_INTERFACE\") == \"nuopc\":\n             rpointer = \"rpointer.cpl\"\n-        elif case.get_value(\"MULTI_DRIVER\"):\n-            rpointer = \"rpointer.drv_0001\"\n         else:\n             rpointer = \"rpointer.drv\"\n+        if case.get_value(\"MULTI_DRIVER\"):\n+            rpointer = rpointer + \"_0001\"\n         expect(\n             os.path.exists(os.path.join(rundir, rpointer)),\n             \"CONTINUE_RUN is true but this case does not appear to have restart files staged in {} {}\".format(\n", "issue": "multi driver restart failing in cime \nERROR: CONTINUE_RUN is true but this case does not appear to have restart files staged in /glade/scratch/joemci/f.c63074.FXHIST.f09_f09_mg17.Heelis_Multi_NoMag_LowKp.001/run rpointer.cpl\r\n\n", "before_files": [{"content": "#!/usr/bin/env python3\n\n\"\"\"\ncase.submit - Submit a cesm workflow to the queueing system or run it\nif there is no queueing system.  A cesm workflow may include multiple\njobs.\nsubmit, check_case and check_da_settings are members of class Case in file case.py\n\"\"\"\nimport configparser\nfrom CIME.XML.standard_module_setup import *\nfrom CIME.utils import expect, run_and_log_case_status, CIMEError\nfrom CIME.locked_files import unlock_file, lock_file\nfrom CIME.test_status import *\n\nimport socket\n\nlogger = logging.getLogger(__name__)\n\n\ndef _build_prereq_str(case, prev_job_ids):\n    delimiter = case.get_value(\"depend_separator\")\n    prereq_str = \"\"\n    for job_id in prev_job_ids.values():\n        prereq_str += str(job_id) + delimiter\n    return prereq_str[:-1]\n\n\ndef _submit(\n    case,\n    job=None,\n    no_batch=False,\n    prereq=None,\n    allow_fail=False,\n    resubmit=False,\n    resubmit_immediate=False,\n    skip_pnl=False,\n    mail_user=None,\n    mail_type=None,\n    batch_args=None,\n    workflow=True,\n    chksum=False,\n):\n    if job is None:\n        job = case.get_first_job()\n    caseroot = case.get_value(\"CASEROOT\")\n    # Check mediator\n    hasMediator = True\n    comp_classes = case.get_values(\"COMP_CLASSES\")\n    if \"CPL\" not in comp_classes:\n        hasMediator = False\n\n    # Check if CONTINUE_RUN value makes sense\n    # if submitted with a prereq don't do this check\n    if (\n        job != \"case.test\"\n        and case.get_value(\"CONTINUE_RUN\")\n        and hasMediator\n        and not prereq\n    ):\n        rundir = case.get_value(\"RUNDIR\")\n        expect(\n            os.path.isdir(rundir),\n            \"CONTINUE_RUN is true but RUNDIR {} does not exist\".format(rundir),\n        )\n        # only checks for the first instance in a multidriver case\n        if case.get_value(\"COMP_INTERFACE\") == \"nuopc\":\n            rpointer = \"rpointer.cpl\"\n        elif case.get_value(\"MULTI_DRIVER\"):\n            rpointer = \"rpointer.drv_0001\"\n        else:\n            rpointer = \"rpointer.drv\"\n        expect(\n            os.path.exists(os.path.join(rundir, rpointer)),\n            \"CONTINUE_RUN is true but this case does not appear to have restart files staged in {} {}\".format(\n                rundir, rpointer\n            ),\n        )\n        # Finally we open the rpointer file and check that it's correct\n        casename = case.get_value(\"CASE\")\n        with open(os.path.join(rundir, rpointer), \"r\") as fd:\n            ncfile = fd.readline().strip()\n            expect(\n                ncfile.startswith(casename)\n                and os.path.exists(os.path.join(rundir, ncfile)),\n                \"File {ncfile} not present or does not match case {casename}\".format(\n                    ncfile=os.path.join(rundir, ncfile), casename=casename\n                ),\n            )\n\n    # if case.submit is called with the no_batch flag then we assume that this\n    # flag will stay in effect for the duration of the RESUBMITs\n    env_batch = case.get_env(\"batch\")\n    external_workflow = case.get_value(\"EXTERNAL_WORKFLOW\")\n    if env_batch.get_batch_system_type() == \"none\" or resubmit and external_workflow:\n        no_batch = True\n\n    if no_batch:\n        batch_system = \"none\"\n    else:\n        batch_system = env_batch.get_batch_system_type()\n    unlock_file(os.path.basename(env_batch.filename), caseroot=caseroot)\n    case.set_value(\"BATCH_SYSTEM\", batch_system)\n\n    env_batch_has_changed = False\n    if not external_workflow:\n        try:\n            case.check_lockedfile(\n                os.path.basename(env_batch.filename), caseroot=caseroot\n            )\n        except:\n            env_batch_has_changed = True\n\n    if batch_system != \"none\" and env_batch_has_changed and not external_workflow:\n        # May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc)\n        logger.warning(\n            \"\"\"\nenv_batch.xml appears to have changed, regenerating batch scripts\nmanual edits to these file will be lost!\n\"\"\"\n        )\n        env_batch.make_all_batch_files(case)\n    case.flush()\n    lock_file(os.path.basename(env_batch.filename), caseroot=caseroot)\n\n    if resubmit:\n        # This is a resubmission, do not reinitialize test values\n        if job == \"case.test\":\n            case.set_value(\"IS_FIRST_RUN\", False)\n\n        resub = case.get_value(\"RESUBMIT\")\n        logger.info(\"Submitting job '{}', resubmit={:d}\".format(job, resub))\n        case.set_value(\"RESUBMIT\", resub - 1)\n        if case.get_value(\"RESUBMIT_SETS_CONTINUE_RUN\"):\n            case.set_value(\"CONTINUE_RUN\", True)\n\n    else:\n        if job == \"case.test\":\n            case.set_value(\"IS_FIRST_RUN\", True)\n\n        if no_batch:\n            batch_system = \"none\"\n        else:\n            batch_system = env_batch.get_batch_system_type()\n\n        case.set_value(\"BATCH_SYSTEM\", batch_system)\n\n        env_batch_has_changed = False\n        try:\n            case.check_lockedfile(os.path.basename(env_batch.filename))\n        except CIMEError:\n            env_batch_has_changed = True\n\n        if env_batch.get_batch_system_type() != \"none\" and env_batch_has_changed:\n            # May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc)\n            logger.warning(\n                \"\"\"\nenv_batch.xml appears to have changed, regenerating batch scripts\nmanual edits to these file will be lost!\n\"\"\"\n            )\n            env_batch.make_all_batch_files(case)\n\n        unlock_file(os.path.basename(env_batch.filename), caseroot=caseroot)\n        lock_file(os.path.basename(env_batch.filename), caseroot=caseroot)\n\n        case.check_case(skip_pnl=skip_pnl, chksum=chksum)\n        if job == case.get_primary_job():\n            case.check_DA_settings()\n            if case.get_value(\"MACH\") == \"mira\":\n                with open(\".original_host\", \"w\") as fd:\n                    fd.write(socket.gethostname())\n\n    # Load Modules\n    case.load_env()\n\n    case.flush()\n\n    logger.warning(\"submit_jobs {}\".format(job))\n    job_ids = case.submit_jobs(\n        no_batch=no_batch,\n        job=job,\n        prereq=prereq,\n        skip_pnl=skip_pnl,\n        resubmit_immediate=resubmit_immediate,\n        allow_fail=allow_fail,\n        mail_user=mail_user,\n        mail_type=mail_type,\n        batch_args=batch_args,\n        workflow=workflow,\n    )\n\n    xml_jobids = []\n    for jobname, jobid in job_ids.items():\n        logger.info(\"Submitted job {} with id {}\".format(jobname, jobid))\n        if jobid:\n            xml_jobids.append(\"{}:{}\".format(jobname, jobid))\n\n    xml_jobid_text = \", \".join(xml_jobids)\n    if xml_jobid_text:\n        case.set_value(\"JOB_IDS\", xml_jobid_text)\n\n    return xml_jobid_text\n\n\ndef submit(\n    self,\n    job=None,\n    no_batch=False,\n    prereq=None,\n    allow_fail=False,\n    resubmit=False,\n    resubmit_immediate=False,\n    skip_pnl=False,\n    mail_user=None,\n    mail_type=None,\n    batch_args=None,\n    workflow=True,\n    chksum=False,\n):\n    if resubmit_immediate and self.get_value(\"MACH\") in [\"mira\", \"cetus\"]:\n        logger.warning(\n            \"resubmit_immediate does not work on Mira/Cetus, submitting normally\"\n        )\n        resubmit_immediate = False\n\n    caseroot = self.get_value(\"CASEROOT\")\n    if self.get_value(\"TEST\"):\n        casebaseid = self.get_value(\"CASEBASEID\")\n        # This should take care of the race condition where the submitted job\n        # begins immediately and tries to set RUN phase. We proactively assume\n        # a passed SUBMIT phase. If this state is already PASS, don't set it again\n        # because then we'll lose RUN phase info if it's there. This info is important\n        # for system_tests_common to know if it needs to reinitialize the test or not.\n        with TestStatus(test_dir=caseroot, test_name=casebaseid) as ts:\n            phase_status = ts.get_status(SUBMIT_PHASE)\n            if phase_status != TEST_PASS_STATUS:\n                ts.set_status(SUBMIT_PHASE, TEST_PASS_STATUS)\n\n    # If this is a resubmit check the hidden file .submit_options for\n    # any submit options used on the original submit and use them again\n    submit_options = os.path.join(caseroot, \".submit_options\")\n    if resubmit and os.path.exists(submit_options):\n        config = configparser.RawConfigParser()\n        config.read(submit_options)\n        if not skip_pnl and config.has_option(\"SubmitOptions\", \"skip_pnl\"):\n            skip_pnl = config.getboolean(\"SubmitOptions\", \"skip_pnl\")\n        if mail_user is None and config.has_option(\"SubmitOptions\", \"mail_user\"):\n            mail_user = config.get(\"SubmitOptions\", \"mail_user\")\n        if mail_type is None and config.has_option(\"SubmitOptions\", \"mail_type\"):\n            mail_type = str(config.get(\"SubmitOptions\", \"mail_type\")).split(\",\")\n        if batch_args is None and config.has_option(\"SubmitOptions\", \"batch_args\"):\n            batch_args = config.get(\"SubmitOptions\", \"batch_args\")\n\n    is_batch = self.get_value(\"BATCH_SYSTEM\") is not None\n\n    try:\n        functor = lambda: _submit(\n            self,\n            job=job,\n            no_batch=no_batch,\n            prereq=prereq,\n            allow_fail=allow_fail,\n            resubmit=resubmit,\n            resubmit_immediate=resubmit_immediate,\n            skip_pnl=skip_pnl,\n            mail_user=mail_user,\n            mail_type=mail_type,\n            batch_args=batch_args,\n            workflow=workflow,\n            chksum=chksum,\n        )\n        run_and_log_case_status(\n            functor,\n            \"case.submit\",\n            caseroot=caseroot,\n            custom_success_msg_functor=lambda x: x.split(\":\")[-1],\n            is_batch=is_batch,\n        )\n    except BaseException:  # Want to catch KeyboardInterrupt too\n        # If something failed in the batch system, make sure to mark\n        # the test as failed if we are running a test.\n        if self.get_value(\"TEST\"):\n            with TestStatus(test_dir=caseroot, test_name=casebaseid) as ts:\n                ts.set_status(SUBMIT_PHASE, TEST_FAIL_STATUS)\n\n        raise\n\n\ndef check_case(self, skip_pnl=False, chksum=False):\n    self.check_lockedfiles()\n    if not skip_pnl:\n        self.create_namelists()  # Must be called before check_all_input_data\n    logger.info(\"Checking that inputdata is available as part of case submission\")\n    self.check_all_input_data(chksum=chksum)\n\n    if self.get_value(\"COMP_WAV\") == \"ww\":\n        # the ww3 buildnml has dependencies on inputdata so we must run it again\n        self.create_namelists(component=\"WAV\")\n\n    expect(\n        self.get_value(\"BUILD_COMPLETE\"),\n        \"Build complete is \" \"not True please rebuild the model by calling case.build\",\n    )\n    logger.info(\"Check case OK\")\n\n\ndef check_DA_settings(self):\n    script = self.get_value(\"DATA_ASSIMILATION_SCRIPT\")\n    cycles = self.get_value(\"DATA_ASSIMILATION_CYCLES\")\n    if len(script) > 0 and os.path.isfile(script) and cycles > 0:\n        logger.info(\n            \"Data Assimilation enabled using script {} with {:d} cycles\".format(\n                script, cycles\n            )\n        )\n", "path": "CIME/case/case_submit.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n\n\"\"\"\ncase.submit - Submit a cesm workflow to the queueing system or run it\nif there is no queueing system.  A cesm workflow may include multiple\njobs.\nsubmit, check_case and check_da_settings are members of class Case in file case.py\n\"\"\"\nimport configparser\nfrom CIME.XML.standard_module_setup import *\nfrom CIME.utils import expect, run_and_log_case_status, CIMEError\nfrom CIME.locked_files import unlock_file, lock_file\nfrom CIME.test_status import *\n\nimport socket\n\nlogger = logging.getLogger(__name__)\n\n\ndef _build_prereq_str(case, prev_job_ids):\n    delimiter = case.get_value(\"depend_separator\")\n    prereq_str = \"\"\n    for job_id in prev_job_ids.values():\n        prereq_str += str(job_id) + delimiter\n    return prereq_str[:-1]\n\n\ndef _submit(\n    case,\n    job=None,\n    no_batch=False,\n    prereq=None,\n    allow_fail=False,\n    resubmit=False,\n    resubmit_immediate=False,\n    skip_pnl=False,\n    mail_user=None,\n    mail_type=None,\n    batch_args=None,\n    workflow=True,\n    chksum=False,\n):\n    if job is None:\n        job = case.get_first_job()\n    caseroot = case.get_value(\"CASEROOT\")\n    # Check mediator\n    hasMediator = True\n    comp_classes = case.get_values(\"COMP_CLASSES\")\n    if \"CPL\" not in comp_classes:\n        hasMediator = False\n\n    # Check if CONTINUE_RUN value makes sense\n    # if submitted with a prereq don't do this check\n    if (\n        job != \"case.test\"\n        and case.get_value(\"CONTINUE_RUN\")\n        and hasMediator\n        and not prereq\n    ):\n        rundir = case.get_value(\"RUNDIR\")\n        expect(\n            os.path.isdir(rundir),\n            \"CONTINUE_RUN is true but RUNDIR {} does not exist\".format(rundir),\n        )\n        # only checks for the first instance in a multidriver case\n        if case.get_value(\"COMP_INTERFACE\") == \"nuopc\":\n            rpointer = \"rpointer.cpl\"\n        else:\n            rpointer = \"rpointer.drv\"\n        if case.get_value(\"MULTI_DRIVER\"):\n            rpointer = rpointer + \"_0001\"\n        expect(\n            os.path.exists(os.path.join(rundir, rpointer)),\n            \"CONTINUE_RUN is true but this case does not appear to have restart files staged in {} {}\".format(\n                rundir, rpointer\n            ),\n        )\n        # Finally we open the rpointer file and check that it's correct\n        casename = case.get_value(\"CASE\")\n        with open(os.path.join(rundir, rpointer), \"r\") as fd:\n            ncfile = fd.readline().strip()\n            expect(\n                ncfile.startswith(casename)\n                and os.path.exists(os.path.join(rundir, ncfile)),\n                \"File {ncfile} not present or does not match case {casename}\".format(\n                    ncfile=os.path.join(rundir, ncfile), casename=casename\n                ),\n            )\n\n    # if case.submit is called with the no_batch flag then we assume that this\n    # flag will stay in effect for the duration of the RESUBMITs\n    env_batch = case.get_env(\"batch\")\n    external_workflow = case.get_value(\"EXTERNAL_WORKFLOW\")\n    if env_batch.get_batch_system_type() == \"none\" or resubmit and external_workflow:\n        no_batch = True\n\n    if no_batch:\n        batch_system = \"none\"\n    else:\n        batch_system = env_batch.get_batch_system_type()\n    unlock_file(os.path.basename(env_batch.filename), caseroot=caseroot)\n    case.set_value(\"BATCH_SYSTEM\", batch_system)\n\n    env_batch_has_changed = False\n    if not external_workflow:\n        try:\n            case.check_lockedfile(\n                os.path.basename(env_batch.filename), caseroot=caseroot\n            )\n        except:\n            env_batch_has_changed = True\n\n    if batch_system != \"none\" and env_batch_has_changed and not external_workflow:\n        # May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc)\n        logger.warning(\n            \"\"\"\nenv_batch.xml appears to have changed, regenerating batch scripts\nmanual edits to these file will be lost!\n\"\"\"\n        )\n        env_batch.make_all_batch_files(case)\n    case.flush()\n    lock_file(os.path.basename(env_batch.filename), caseroot=caseroot)\n\n    if resubmit:\n        # This is a resubmission, do not reinitialize test values\n        if job == \"case.test\":\n            case.set_value(\"IS_FIRST_RUN\", False)\n\n        resub = case.get_value(\"RESUBMIT\")\n        logger.info(\"Submitting job '{}', resubmit={:d}\".format(job, resub))\n        case.set_value(\"RESUBMIT\", resub - 1)\n        if case.get_value(\"RESUBMIT_SETS_CONTINUE_RUN\"):\n            case.set_value(\"CONTINUE_RUN\", True)\n\n    else:\n        if job == \"case.test\":\n            case.set_value(\"IS_FIRST_RUN\", True)\n\n        if no_batch:\n            batch_system = \"none\"\n        else:\n            batch_system = env_batch.get_batch_system_type()\n\n        case.set_value(\"BATCH_SYSTEM\", batch_system)\n\n        env_batch_has_changed = False\n        try:\n            case.check_lockedfile(os.path.basename(env_batch.filename))\n        except CIMEError:\n            env_batch_has_changed = True\n\n        if env_batch.get_batch_system_type() != \"none\" and env_batch_has_changed:\n            # May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc)\n            logger.warning(\n                \"\"\"\nenv_batch.xml appears to have changed, regenerating batch scripts\nmanual edits to these file will be lost!\n\"\"\"\n            )\n            env_batch.make_all_batch_files(case)\n\n        unlock_file(os.path.basename(env_batch.filename), caseroot=caseroot)\n        lock_file(os.path.basename(env_batch.filename), caseroot=caseroot)\n\n        case.check_case(skip_pnl=skip_pnl, chksum=chksum)\n        if job == case.get_primary_job():\n            case.check_DA_settings()\n            if case.get_value(\"MACH\") == \"mira\":\n                with open(\".original_host\", \"w\") as fd:\n                    fd.write(socket.gethostname())\n\n    # Load Modules\n    case.load_env()\n\n    case.flush()\n\n    logger.warning(\"submit_jobs {}\".format(job))\n    job_ids = case.submit_jobs(\n        no_batch=no_batch,\n        job=job,\n        prereq=prereq,\n        skip_pnl=skip_pnl,\n        resubmit_immediate=resubmit_immediate,\n        allow_fail=allow_fail,\n        mail_user=mail_user,\n        mail_type=mail_type,\n        batch_args=batch_args,\n        workflow=workflow,\n    )\n\n    xml_jobids = []\n    for jobname, jobid in job_ids.items():\n        logger.info(\"Submitted job {} with id {}\".format(jobname, jobid))\n        if jobid:\n            xml_jobids.append(\"{}:{}\".format(jobname, jobid))\n\n    xml_jobid_text = \", \".join(xml_jobids)\n    if xml_jobid_text:\n        case.set_value(\"JOB_IDS\", xml_jobid_text)\n\n    return xml_jobid_text\n\n\ndef submit(\n    self,\n    job=None,\n    no_batch=False,\n    prereq=None,\n    allow_fail=False,\n    resubmit=False,\n    resubmit_immediate=False,\n    skip_pnl=False,\n    mail_user=None,\n    mail_type=None,\n    batch_args=None,\n    workflow=True,\n    chksum=False,\n):\n    if resubmit_immediate and self.get_value(\"MACH\") in [\"mira\", \"cetus\"]:\n        logger.warning(\n            \"resubmit_immediate does not work on Mira/Cetus, submitting normally\"\n        )\n        resubmit_immediate = False\n\n    caseroot = self.get_value(\"CASEROOT\")\n    if self.get_value(\"TEST\"):\n        casebaseid = self.get_value(\"CASEBASEID\")\n        # This should take care of the race condition where the submitted job\n        # begins immediately and tries to set RUN phase. We proactively assume\n        # a passed SUBMIT phase. If this state is already PASS, don't set it again\n        # because then we'll lose RUN phase info if it's there. This info is important\n        # for system_tests_common to know if it needs to reinitialize the test or not.\n        with TestStatus(test_dir=caseroot, test_name=casebaseid) as ts:\n            phase_status = ts.get_status(SUBMIT_PHASE)\n            if phase_status != TEST_PASS_STATUS:\n                ts.set_status(SUBMIT_PHASE, TEST_PASS_STATUS)\n\n    # If this is a resubmit check the hidden file .submit_options for\n    # any submit options used on the original submit and use them again\n    submit_options = os.path.join(caseroot, \".submit_options\")\n    if resubmit and os.path.exists(submit_options):\n        config = configparser.RawConfigParser()\n        config.read(submit_options)\n        if not skip_pnl and config.has_option(\"SubmitOptions\", \"skip_pnl\"):\n            skip_pnl = config.getboolean(\"SubmitOptions\", \"skip_pnl\")\n        if mail_user is None and config.has_option(\"SubmitOptions\", \"mail_user\"):\n            mail_user = config.get(\"SubmitOptions\", \"mail_user\")\n        if mail_type is None and config.has_option(\"SubmitOptions\", \"mail_type\"):\n            mail_type = str(config.get(\"SubmitOptions\", \"mail_type\")).split(\",\")\n        if batch_args is None and config.has_option(\"SubmitOptions\", \"batch_args\"):\n            batch_args = config.get(\"SubmitOptions\", \"batch_args\")\n\n    is_batch = self.get_value(\"BATCH_SYSTEM\") is not None\n\n    try:\n        functor = lambda: _submit(\n            self,\n            job=job,\n            no_batch=no_batch,\n            prereq=prereq,\n            allow_fail=allow_fail,\n            resubmit=resubmit,\n            resubmit_immediate=resubmit_immediate,\n            skip_pnl=skip_pnl,\n            mail_user=mail_user,\n            mail_type=mail_type,\n            batch_args=batch_args,\n            workflow=workflow,\n            chksum=chksum,\n        )\n        run_and_log_case_status(\n            functor,\n            \"case.submit\",\n            caseroot=caseroot,\n            custom_success_msg_functor=lambda x: x.split(\":\")[-1],\n            is_batch=is_batch,\n        )\n    except BaseException:  # Want to catch KeyboardInterrupt too\n        # If something failed in the batch system, make sure to mark\n        # the test as failed if we are running a test.\n        if self.get_value(\"TEST\"):\n            with TestStatus(test_dir=caseroot, test_name=casebaseid) as ts:\n                ts.set_status(SUBMIT_PHASE, TEST_FAIL_STATUS)\n\n        raise\n\n\ndef check_case(self, skip_pnl=False, chksum=False):\n    self.check_lockedfiles()\n    if not skip_pnl:\n        self.create_namelists()  # Must be called before check_all_input_data\n    logger.info(\"Checking that inputdata is available as part of case submission\")\n    self.check_all_input_data(chksum=chksum)\n\n    if self.get_value(\"COMP_WAV\") == \"ww\":\n        # the ww3 buildnml has dependencies on inputdata so we must run it again\n        self.create_namelists(component=\"WAV\")\n\n    expect(\n        self.get_value(\"BUILD_COMPLETE\"),\n        \"Build complete is \" \"not True please rebuild the model by calling case.build\",\n    )\n    logger.info(\"Check case OK\")\n\n\ndef check_DA_settings(self):\n    script = self.get_value(\"DATA_ASSIMILATION_SCRIPT\")\n    cycles = self.get_value(\"DATA_ASSIMILATION_CYCLES\")\n    if len(script) > 0 and os.path.isfile(script) and cycles > 0:\n        logger.info(\n            \"Data Assimilation enabled using script {} with {:d} cycles\".format(\n                script, cycles\n            )\n        )\n", "path": "CIME/case/case_submit.py"}]}