We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Support some interface to the shell in our XML
Users like @amametjanov are asking for access to the shell within our XML files. Something like:

```
<env name="NETCDF_PATH">$SHELL{which nc-config | xargs dirname | xargs dirname}</env>
```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `utils/python/CIME/XML/generic_xml.py`
Content:
```
1 """
2 Common interface to XML files, this is an abstract class and is expected to
3 be used by other XML interface modules and not directly.
4 """
5 from CIME.XML.standard_module_setup import *
6 from distutils.spawn import find_executable
7 from xml.dom import minidom
8 from CIME.utils import expect, get_cime_root
9 
10 logger = logging.getLogger(__name__)
11 
12 class GenericXML(object):
13 
14     def __init__(self, infile=None, schema=None):
15         """
16         Initialize an object
17         """
18 
19         logger.debug("Initializing %s" , infile)
20         self.tree = None
21 
22         if infile == None:
23             # if file is not defined just return
24             self.filename = None
25             return
26 
27         if os.path.isfile(infile) and os.access(infile, os.R_OK):
28             # If file is defined and exists, read it
29             self.filename = infile
30             self.read(infile, schema)
31         else:
32             # if file does not exist create a root xml element
33             # and set it's id to file
34 
35             logger.debug("File %s does not exists." , infile)
36             expect("$" not in infile,"File path not fully resolved %s"%infile)
37 
38             self.filename = infile
39             root = ET.Element("xml")
40             self.root = ET.SubElement(root, "file")
41             self.root.set("id", os.path.basename(infile))
42             self.tree = ET.ElementTree(root)
43 
44     def read(self, infile, schema=None):
45         """
46         Read and parse an xml file into the object
47         """
48         logger.debug("read: " + infile)
49         if self.tree:
50             self.root.append(ET.parse(infile).getroot())
51         else:
52             self.tree = ET.parse(infile)
53             self.root = self.tree.getroot()
54 
55         if schema is not None and self.get_version() > 1.0:
56             self.validate_xml_file(infile, schema)
57 
58         logger.debug("File version is %s"%str(self.get_version()))
59 
60     def get_version(self):
61         version = self.root.get("version")
62         version = 1.0 if version is None else float(version)
63         return version
64 
65     def write(self, outfile=None):
66         """
67         Write an xml file from data in self
68         """
69         if outfile is None:
70             outfile = self.filename
71 
72         logger.debug("write: " + outfile)
73 
74         xmlstr = self.get_raw_record()
75 
76         # xmllint provides a better format option for the output file
77         xmllint = find_executable("xmllint")
78         if xmllint is not None:
79             run_cmd_no_fail("%s --format --output %s -"%(xmllint,outfile), input_str=xmlstr)
80         else:
81             doc = minidom.parseString(xmlstr)
82             with open(outfile,'w') as xmlout:
83                 doc.writexml(xmlout,addindent='  ')
84 
85     def get_node(self, nodename, attributes=None, root=None, xpath=None):
86         """
87         Get an xml element matching nodename with optional attributes.
88 
89         Error unless exactly one match.
90         """
91 
92         nodes = self.get_nodes(nodename, attributes=attributes, root=root, xpath=xpath)
93 
94         expect(len(nodes) == 1, "Incorrect number of matches, %d, for nodename '%s' and attrs '%s' in file '%s'" %
95                (len(nodes), nodename, attributes, self.filename))
96         return nodes[0]
97 
98     def get_optional_node(self, nodename, attributes=None, root=None, xpath=None):
99         """
100         Get an xml element matching nodename with optional attributes.
101 
102         Return None if no match.
103         """
104         nodes = self.get_nodes(nodename, attributes=attributes, root=root, xpath=xpath)
105 
106         expect(len(nodes) <= 1, "Multiple matches for nodename '%s' and attrs '%s' in file '%s'" %
107                (nodename, attributes, self.filename))
108         return nodes[0] if nodes else None
109 
110     def get_nodes(self, nodename, attributes=None, root=None, xpath=None):
111 
112         logger.debug("(get_nodes) Input values: %s , %s , %s , %s , %s" , self.__class__.__name__ , nodename , attributes , root , xpath)
113 
114         if root is None:
115             root = self.root
116         nodes = []
117 
118         expect(attributes is None or xpath is None,
119                " Arguments attributes and xpath are exclusive")
120         if xpath is None:
121             xpath = ".//"+nodename
122 
123         if attributes:
124             # xml.etree has limited support for xpath and does not allow more than
125             # one attribute in an xpath query so we query seperately for each attribute
126             # and create a result with the intersection of those lists
127 
128             for key, value in attributes.iteritems():
129                 if value is not None:
130                     expect(isinstance(value, basestring),
131                            " Bad value passed for key %s"%key)
132                     xpath = ".//%s[@%s=\'%s\']" % (nodename, key, value)
133                     logger.debug("xpath is %s"%xpath)
134 
135                     try:
136                         newnodes = root.findall(xpath)
137                     except Exception as e:
138                         expect(False, "Bad xpath search term '%s', error: %s" % (xpath, e))
139 
140                     if not nodes:
141                         nodes = newnodes
142                     else:
143                         for node in nodes[:]:
144                             if node not in newnodes:
145                                 nodes.remove(node)
146                     if not nodes:
147                         return []
148 
149         else:
150             logger.debug("xpath: %s" , xpath)
151             nodes = root.findall(xpath)
152 
153         logger.debug("Returning %s nodes (%s)" , len(nodes), nodes)
154 
155         return nodes
156 
157     def add_child(self, node, root=None):
158         """
159         Add element node to self at root
160         """
161         if root is None:
162             root = self.root
163         self.root.append(node)
164 
165     def get_value(self, item, attribute=None, resolved=True, subgroup=None): # pylint: disable=unused-argument
166         """
167         get_value is expected to be defined by the derived classes, if you get here
168         the value was not found in the class.
169         """
170         logger.debug("Get Value for " + item)
171         return None
172 
173     def get_values(self, vid, attribute=None, resolved=True, subgroup=None):# pylint: disable=unused-argument
174         logger.debug("Get Values for " + vid)
175         return []
176 
177     def set_value(self, vid, value, subgroup=None, ignore_type=True): # pylint: disable=unused-argument
178         """
179         ignore_type is not used in this flavor
180         """
181         valnodes = self.get_nodes(vid)
182         if valnodes:
183             for node in valnodes:
184                 node.text = value
185 
186     def get_resolved_value(self, raw_value):
187         """
188         A value in the xml file may contain references to other xml
189         variables or to environment variables. These are refered to in
190         the perl style with $name and $ENV{name}.
191 
192         >>> obj = GenericXML()
193         >>> os.environ["FOO"] = "BAR"
194         >>> os.environ["BAZ"] = "BARF"
195         >>> obj.get_resolved_value("one $ENV{FOO} two $ENV{BAZ} three")
196         'one BAR two BARF three'
197         >>> obj.get_resolved_value("2 + 3 - 1")
198         '4'
199         >>> obj.get_resolved_value("0001-01-01")
200         '0001-01-01'
201         """
202         logger.debug("raw_value %s" % raw_value)
203         reference_re = re.compile(r'\${?(\w+)}?')
204         env_ref_re   = re.compile(r'\$ENV\{(\w+)\}')
205         math_re = re.compile(r'\s[+-/*]\s')
206         item_data = raw_value
207 
208         if item_data is None:
209             return None
210 
211         if type(item_data) is not str:
212             return item_data
213 
214         for m in env_ref_re.finditer(item_data):
215             logger.debug("look for %s in env" % item_data)
216             env_var = m.groups()[0]
217             expect(env_var in os.environ, "Undefined env var '%s'" % env_var)
218             item_data = item_data.replace(m.group(), os.environ[env_var])
219 
220         for m in reference_re.finditer(item_data):
221             var = m.groups()[0]
222             logger.debug("find: %s" % var)
223             ref = self.get_value(var)
224             if ref is not None:
225                 logger.debug("resolve: " + str(ref))
226                 item_data = item_data.replace(m.group(), self.get_resolved_value(str(ref)))
227             elif var == "CIMEROOT":
228                 cimeroot = get_cime_root()
229                 item_data = item_data.replace(m.group(), cimeroot)
230             elif var == "SRCROOT":
231                 srcroot = os.path.join(get_cime_root(),"..")
232                 item_data = item_data.replace(m.group(), srcroot)
233             elif var in os.environ:
234                 # this is a list of suppressed warnings (things normally expected to be resolved in env)
235                 if var not in ("USER",):
236                     logging.debug("Resolved from env: " + var)
237                 item_data = item_data.replace(m.group(), os.environ[var])
238         if math_re.search(item_data):
239             try:
240                 tmp = eval(item_data)
241             except:
242                 tmp = item_data
243             item_data = str(tmp)
244 
245         return item_data
246 
247     def add_sub_node(self, node, subnode_name, subnode_text):
248         expect(node is not None," Bad value passed")
249         subnode = ET.Element(subnode_name)
250         subnode.text = subnode_text
251         node.append(subnode)
252         return node
253 
254     def validate_xml_file(self, filename, schema):
255         """
256         validate an XML file against a provided schema file using pylint
257         """
258         expect(os.path.isfile(filename),"xml file not found %s"%filename)
259         expect(os.path.isfile(schema),"schema file not found %s"%schema)
260         xmllint = find_executable("xmllint")
261         if xmllint is not None:
262             logger.debug("Checking file %s against schema %s"%(filename, schema))
263             run_cmd_no_fail("%s --noout --schema %s %s"%(xmllint, schema, filename))
264         else:
265             logger.warn("xmllint not found, could not validate file %s"%filename)
266 
267     def get_element_text(self, element_name, attributes=None, root=None, xpath=None):
268         element_node = self.get_optional_node(element_name, attributes, root, xpath)
269         if element_node is not None:
270             return element_node.text
271         return None
272 
273     def set_element_text(self, element_name, new_text, attributes=None, root=None, xpath=None):
274         element_node = self.get_optional_node(element_name, attributes, root, xpath)
275         if element_node is not None:
276             element_node.text = new_text
277             return new_text
278         return None
279 
280     def get_raw_record(self, root=None):
281         if root is None:
282             root = self.root
283         try:
284             xmlstr = ET.tostring(root)
285         except ET.ParseError as e:
286             ET.dump(root)
287             expect(False, "Could not write file %s, xml formatting error '%s'" % (self.filename, e))
288         return xmlstr
289 
290 
291
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/utils/python/CIME/XML/generic_xml.py b/utils/python/CIME/XML/generic_xml.py
--- a/utils/python/CIME/XML/generic_xml.py
+++ b/utils/python/CIME/XML/generic_xml.py
@@ -7,6 +7,8 @@
 from xml.dom import minidom
 from CIME.utils import expect, get_cime_root
 
+import getpass
+
 logger = logging.getLogger(__name__)
 
 class GenericXML(object):
@@ -198,10 +200,13 @@
         '4'
         >>> obj.get_resolved_value("0001-01-01")
         '0001-01-01'
+        >>> obj.get_resolved_value("$SHELL{echo hi}")
+        'hi'
         """
         logger.debug("raw_value %s" % raw_value)
         reference_re = re.compile(r'\${?(\w+)}?')
         env_ref_re   = re.compile(r'\$ENV\{(\w+)\}')
+        shell_ref_re = re.compile(r'\$SHELL\{([^}]+)\}')
         math_re = re.compile(r'\s[+-/*]\s')
         item_data = raw_value
 
@@ -217,6 +222,11 @@
             expect(env_var in os.environ, "Undefined env var '%s'" % env_var)
             item_data = item_data.replace(m.group(), os.environ[env_var])
 
+        for s in shell_ref_re.finditer(item_data):
+            logger.debug("execute %s in shell" % item_data)
+            shell_cmd = s.groups()[0]
+            item_data = item_data.replace(s.group(), run_cmd_no_fail(shell_cmd))
+
         for m in reference_re.finditer(item_data):
             var = m.groups()[0]
             logger.debug("find: %s" % var)
@@ -230,11 +240,9 @@
             elif var == "SRCROOT":
                 srcroot = os.path.join(get_cime_root(),"..")
                 item_data = item_data.replace(m.group(), srcroot)
-            elif var in os.environ:
-                # this is a list of suppressed warnings (things normally expected to be resolved in env)
-                if var not in ("USER",):
-                    logging.debug("Resolved from env: " + var)
-                item_data = item_data.replace(m.group(), os.environ[var])
+            elif var == "USER":
+                item_data = item_data.replace(m.group(), getpass.getuser())
+
         if math_re.search(item_data):
             try:
                 tmp = eval(item_data)

{"golden_diff": "diff --git a/utils/python/CIME/XML/generic_xml.py b/utils/python/CIME/XML/generic_xml.py\n--- a/utils/python/CIME/XML/generic_xml.py\n+++ b/utils/python/CIME/XML/generic_xml.py\n@@ -7,6 +7,8 @@\n from xml.dom import minidom\n from CIME.utils import expect, get_cime_root\n \n+import getpass\n+\n logger = logging.getLogger(__name__)\n \n class GenericXML(object):\n@@ -198,10 +200,13 @@\n         '4'\n         >>> obj.get_resolved_value(\"0001-01-01\")\n         '0001-01-01'\n+        >>> obj.get_resolved_value(\"$SHELL{echo hi}\")\n+        'hi'\n         \"\"\"\n         logger.debug(\"raw_value %s\" % raw_value)\n         reference_re = re.compile(r'\\${?(\\w+)}?')\n         env_ref_re   = re.compile(r'\\$ENV\\{(\\w+)\\}')\n+        shell_ref_re = re.compile(r'\\$SHELL\\{([^}]+)\\}')\n         math_re = re.compile(r'\\s[+-/*]\\s')\n         item_data = raw_value\n \n@@ -217,6 +222,11 @@\n             expect(env_var in os.environ, \"Undefined env var '%s'\" % env_var)\n             item_data = item_data.replace(m.group(), os.environ[env_var])\n \n+        for s in shell_ref_re.finditer(item_data):\n+            logger.debug(\"execute %s in shell\" % item_data)\n+            shell_cmd = s.groups()[0]\n+            item_data = item_data.replace(s.group(), run_cmd_no_fail(shell_cmd))\n+\n         for m in reference_re.finditer(item_data):\n             var = m.groups()[0]\n             logger.debug(\"find: %s\" % var)\n@@ -230,11 +240,9 @@\n             elif var == \"SRCROOT\":\n                 srcroot = os.path.join(get_cime_root(),\"..\")\n                 item_data = item_data.replace(m.group(), srcroot)\n-            elif var in os.environ:\n-                # this is a list of suppressed warnings (things normally expected to be resolved in env)\n-                if var not in (\"USER\",):\n-                    logging.debug(\"Resolved from env: \" + var)\n-                item_data = item_data.replace(m.group(), os.environ[var])\n+            elif var == \"USER\":\n+                item_data = item_data.replace(m.group(), getpass.getuser())\n+\n         if math_re.search(item_data):\n             try:\n                 tmp = eval(item_data)\n", "issue": "Support some interface to the shell in our XML\nUsers like @amametjanov are asking for access to the shell within our XML files. Something like:\r\n\r\n```\r\n<env name=\"NETCDF_PATH\">$SHELL{which nc-config | xargs dirname | xargs dirname}</env>\r\n```\n", "before_files": [{"content": "\"\"\"\nCommon interface to XML files, this is an abstract class and is expected to\nbe used by other XML interface modules and not directly.\n\"\"\"\nfrom CIME.XML.standard_module_setup import *\nfrom distutils.spawn import find_executable\nfrom xml.dom import minidom\nfrom CIME.utils import expect, get_cime_root\n\nlogger = logging.getLogger(__name__)\n\nclass GenericXML(object):\n\n    def __init__(self, infile=None, schema=None):\n        \"\"\"\n        Initialize an object\n        \"\"\"\n\n        logger.debug(\"Initializing %s\" , infile)\n        self.tree = None\n\n        if infile == None:\n            # if file is not defined just return\n            self.filename = None\n            return\n\n        if os.path.isfile(infile) and os.access(infile, os.R_OK):\n            # If file is defined and exists, read it\n            self.filename = infile\n            self.read(infile, schema)\n        else:\n            # if file does not exist create a root xml element\n            # and set it's id to file\n\n            logger.debug(\"File %s does not exists.\" , infile)\n            expect(\"$\" not in infile,\"File path not fully resolved %s\"%infile)\n\n            self.filename = infile\n            root = ET.Element(\"xml\")\n            self.root = ET.SubElement(root, \"file\")\n            self.root.set(\"id\", os.path.basename(infile))\n            self.tree = ET.ElementTree(root)\n\n    def read(self, infile, schema=None):\n        \"\"\"\n        Read and parse an xml file into the object\n        \"\"\"\n        logger.debug(\"read: \" + infile)\n        if self.tree:\n            self.root.append(ET.parse(infile).getroot())\n        else:\n            self.tree = ET.parse(infile)\n            self.root = self.tree.getroot()\n\n        if schema is not None and self.get_version() > 1.0:\n            self.validate_xml_file(infile, schema)\n\n        logger.debug(\"File version is %s\"%str(self.get_version()))\n\n    def get_version(self):\n        version = self.root.get(\"version\")\n        version = 1.0 if version is None else float(version)\n        return version\n\n    def write(self, outfile=None):\n        \"\"\"\n        Write an xml file from data in self\n        \"\"\"\n        if outfile is None:\n            outfile = self.filename\n\n        logger.debug(\"write: \" + outfile)\n\n        xmlstr = self.get_raw_record()\n\n        # xmllint provides a better format option for the output file\n        xmllint = find_executable(\"xmllint\")\n        if xmllint is not None:\n            run_cmd_no_fail(\"%s --format --output %s -\"%(xmllint,outfile), input_str=xmlstr)\n        else:\n            doc = minidom.parseString(xmlstr)\n            with open(outfile,'w') as xmlout:\n                doc.writexml(xmlout,addindent='  ')\n\n    def get_node(self, nodename, attributes=None, root=None, xpath=None):\n        \"\"\"\n        Get an xml element matching nodename with optional attributes.\n\n        Error unless exactly one match.\n        \"\"\"\n\n        nodes = self.get_nodes(nodename, attributes=attributes, root=root, xpath=xpath)\n\n        expect(len(nodes) == 1, \"Incorrect number of matches, %d, for nodename '%s' and attrs '%s' in file '%s'\" %\n               (len(nodes), nodename, attributes, self.filename))\n        return nodes[0]\n\n    def get_optional_node(self, nodename, attributes=None, root=None, xpath=None):\n        \"\"\"\n        Get an xml element matching nodename with optional attributes.\n\n        Return None if no match.\n        \"\"\"\n        nodes = self.get_nodes(nodename, attributes=attributes, root=root, xpath=xpath)\n\n        expect(len(nodes) <= 1, \"Multiple matches for nodename '%s' and attrs '%s' in file '%s'\" %\n               (nodename, attributes, self.filename))\n        return nodes[0] if nodes else None\n\n    def get_nodes(self, nodename, attributes=None, root=None, xpath=None):\n\n        logger.debug(\"(get_nodes) Input values: %s , %s , %s , %s , %s\" , self.__class__.__name__ , nodename , attributes , root , xpath)\n\n        if root is None:\n            root = self.root\n        nodes = []\n\n        expect(attributes is None or xpath is None,\n               \" Arguments attributes and xpath are exclusive\")\n        if xpath is None:\n            xpath = \".//\"+nodename\n\n        if attributes:\n            # xml.etree has limited support for xpath and does not allow more than\n            # one attribute in an xpath query so we query seperately for each attribute\n            # and create a result with the intersection of those lists\n\n            for key, value in attributes.iteritems():\n                if value is not None:\n                    expect(isinstance(value, basestring),\n                           \" Bad value passed for key %s\"%key)\n                    xpath = \".//%s[@%s=\\'%s\\']\" % (nodename, key, value)\n                    logger.debug(\"xpath is %s\"%xpath)\n\n                    try:\n                        newnodes = root.findall(xpath)\n                    except Exception as e:\n                        expect(False, \"Bad xpath search term '%s', error: %s\" % (xpath, e))\n\n                    if not nodes:\n                        nodes = newnodes\n                    else:\n                        for node in nodes[:]:\n                            if node not in newnodes:\n                                nodes.remove(node)\n                    if not nodes:\n                        return []\n\n        else:\n            logger.debug(\"xpath: %s\" , xpath)\n            nodes = root.findall(xpath)\n\n        logger.debug(\"Returning %s nodes (%s)\" , len(nodes), nodes)\n\n        return nodes\n\n    def add_child(self, node, root=None):\n        \"\"\"\n        Add element node to self at root\n        \"\"\"\n        if root is None:\n            root = self.root\n        self.root.append(node)\n\n    def get_value(self, item, attribute=None, resolved=True, subgroup=None): # pylint: disable=unused-argument\n        \"\"\"\n        get_value is expected to be defined by the derived classes, if you get here\n        the value was not found in the class.\n        \"\"\"\n        logger.debug(\"Get Value for \" + item)\n        return None\n\n    def get_values(self, vid, attribute=None, resolved=True, subgroup=None):# pylint: disable=unused-argument\n        logger.debug(\"Get Values for \" + vid)\n        return []\n\n    def set_value(self, vid, value, subgroup=None, ignore_type=True): # pylint: disable=unused-argument\n        \"\"\"\n        ignore_type is not used in this flavor\n        \"\"\"\n        valnodes = self.get_nodes(vid)\n        if valnodes:\n            for node in valnodes:\n                node.text = value\n\n    def get_resolved_value(self, raw_value):\n        \"\"\"\n        A value in the xml file may contain references to other xml\n        variables or to environment variables. These are refered to in\n        the perl style with $name and $ENV{name}.\n\n        >>> obj = GenericXML()\n        >>> os.environ[\"FOO\"] = \"BAR\"\n        >>> os.environ[\"BAZ\"] = \"BARF\"\n        >>> obj.get_resolved_value(\"one $ENV{FOO} two $ENV{BAZ} three\")\n        'one BAR two BARF three'\n        >>> obj.get_resolved_value(\"2 + 3 - 1\")\n        '4'\n        >>> obj.get_resolved_value(\"0001-01-01\")\n        '0001-01-01'\n        \"\"\"\n        logger.debug(\"raw_value %s\" % raw_value)\n        reference_re = re.compile(r'\\${?(\\w+)}?')\n        env_ref_re   = re.compile(r'\\$ENV\\{(\\w+)\\}')\n        math_re = re.compile(r'\\s[+-/*]\\s')\n        item_data = raw_value\n\n        if item_data is None:\n            return None\n\n        if type(item_data) is not str:\n            return item_data\n\n        for m in env_ref_re.finditer(item_data):\n            logger.debug(\"look for %s in env\" % item_data)\n            env_var = m.groups()[0]\n            expect(env_var in os.environ, \"Undefined env var '%s'\" % env_var)\n            item_data = item_data.replace(m.group(), os.environ[env_var])\n\n        for m in reference_re.finditer(item_data):\n            var = m.groups()[0]\n            logger.debug(\"find: %s\" % var)\n            ref = self.get_value(var)\n            if ref is not None:\n                logger.debug(\"resolve: \" + str(ref))\n                item_data = item_data.replace(m.group(), self.get_resolved_value(str(ref)))\n            elif var == \"CIMEROOT\":\n                cimeroot = get_cime_root()\n                item_data = item_data.replace(m.group(), cimeroot)\n            elif var == \"SRCROOT\":\n                srcroot = os.path.join(get_cime_root(),\"..\")\n                item_data = item_data.replace(m.group(), srcroot)\n            elif var in os.environ:\n                # this is a list of suppressed warnings (things normally expected to be resolved in env)\n                if var not in (\"USER\",):\n                    logging.debug(\"Resolved from env: \" + var)\n                item_data = item_data.replace(m.group(), os.environ[var])\n        if math_re.search(item_data):\n            try:\n                tmp = eval(item_data)\n            except:\n                tmp = item_data\n            item_data = str(tmp)\n\n        return item_data\n\n    def add_sub_node(self, node, subnode_name, subnode_text):\n        expect(node is not None,\" Bad value passed\")\n        subnode = ET.Element(subnode_name)\n        subnode.text = subnode_text\n        node.append(subnode)\n        return node\n\n    def validate_xml_file(self, filename, schema):\n        \"\"\"\n        validate an XML file against a provided schema file using pylint\n        \"\"\"\n        expect(os.path.isfile(filename),\"xml file not found %s\"%filename)\n        expect(os.path.isfile(schema),\"schema file not found %s\"%schema)\n        xmllint = find_executable(\"xmllint\")\n        if xmllint is not None:\n            logger.debug(\"Checking file %s against schema %s\"%(filename, schema))\n            run_cmd_no_fail(\"%s --noout --schema %s %s\"%(xmllint, schema, filename))\n        else:\n            logger.warn(\"xmllint not found, could not validate file %s\"%filename)\n\n    def get_element_text(self, element_name, attributes=None, root=None, xpath=None):\n        element_node = self.get_optional_node(element_name, attributes, root, xpath)\n        if element_node is not None:\n            return element_node.text\n        return None\n\n    def set_element_text(self, element_name, new_text, attributes=None, root=None, xpath=None):\n        element_node = self.get_optional_node(element_name, attributes, root, xpath)\n        if element_node is not None:\n            element_node.text = new_text\n            return new_text\n        return None\n\n    def get_raw_record(self, root=None):\n        if root is None:\n            root = self.root\n        try:\n            xmlstr = ET.tostring(root)\n        except ET.ParseError as e:\n            ET.dump(root)\n            expect(False, \"Could not write file %s, xml formatting error '%s'\" % (self.filename, e))\n        return xmlstr\n\n\n", "path": "utils/python/CIME/XML/generic_xml.py"}], "after_files": [{"content": "\"\"\"\nCommon interface to XML files, this is an abstract class and is expected to\nbe used by other XML interface modules and not directly.\n\"\"\"\nfrom CIME.XML.standard_module_setup import *\nfrom distutils.spawn import find_executable\nfrom xml.dom import minidom\nfrom CIME.utils import expect, get_cime_root\n\nimport getpass\n\nlogger = logging.getLogger(__name__)\n\nclass GenericXML(object):\n\n    def __init__(self, infile=None, schema=None):\n        \"\"\"\n        Initialize an object\n        \"\"\"\n\n        logger.debug(\"Initializing %s\" , infile)\n        self.tree = None\n\n        if infile == None:\n            # if file is not defined just return\n            self.filename = None\n            return\n\n        if os.path.isfile(infile) and os.access(infile, os.R_OK):\n            # If file is defined and exists, read it\n            self.filename = infile\n            self.read(infile, schema)\n        else:\n            # if file does not exist create a root xml element\n            # and set it's id to file\n\n            logger.debug(\"File %s does not exists.\" , infile)\n            expect(\"$\" not in infile,\"File path not fully resolved %s\"%infile)\n\n            self.filename = infile\n            root = ET.Element(\"xml\")\n            self.root = ET.SubElement(root, \"file\")\n            self.root.set(\"id\", os.path.basename(infile))\n            self.tree = ET.ElementTree(root)\n\n    def read(self, infile, schema=None):\n        \"\"\"\n        Read and parse an xml file into the object\n        \"\"\"\n        logger.debug(\"read: \" + infile)\n        if self.tree:\n            self.root.append(ET.parse(infile).getroot())\n        else:\n            self.tree = ET.parse(infile)\n            self.root = self.tree.getroot()\n\n        if schema is not None and self.get_version() > 1.0:\n            self.validate_xml_file(infile, schema)\n\n        logger.debug(\"File version is %s\"%str(self.get_version()))\n\n    def get_version(self):\n        version = self.root.get(\"version\")\n        version = 1.0 if version is None else float(version)\n        return version\n\n    def write(self, outfile=None):\n        \"\"\"\n        Write an xml file from data in self\n        \"\"\"\n        if outfile is None:\n            outfile = self.filename\n\n        logger.debug(\"write: \" + outfile)\n\n        xmlstr = self.get_raw_record()\n\n        # xmllint provides a better format option for the output file\n        xmllint = find_executable(\"xmllint\")\n        if xmllint is not None:\n            run_cmd_no_fail(\"%s --format --output %s -\"%(xmllint,outfile), input_str=xmlstr)\n        else:\n            doc = minidom.parseString(xmlstr)\n            with open(outfile,'w') as xmlout:\n                doc.writexml(xmlout,addindent='  ')\n\n    def get_node(self, nodename, attributes=None, root=None, xpath=None):\n        \"\"\"\n        Get an xml element matching nodename with optional attributes.\n\n        Error unless exactly one match.\n        \"\"\"\n\n        nodes = self.get_nodes(nodename, attributes=attributes, root=root, xpath=xpath)\n\n        expect(len(nodes) == 1, \"Incorrect number of matches, %d, for nodename '%s' and attrs '%s' in file '%s'\" %\n               (len(nodes), nodename, attributes, self.filename))\n        return nodes[0]\n\n    def get_optional_node(self, nodename, attributes=None, root=None, xpath=None):\n        \"\"\"\n        Get an xml element matching nodename with optional attributes.\n\n        Return None if no match.\n        \"\"\"\n        nodes = self.get_nodes(nodename, attributes=attributes, root=root, xpath=xpath)\n\n        expect(len(nodes) <= 1, \"Multiple matches for nodename '%s' and attrs '%s' in file '%s'\" %\n               (nodename, attributes, self.filename))\n        return nodes[0] if nodes else None\n\n    def get_nodes(self, nodename, attributes=None, root=None, xpath=None):\n\n        logger.debug(\"(get_nodes) Input values: %s , %s , %s , %s , %s\" , self.__class__.__name__ , nodename , attributes , root , xpath)\n\n        if root is None:\n            root = self.root\n        nodes = []\n\n        expect(attributes is None or xpath is None,\n               \" Arguments attributes and xpath are exclusive\")\n        if xpath is None:\n            xpath = \".//\"+nodename\n\n        if attributes:\n            # xml.etree has limited support for xpath and does not allow more than\n            # one attribute in an xpath query so we query seperately for each attribute\n            # and create a result with the intersection of those lists\n\n            for key, value in attributes.iteritems():\n                if value is not None:\n                    expect(isinstance(value, basestring),\n                           \" Bad value passed for key %s\"%key)\n                    xpath = \".//%s[@%s=\\'%s\\']\" % (nodename, key, value)\n                    logger.debug(\"xpath is %s\"%xpath)\n\n                    try:\n                        newnodes = root.findall(xpath)\n                    except Exception as e:\n                        expect(False, \"Bad xpath search term '%s', error: %s\" % (xpath, e))\n\n                    if not nodes:\n                        nodes = newnodes\n                    else:\n                        for node in nodes[:]:\n                            if node not in newnodes:\n                                nodes.remove(node)\n                    if not nodes:\n                        return []\n\n        else:\n            logger.debug(\"xpath: %s\" , xpath)\n            nodes = root.findall(xpath)\n\n        logger.debug(\"Returning %s nodes (%s)\" , len(nodes), nodes)\n\n        return nodes\n\n    def add_child(self, node, root=None):\n        \"\"\"\n        Add element node to self at root\n        \"\"\"\n        if root is None:\n            root = self.root\n        self.root.append(node)\n\n    def get_value(self, item, attribute=None, resolved=True, subgroup=None): # pylint: disable=unused-argument\n        \"\"\"\n        get_value is expected to be defined by the derived classes, if you get here\n        the value was not found in the class.\n        \"\"\"\n        logger.debug(\"Get Value for \" + item)\n        return None\n\n    def get_values(self, vid, attribute=None, resolved=True, subgroup=None):# pylint: disable=unused-argument\n        logger.debug(\"Get Values for \" + vid)\n        return []\n\n    def set_value(self, vid, value, subgroup=None, ignore_type=True): # pylint: disable=unused-argument\n        \"\"\"\n        ignore_type is not used in this flavor\n        \"\"\"\n        valnodes = self.get_nodes(vid)\n        if valnodes:\n            for node in valnodes:\n                node.text = value\n\n    def get_resolved_value(self, raw_value):\n        \"\"\"\n        A value in the xml file may contain references to other xml\n        variables or to environment variables. These are refered to in\n        the perl style with $name and $ENV{name}.\n\n        >>> obj = GenericXML()\n        >>> os.environ[\"FOO\"] = \"BAR\"\n        >>> os.environ[\"BAZ\"] = \"BARF\"\n        >>> obj.get_resolved_value(\"one $ENV{FOO} two $ENV{BAZ} three\")\n        'one BAR two BARF three'\n        >>> obj.get_resolved_value(\"2 + 3 - 1\")\n        '4'\n        >>> obj.get_resolved_value(\"0001-01-01\")\n        '0001-01-01'\n        >>> obj.get_resolved_value(\"$SHELL{echo hi}\")\n        'hi'\n        \"\"\"\n        logger.debug(\"raw_value %s\" % raw_value)\n        reference_re = re.compile(r'\\${?(\\w+)}?')\n        env_ref_re   = re.compile(r'\\$ENV\\{(\\w+)\\}')\n        shell_ref_re = re.compile(r'\\$SHELL\\{([^}]+)\\}')\n        math_re = re.compile(r'\\s[+-/*]\\s')\n        item_data = raw_value\n\n        if item_data is None:\n            return None\n\n        if type(item_data) is not str:\n            return item_data\n\n        for m in env_ref_re.finditer(item_data):\n            logger.debug(\"look for %s in env\" % item_data)\n            env_var = m.groups()[0]\n            expect(env_var in os.environ, \"Undefined env var '%s'\" % env_var)\n            item_data = item_data.replace(m.group(), os.environ[env_var])\n\n        for s in shell_ref_re.finditer(item_data):\n            logger.debug(\"execute %s in shell\" % item_data)\n            shell_cmd = s.groups()[0]\n            item_data = item_data.replace(s.group(), run_cmd_no_fail(shell_cmd))\n\n        for m in reference_re.finditer(item_data):\n            var = m.groups()[0]\n            logger.debug(\"find: %s\" % var)\n            ref = self.get_value(var)\n            if ref is not None:\n                logger.debug(\"resolve: \" + str(ref))\n                item_data = item_data.replace(m.group(), self.get_resolved_value(str(ref)))\n            elif var == \"CIMEROOT\":\n                cimeroot = get_cime_root()\n                item_data = item_data.replace(m.group(), cimeroot)\n            elif var == \"SRCROOT\":\n                srcroot = os.path.join(get_cime_root(),\"..\")\n                item_data = item_data.replace(m.group(), srcroot)\n            elif var == \"USER\":\n                item_data = item_data.replace(m.group(), getpass.getuser())\n\n        if math_re.search(item_data):\n            try:\n                tmp = eval(item_data)\n            except:\n                tmp = item_data\n            item_data = str(tmp)\n\n        return item_data\n\n    def add_sub_node(self, node, subnode_name, subnode_text):\n        expect(node is not None,\" Bad value passed\")\n        subnode = ET.Element(subnode_name)\n        subnode.text = subnode_text\n        node.append(subnode)\n        return node\n\n    def validate_xml_file(self, filename, schema):\n        \"\"\"\n        validate an XML file against a provided schema file using pylint\n        \"\"\"\n        expect(os.path.isfile(filename),\"xml file not found %s\"%filename)\n        expect(os.path.isfile(schema),\"schema file not found %s\"%schema)\n        xmllint = find_executable(\"xmllint\")\n        if xmllint is not None:\n            logger.debug(\"Checking file %s against schema %s\"%(filename, schema))\n            run_cmd_no_fail(\"%s --noout --schema %s %s\"%(xmllint, schema, filename))\n        else:\n            logger.warn(\"xmllint not found, could not validate file %s\"%filename)\n\n    def get_element_text(self, element_name, attributes=None, root=None, xpath=None):\n        element_node = self.get_optional_node(element_name, attributes, root, xpath)\n        if element_node is not None:\n            return element_node.text\n        return None\n\n    def set_element_text(self, element_name, new_text, attributes=None, root=None, xpath=None):\n        element_node = self.get_optional_node(element_name, attributes, root, xpath)\n        if element_node is not None:\n            element_node.text = new_text\n            return new_text\n        return None\n\n    def get_raw_record(self, root=None):\n        if root is None:\n            root = self.root\n        try:\n            xmlstr = ET.tostring(root)\n        except ET.ParseError as e:\n            ET.dump(root)\n            expect(False, \"Could not write file %s, xml formatting error '%s'\" % (self.filename, e))\n        return xmlstr\n\n\n", "path": "utils/python/CIME/XML/generic_xml.py"}]}