prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  1.20.4rc2 storm protocol bug
  Motivation
  ----------
  The Storm protocol in RSEManager returns the input lfn as the pfn in lfns2pfns. This causes a crash as an InternalScope is then used as a dictionary key in list_replicas.
  
  Modification
  ------------
  The lfns dictionary should be sanitised so that scope is returned as an external string.
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `lib/rucio/rse/protocols/storm.py`
  Content:
  ```
  1 # Copyright European Organization for Nuclear Research (CERN)
  2 #
  3 # Licensed under the Apache License, Version 2.0 (the "License");
  4 # You may not use this file except in compliance with the License.
  5 # You may obtain a copy of the License at
  6 # http://www.apache.org/licenses/LICENSE-2.0
  7 #
  8 # Authors:
  9 # - Tomas Javor Javurek, <tomas.javurek@cern.ch>, 2019
  10 
  11 
  12 import os
  13 
  14 from exceptions import NotImplementedError
  15 from xml.dom import minidom
  16 
  17 from rucio.common import exception
  18 from rucio.common.utils import run_cmd_process
  19 from rucio.rse.protocols import protocol
  20 
  21 
  22 class Default(protocol.RSEProtocol):
  23     """ Implementing access to RSEs using the local filesystem."""
  24 
  25     def __init__(self, protocol_attr, rse_settings):
  26         """ Initializes the object with information about the referred RSE.
  27 
  28             :param props Properties derived from the RSE Repository
  29         """
  30         super(Default, self).__init__(protocol_attr, rse_settings)
  31         self.attributes.pop('determinism_type', None)
  32         self.files = []
  33 
  34     def _get_path(self, scope, name):
  35         """ Transforms the physical file name into the local URI in the referred RSE.
  36             Suitable for sites implementoing the RUCIO naming convention.
  37 
  38             :param name: filename
  39             :param scope: scope
  40 
  41             :returns: RSE specific URI of the physical file
  42         """
  43         return '%s/%s' % (scope, name)
  44 
  45     def lfns2pfns(self, lfns):
  46         """ In this case, just returns back lfn. """
  47         return lfns
  48 
  49     def path2pfn(self, path):
  50         """
  51             Retruns a fully qualified PFN for the file referred by path.
  52 
  53             :param path: The path to the file.
  54 
  55             :returns: Fully qualified PFN.
  56 
  57         """
  58         return ''.join([self.rse['scheme'], '://%s' % self.rse['hostname'], path])
  59 
  60     def exists(self, pfn):
  61         """ Checks if the requested file is known by the referred RSE.
  62 
  63             :param pfn Physical file name
  64 
  65             :returns: True if the file exists, False if it doesn't
  66 
  67             :raise  ServiceUnavailable
  68         """
  69         raise NotImplementedError
  70 
  71     def connect(self):
  72         """ Establishes the actual connection to the referred RSE.
  73 
  74             :param credentials Provide all necessary information to establish a connection
  75                 to the referred storage system. Some is loaded from the repository inside the
  76                 RSE class and some must be provided specific for the SFTP protocol like
  77                 username, password, private_key, private_key_pass, port.
  78                 For details about possible additional parameters and details about their usage
  79                 see the pysftp.Connection() documentation.
  80                 NOTE: the host parametrer is overwritten with the value provided by the repository
  81 
  82             :raise RSEAccessDenied
  83         """
  84         pass
  85 
  86     def close(self):
  87         """ Closes the connection to RSE."""
  88         pass
  89 
  90     def get(self, pfn, dest, transfer_timeout=None):
  91         """ Provides access to files stored inside connected the RSE.
  92 
  93             :param pfn Physical file name of requested file
  94             :param dest Name and path of the files when stored at the client
  95             :param transfer_timeout Transfer timeout (in seconds)
  96 
  97             :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
  98          """
  99 
  100         # storm prefix needs to be replaced by davs in order to get etag
  101         pfn = 'davs' + pfn[5:]
  102 
  103         # retrieve the TURL from the webdav etag, TODO: make it configurable
  104         cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % pfn
  105         try:
  106             rcode, output = run_cmd_process(cmd, timeout=10)
  107         except Exception as e:
  108             raise exception.ServiceUnavailable('Could not retrieve STORM WebDAV ETag: %s' % str(e))
  109         p_output = minidom.parseString(output)
  110 
  111         # we need to strip off the quotation marks and the <timestamp> from the etag
  112         # but since we can have multiple underscores, we have to rely on the uniqueness
  113         # of the full LFN to make the split
  114         target = p_output.getElementsByTagName('d:getetag')[0].childNodes[0].nodeValue.replace('"', '')
  115         target_ending = '_' + target.split('_')[-1]
  116         target = target.split(target_ending)[0]
  117 
  118         # make the symlink
  119         try:
  120             os.symlink(target, dest)
  121         except Exception as e:
  122             exception.ServiceUnavailable('Could not create symlink: %s for target %s' % (str(e), str(target)))
  123 
  124     def put(self, source, target, source_dir=None, transfer_timeout=None):
  125         """ Allows to store files inside the referred RSE.
  126 
  127             :param source Physical file name
  128             :param target Name of the file on the storage system e.g. with prefixed scope
  129             :param source_dir Path where the to be transferred files are stored in the local file system
  130             :param transfer_timeout Transfer timeout (in seconds)
  131 
  132             :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
  133             """
  134         raise NotImplementedError
  135 
  136     def delete(self, pfn):
  137         """ Deletes a file from the connected RSE.
  138 
  139             :param pfn Physical file name
  140 
  141             :raises ServiceUnavailable, SourceNotFound
  142         """
  143         raise NotImplementedError
  144 
  145     def rename(self, pfn, new_pfn):
  146         """ Allows to rename a file stored inside the connected RSE.
  147 
  148             :param pfn      Current physical file name
  149             :param new_pfn  New physical file name
  150 
  151             :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
  152         """
  153         raise NotImplementedError
  154
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/lib/rucio/rse/protocols/storm.py b/lib/rucio/rse/protocols/storm.py
  --- a/lib/rucio/rse/protocols/storm.py
  +++ b/lib/rucio/rse/protocols/storm.py
  @@ -43,8 +43,23 @@
           return '%s/%s' % (scope, name)
   
       def lfns2pfns(self, lfns):
  -        """ In this case, just returns back lfn. """
  -        return lfns
  +        """ In this case, just returns back lfn with external scope. """
  +        if type(lfns) == dict:
  +            val = lfns.copy()
  +            if 'scope' in val and val['scope'] is not None:
  +                val['scope'] = val['scope'].external
  +
  +        elif type(lfns) == list:
  +            val = []
  +            for l in lfns:
  +                v = l.copy()
  +                if 'scope' in v and v['scope'] is not None:
  +                    v['scope'] = v['scope'].external
  +                val.append(v)
  +
  +        else:
  +            val = lfns
  +        return val
   
       def path2pfn(self, path):
           """
  
verification_info: {"golden_diff": "diff --git a/lib/rucio/rse/protocols/storm.py b/lib/rucio/rse/protocols/storm.py\n--- a/lib/rucio/rse/protocols/storm.py\n+++ b/lib/rucio/rse/protocols/storm.py\n@@ -43,8 +43,23 @@\n         return '%s/%s' % (scope, name)\n \n     def lfns2pfns(self, lfns):\n-        \"\"\" In this case, just returns back lfn. \"\"\"\n-        return lfns\n+        \"\"\" In this case, just returns back lfn with external scope. \"\"\"\n+        if type(lfns) == dict:\n+            val = lfns.copy()\n+            if 'scope' in val and val['scope'] is not None:\n+                val['scope'] = val['scope'].external\n+\n+        elif type(lfns) == list:\n+            val = []\n+            for l in lfns:\n+                v = l.copy()\n+                if 'scope' in v and v['scope'] is not None:\n+                    v['scope'] = v['scope'].external\n+                val.append(v)\n+\n+        else:\n+            val = lfns\n+        return val\n \n     def path2pfn(self, path):\n         \"\"\"\n", "issue": "1.20.4rc2 storm protocol bug\nMotivation\r\n----------\r\nThe Storm protocol in RSEManager returns the input lfn as the pfn in lfns2pfns. This causes a crash as an InternalScope is then used as a dictionary key in list_replicas.\r\n\r\nModification\r\n------------\r\nThe lfns dictionary should be sanitised so that scope is returned as an external string.\r\n\n", "before_files": [{"content": "# Copyright European Organization for Nuclear Research (CERN)\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# You may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Authors:\n# - Tomas Javor Javurek, <tomas.javurek@cern.ch>, 2019\n\n\nimport os\n\nfrom exceptions import NotImplementedError\nfrom xml.dom import minidom\n\nfrom rucio.common import exception\nfrom rucio.common.utils import run_cmd_process\nfrom rucio.rse.protocols import protocol\n\n\nclass Default(protocol.RSEProtocol):\n    \"\"\" Implementing access to RSEs using the local filesystem.\"\"\"\n\n    def __init__(self, protocol_attr, rse_settings):\n        \"\"\" Initializes the object with information about the referred RSE.\n\n            :param props Properties derived from the RSE Repository\n        \"\"\"\n        super(Default, self).__init__(protocol_attr, rse_settings)\n        self.attributes.pop('determinism_type', None)\n        self.files = []\n\n    def _get_path(self, scope, name):\n        \"\"\" Transforms the physical file name into the local URI in the referred RSE.\n            Suitable for sites implementoing the RUCIO naming convention.\n\n            :param name: filename\n            :param scope: scope\n\n            :returns: RSE specific URI of the physical file\n        \"\"\"\n        return '%s/%s' % (scope, name)\n\n    def lfns2pfns(self, lfns):\n        \"\"\" In this case, just returns back lfn. \"\"\"\n        return lfns\n\n    def path2pfn(self, path):\n        \"\"\"\n            Retruns a fully qualified PFN for the file referred by path.\n\n            :param path: The path to the file.\n\n            :returns: Fully qualified PFN.\n\n        \"\"\"\n        return ''.join([self.rse['scheme'], '://%s' % self.rse['hostname'], path])\n\n    def exists(self, pfn):\n        \"\"\" Checks if the requested file is known by the referred RSE.\n\n            :param pfn Physical file name\n\n            :returns: True if the file exists, False if it doesn't\n\n            :raise  ServiceUnavailable\n        \"\"\"\n        raise NotImplementedError\n\n    def connect(self):\n        \"\"\" Establishes the actual connection to the referred RSE.\n\n            :param credentials Provide all necessary information to establish a connection\n                to the referred storage system. Some is loaded from the repository inside the\n                RSE class and some must be provided specific for the SFTP protocol like\n                username, password, private_key, private_key_pass, port.\n                For details about possible additional parameters and details about their usage\n                see the pysftp.Connection() documentation.\n                NOTE: the host parametrer is overwritten with the value provided by the repository\n\n            :raise RSEAccessDenied\n        \"\"\"\n        pass\n\n    def close(self):\n        \"\"\" Closes the connection to RSE.\"\"\"\n        pass\n\n    def get(self, pfn, dest, transfer_timeout=None):\n        \"\"\" Provides access to files stored inside connected the RSE.\n\n            :param pfn Physical file name of requested file\n            :param dest Name and path of the files when stored at the client\n            :param transfer_timeout Transfer timeout (in seconds)\n\n            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound\n         \"\"\"\n\n        # storm prefix needs to be replaced by davs in order to get etag\n        pfn = 'davs' + pfn[5:]\n\n        # retrieve the TURL from the webdav etag, TODO: make it configurable\n        cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % pfn\n        try:\n            rcode, output = run_cmd_process(cmd, timeout=10)\n        except Exception as e:\n            raise exception.ServiceUnavailable('Could not retrieve STORM WebDAV ETag: %s' % str(e))\n        p_output = minidom.parseString(output)\n\n        # we need to strip off the quotation marks and the <timestamp> from the etag\n        # but since we can have multiple underscores, we have to rely on the uniqueness\n        # of the full LFN to make the split\n        target = p_output.getElementsByTagName('d:getetag')[0].childNodes[0].nodeValue.replace('\"', '')\n        target_ending = '_' + target.split('_')[-1]\n        target = target.split(target_ending)[0]\n\n        # make the symlink\n        try:\n            os.symlink(target, dest)\n        except Exception as e:\n            exception.ServiceUnavailable('Could not create symlink: %s for target %s' % (str(e), str(target)))\n\n    def put(self, source, target, source_dir=None, transfer_timeout=None):\n        \"\"\" Allows to store files inside the referred RSE.\n\n            :param source Physical file name\n            :param target Name of the file on the storage system e.g. with prefixed scope\n            :param source_dir Path where the to be transferred files are stored in the local file system\n            :param transfer_timeout Transfer timeout (in seconds)\n\n            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound\n            \"\"\"\n        raise NotImplementedError\n\n    def delete(self, pfn):\n        \"\"\" Deletes a file from the connected RSE.\n\n            :param pfn Physical file name\n\n            :raises ServiceUnavailable, SourceNotFound\n        \"\"\"\n        raise NotImplementedError\n\n    def rename(self, pfn, new_pfn):\n        \"\"\" Allows to rename a file stored inside the connected RSE.\n\n            :param pfn      Current physical file name\n            :param new_pfn  New physical file name\n\n            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound\n        \"\"\"\n        raise NotImplementedError\n", "path": "lib/rucio/rse/protocols/storm.py"}], "after_files": [{"content": "# Copyright European Organization for Nuclear Research (CERN)\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# You may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Authors:\n# - Tomas Javor Javurek, <tomas.javurek@cern.ch>, 2019\n\n\nimport os\n\nfrom exceptions import NotImplementedError\nfrom xml.dom import minidom\n\nfrom rucio.common import exception\nfrom rucio.common.utils import run_cmd_process\nfrom rucio.rse.protocols import protocol\n\n\nclass Default(protocol.RSEProtocol):\n    \"\"\" Implementing access to RSEs using the local filesystem.\"\"\"\n\n    def __init__(self, protocol_attr, rse_settings):\n        \"\"\" Initializes the object with information about the referred RSE.\n\n            :param props Properties derived from the RSE Repository\n        \"\"\"\n        super(Default, self).__init__(protocol_attr, rse_settings)\n        self.attributes.pop('determinism_type', None)\n        self.files = []\n\n    def _get_path(self, scope, name):\n        \"\"\" Transforms the physical file name into the local URI in the referred RSE.\n            Suitable for sites implementoing the RUCIO naming convention.\n\n            :param name: filename\n            :param scope: scope\n\n            :returns: RSE specific URI of the physical file\n        \"\"\"\n        return '%s/%s' % (scope, name)\n\n    def lfns2pfns(self, lfns):\n        \"\"\" In this case, just returns back lfn with external scope. \"\"\"\n        if type(lfns) == dict:\n            val = lfns.copy()\n            if 'scope' in val and val['scope'] is not None:\n                val['scope'] = val['scope'].external\n\n        elif type(lfns) == list:\n            val = []\n            for l in lfns:\n                v = l.copy()\n                if 'scope' in v and v['scope'] is not None:\n                    v['scope'] = v['scope'].external\n                val.append(v)\n\n        else:\n            val = lfns\n        return val\n\n    def path2pfn(self, path):\n        \"\"\"\n            Retruns a fully qualified PFN for the file referred by path.\n\n            :param path: The path to the file.\n\n            :returns: Fully qualified PFN.\n\n        \"\"\"\n        return ''.join([self.rse['scheme'], '://%s' % self.rse['hostname'], path])\n\n    def exists(self, pfn):\n        \"\"\" Checks if the requested file is known by the referred RSE.\n\n            :param pfn Physical file name\n\n            :returns: True if the file exists, False if it doesn't\n\n            :raise  ServiceUnavailable\n        \"\"\"\n        raise NotImplementedError\n\n    def connect(self):\n        \"\"\" Establishes the actual connection to the referred RSE.\n\n            :param credentials Provide all necessary information to establish a connection\n                to the referred storage system. Some is loaded from the repository inside the\n                RSE class and some must be provided specific for the SFTP protocol like\n                username, password, private_key, private_key_pass, port.\n                For details about possible additional parameters and details about their usage\n                see the pysftp.Connection() documentation.\n                NOTE: the host parametrer is overwritten with the value provided by the repository\n\n            :raise RSEAccessDenied\n        \"\"\"\n        pass\n\n    def close(self):\n        \"\"\" Closes the connection to RSE.\"\"\"\n        pass\n\n    def get(self, pfn, dest, transfer_timeout=None):\n        \"\"\" Provides access to files stored inside connected the RSE.\n\n            :param pfn Physical file name of requested file\n            :param dest Name and path of the files when stored at the client\n            :param transfer_timeout Transfer timeout (in seconds)\n\n            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound\n         \"\"\"\n\n        # storm prefix needs to be replaced by davs in order to get etag\n        pfn = 'davs' + pfn[5:]\n\n        # retrieve the TURL from the webdav etag, TODO: make it configurable\n        cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % pfn\n        try:\n            rcode, output = run_cmd_process(cmd, timeout=10)\n        except Exception as e:\n            raise exception.ServiceUnavailable('Could not retrieve STORM WebDAV ETag: %s' % str(e))\n        p_output = minidom.parseString(output)\n\n        # we need to strip off the quotation marks and the <timestamp> from the etag\n        # but since we can have multiple underscores, we have to rely on the uniqueness\n        # of the full LFN to make the split\n        target = p_output.getElementsByTagName('d:getetag')[0].childNodes[0].nodeValue.replace('\"', '')\n        target_ending = '_' + target.split('_')[-1]\n        target = target.split(target_ending)[0]\n\n        # make the symlink\n        try:\n            os.symlink(target, dest)\n        except Exception as e:\n            exception.ServiceUnavailable('Could not create symlink: %s for target %s' % (str(e), str(target)))\n\n    def put(self, source, target, source_dir=None, transfer_timeout=None):\n        \"\"\" Allows to store files inside the referred RSE.\n\n            :param source Physical file name\n            :param target Name of the file on the storage system e.g. with prefixed scope\n            :param source_dir Path where the to be transferred files are stored in the local file system\n            :param transfer_timeout Transfer timeout (in seconds)\n\n            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound\n            \"\"\"\n        raise NotImplementedError\n\n    def delete(self, pfn):\n        \"\"\" Deletes a file from the connected RSE.\n\n            :param pfn Physical file name\n\n            :raises ServiceUnavailable, SourceNotFound\n        \"\"\"\n        raise NotImplementedError\n\n    def rename(self, pfn, new_pfn):\n        \"\"\" Allows to rename a file stored inside the connected RSE.\n\n            :param pfn      Current physical file name\n            :param new_pfn  New physical file name\n\n            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound\n        \"\"\"\n        raise NotImplementedError\n", "path": "lib/rucio/rse/protocols/storm.py"}]}