We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
get_url_for_time_range function in stereo.py in dataretriever not working correctly.
The following query:-

``` python
from sunpy.time.timerange import TimeRange
from sunpy.net.vso.attrs import Time, Instrument
from sunpy.net.dataretriever.client import QueryResponse
import sunpy.net.dataretriever.sources.stereo as stereo

LCClient = stereo.HETClient()
urls = LCClient._get_url_for_timerange(TimeRange('2008/12/01','2010/12/01'),'ahead', 15*u.min)

```

Should return a non-empty list of urls but instead returns an empty list. Possible problem stems from the implementation of scraper.py in sunpy.util. The scraper doesn't work as intended on http://www.srl.caltech.edu/STEREO/DATA/HET.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `sunpy/util/scraper.py`
Content:
```
1 from __future__ import absolute_import, division, print_function
2 
3 import os
4 import datetime
5 import re
6 
7 from bs4 import BeautifulSoup
8 from sunpy.extern import six
9 from sunpy.extern.six.moves import range, zip
10 from sunpy.extern.six.moves.urllib.request import urlopen
11 
12 __all__ = ['Scraper']
13 
14 # regular expressions to convert datetime format
15 TIME_CONVERSIONS = {'%Y': '\d{4}', '%y': '\d{2}',
16                     '%b': '[A-Z]..', '%B': '\W', '%m': '\d{2}',
17                     '%d': '\d{2}', '%j': '\d{3}',
18                     '%H': '\d{2}', '%I': '\d{2}',
19                     '%M': '\d{2}',
20                     '%S': '\d{2}'}
21 
22 class Scraper(object):
23     """
24     A Scraper to scrap web data archives based on dates.
25 
26     Parameters
27     ----------
28     pattern : string
29         A string containing the url with the date encoded as
30         datetime formats, and any other parameter as kwargs
31         as string format.
32 
33     Attributes
34     ----------
35     pattern : string
36         A converted string with the kwargs.
37     now : datetime.datetime
38         The pattern with the actual date.
39 
40     Examples
41     --------
42     >>> # Downloading data from SolarMonitor.org
43     >>> from sunpy.util.scraper import Scraper
44     >>> solmon_pattern = ('http://solarmonitor.org/data/'
45                           '%Y/%m/%d/fits/{instrument}/'
46                           '{instrument}_{wave:05d}_fd_%Y%m%d_%H%M%S.fts.gz')
47     >>> solmon = Scraper(solmon_pattern, instrument = 'swap', wave = 174)
48     >>> print(solmon.pattern)
49     http://solarmonitor.org/data/%Y/%m/%d/fits/swap/swap_00174_fd_%Y%m%d_%H%M%S.fts.gz
50     >>> print(solmon.now)
51     http://solarmonitor.org/data/2012/01/25/fits/swap/swap_00174_fd_20120125_173301.fts.gz
52 
53     Notes
54     -----
55     The now attribute does not return an existent file, but just how the
56     pattern looks with the actual time.
57     """
58     def __init__(self, pattern, **kwargs):
59         self.pattern = pattern.format(**kwargs)
60         self.now = datetime.datetime.now().strftime(self.pattern)
61 
62     def matches(self, filepath, date):
63         return date.strftime(self.pattern) == filepath
64 
65     def range(self, timerange):
66         """
67         Gets the directories for a certain range of time
68         (i.e. using `~sunpy.time.TimeRange`).
69 
70         Parameters
71         ----------
72 
73         timerange : `~sunpy.time.timerange.TimeRange`
74             Time interval where to find the directories for a given
75             pattern.
76 
77         Returns
78         -------
79 
80         directories : list of strings
81             List of all the possible directories valid for the time
82             range given. Notice that these directories may not exist
83             in the archive.
84         """
85         #find directory structure - without file names
86         directorypattern = os.path.dirname(self.pattern) + '/'
87         #TODO what if there's not slashes?
88         rangedelta = timerange.dt
89         timestep = self._smallerPattern(directorypattern)
90         if timestep is None:
91             return [directorypattern]
92         else:
93             # Number of elements in the time range (including end)
94             n_steps = rangedelta.total_seconds()/timestep.total_seconds()
95             TotalTimeElements = int(round(n_steps)) + 1
96             directories = [(timerange.start + n * timestep).strftime(directorypattern)
97                         for n in range(TotalTimeElements)] #todo if date <= endate
98             return directories
99 
100     def _URL_followsPattern(self, url):
101         """Check whether the url provided follows the pattern"""
102         pattern = self.pattern
103         for k,v in six.iteritems(TIME_CONVERSIONS):
104             pattern = pattern.replace(k, v)
105         matches = re.match(pattern, url)
106         if matches:
107             return matches.end() == matches.endpos == len(self.now)
108         return False
109 
110     def _extractDateURL(self, url):
111         """Extracts the date from a particular url following the pattern"""
112         # url_to_list substitutes '.' and '_' for '/' to then create
113         # a list of all the blocks in times - assuming they are all
114         # separated with either '.', '_' or '/'
115         url_to_list = lambda txt: re.sub(r'\.|_', '/', txt).split('/')
116         pattern_list = url_to_list(self.pattern)
117         url_list = url_to_list(url)
118 
119         time_order = ['%Y', '%y', '%b', '%B', '%m', '%d', '%j',
120                       '%H', '%I', '%M', '%S']
121         final_date = []
122         final_pattern = []
123         # Find in directory and filename
124         for pattern_elem, url_elem in zip(pattern_list, url_list):
125             time_formats = [x for x in time_order if x in pattern_elem]
126             if len(time_formats) > 0:
127                 final_date.append(url_elem)
128                 final_pattern.append(pattern_elem)
129                 for time_bit in time_formats:
130                     time_order.remove(time_bit)
131         # Find and remove repeated elements eg: %Y in ['%Y', '%Y%m%d']
132         #   Make all as single strings
133         date_together = ''.join(final_date)
134         pattern_together = ''.join(final_pattern)
135         re_together = pattern_together
136         for k, v in six.iteritems(TIME_CONVERSIONS):
137             re_together = re_together.replace(k, v)
138 
139         #   Create new empty lists
140         final_date = list()
141         final_pattern = list()
142         for p,r in zip(pattern_together.split('%')[1:], re_together.split('\\')[1:]):
143             regexp = '\\{}'.format(r)
144             pattern = '%{}'.format(p)
145             date_part = re.match(regexp, date_together)
146             date_together = date_together[:date_part.start()] + \
147                             date_together[date_part.end():]
148             if pattern not in final_pattern:
149                 final_pattern.append('%{}'.format(p))
150                 final_date.append(date_part.group())
151         return datetime.datetime.strptime(' '.join(final_date),
152                                           ' '.join(final_pattern))
153 
154     def filelist(self, timerange):
155         """
156         Returns the list of existent files in the archive for the
157         given time range.
158 
159         Parameters
160         ----------
161 
162         timerange : `~sunpy.time.TimeRange`
163             Time interval where to find the directories for a given
164             pattern.
165 
166         Returns
167         -------
168 
169         filesurls : list of strings
170             List of all the files found between the time range given.
171 
172         Examples
173         --------
174         >>> from sunpy.time import TimeRange
175         >>> timerange = TimeRange('2015-01-01','2015-01-01T16:00:00')
176         >>> print(solmon.filelist(timerange))
177         ['http://solarmonitor.org/data/2015/01/01/fits/swap/swap_00174_fd_20150101_025423.fts.gz']
178         """
179         directories = self.range(timerange)
180         filesurls = []
181         for directory in directories:
182             try:
183                 opn = urlopen(directory)
184                 try:
185                     soup = BeautifulSoup(opn)
186                     for link in soup.find_all("a"):
187                         href = link.get("href")
188                         if href.endswith(self.pattern.split('.')[-1]):
189                             fullpath = directory + href
190                             if self._URL_followsPattern(fullpath):
191                                 datehref = self._extractDateURL(fullpath)
192                                 if (datehref >= timerange.start and
193                                     datehref <= timerange.end):
194                                     filesurls.append(fullpath)
195                 finally:
196                     opn.close()
197             except:
198                 pass
199         return filesurls
200 
201     def _smallerPattern(self, directoryPattern):
202         """Obtain the smaller time step for the given pattern"""
203         try:
204             if "%S" in directoryPattern:
205                 return datetime.timedelta(seconds=1)
206             elif "%M" in directoryPattern:
207                 return datetime.timedelta(minutes=1)
208             elif any(hour in directoryPattern for hour in ["%H", "%I"]):
209                 return datetime.timedelta(hours=1)
210             elif any(day in directoryPattern for day in ["%d", "%j"]):
211                 return datetime.timedelta(days=1)
212             elif any(month in directoryPattern for month in ["%b","%B","%m"]):
213                 return datetime.timedelta(days=31)
214             elif any(year in directoryPattern for year in ["%Y", "%y"]):
215                 return datetime.timedelta(days=365)
216             else:
217                 return None
218         except:
219             raise
220 
221
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/sunpy/util/scraper.py b/sunpy/util/scraper.py
--- a/sunpy/util/scraper.py
+++ b/sunpy/util/scraper.py
@@ -12,12 +12,13 @@
 __all__ = ['Scraper']
 
 # regular expressions to convert datetime format
+# added `%e` as for milliseconds `%f/1000`
 TIME_CONVERSIONS = {'%Y': '\d{4}', '%y': '\d{2}',
-                    '%b': '[A-Z]..', '%B': '\W', '%m': '\d{2}',
+                    '%b': '[A-Z][a-z]{2}', '%B': '\W', '%m': '\d{2}',
                     '%d': '\d{2}', '%j': '\d{3}',
                     '%H': '\d{2}', '%I': '\d{2}',
                     '%M': '\d{2}',
-                    '%S': '\d{2}'}
+                    '%S': '\d{2}', '%e': '\d{3}', '%f': '\d{6}'}
 
 class Scraper(object):
     """
@@ -57,7 +58,13 @@
     """
     def __init__(self, pattern, **kwargs):
         self.pattern = pattern.format(**kwargs)
-        self.now = datetime.datetime.now().strftime(self.pattern)
+        milliseconds = re.search('\%e', self.pattern)
+        if not milliseconds:
+            self.now = datetime.datetime.now().strftime(self.pattern)
+        else:
+            now = datetime.datetime.now()
+            milliseconds_ = int(now.microsecond / 1000.)
+            self.now = now.strftime(self.pattern[0:milliseconds.start()] + str(milliseconds_) + self.pattern[milliseconds.end():])
 
     def matches(self, filepath, date):
         return date.strftime(self.pattern) == filepath
@@ -115,9 +122,8 @@
         url_to_list = lambda txt: re.sub(r'\.|_', '/', txt).split('/')
         pattern_list = url_to_list(self.pattern)
         url_list = url_to_list(url)
-
         time_order = ['%Y', '%y', '%b', '%B', '%m', '%d', '%j',
-                      '%H', '%I', '%M', '%S']
+                      '%H', '%I', '%M', '%S', '%e', '%f']
         final_date = []
         final_pattern = []
         # Find in directory and filename
@@ -139,10 +145,13 @@
         #   Create new empty lists
         final_date = list()
         final_pattern = list()
+        re_together = re_together.replace('[A-Z]', '\\[A-Z]')
         for p,r in zip(pattern_together.split('%')[1:], re_together.split('\\')[1:]):
-            regexp = '\\{}'.format(r)
+            if p == 'e':
+                continue
+            regexp = '\\{}'.format(r) if not r.startswith('[') else r
             pattern = '%{}'.format(p)
-            date_part = re.match(regexp, date_together)
+            date_part = re.search(regexp, date_together)
             date_together = date_together[:date_part.start()] + \
                             date_together[date_part.end():]
             if pattern not in final_pattern:
@@ -182,7 +191,7 @@
             try:
                 opn = urlopen(directory)
                 try:
-                    soup = BeautifulSoup(opn)
+                    soup = BeautifulSoup(opn, "lxml")
                     for link in soup.find_all("a"):
                         href = link.get("href")
                         if href.endswith(self.pattern.split('.')[-1]):

{"golden_diff": "diff --git a/sunpy/util/scraper.py b/sunpy/util/scraper.py\n--- a/sunpy/util/scraper.py\n+++ b/sunpy/util/scraper.py\n@@ -12,12 +12,13 @@\n __all__ = ['Scraper']\n \n # regular expressions to convert datetime format\n+# added `%e` as for milliseconds `%f/1000`\n TIME_CONVERSIONS = {'%Y': '\\d{4}', '%y': '\\d{2}',\n-                    '%b': '[A-Z]..', '%B': '\\W', '%m': '\\d{2}',\n+                    '%b': '[A-Z][a-z]{2}', '%B': '\\W', '%m': '\\d{2}',\n                     '%d': '\\d{2}', '%j': '\\d{3}',\n                     '%H': '\\d{2}', '%I': '\\d{2}',\n                     '%M': '\\d{2}',\n-                    '%S': '\\d{2}'}\n+                    '%S': '\\d{2}', '%e': '\\d{3}', '%f': '\\d{6}'}\n \n class Scraper(object):\n     \"\"\"\n@@ -57,7 +58,13 @@\n     \"\"\"\n     def __init__(self, pattern, **kwargs):\n         self.pattern = pattern.format(**kwargs)\n-        self.now = datetime.datetime.now().strftime(self.pattern)\n+        milliseconds = re.search('\\%e', self.pattern)\n+        if not milliseconds:\n+            self.now = datetime.datetime.now().strftime(self.pattern)\n+        else:\n+            now = datetime.datetime.now()\n+            milliseconds_ = int(now.microsecond / 1000.)\n+            self.now = now.strftime(self.pattern[0:milliseconds.start()] + str(milliseconds_) + self.pattern[milliseconds.end():])\n \n     def matches(self, filepath, date):\n         return date.strftime(self.pattern) == filepath\n@@ -115,9 +122,8 @@\n         url_to_list = lambda txt: re.sub(r'\\.|_', '/', txt).split('/')\n         pattern_list = url_to_list(self.pattern)\n         url_list = url_to_list(url)\n-\n         time_order = ['%Y', '%y', '%b', '%B', '%m', '%d', '%j',\n-                      '%H', '%I', '%M', '%S']\n+                      '%H', '%I', '%M', '%S', '%e', '%f']\n         final_date = []\n         final_pattern = []\n         # Find in directory and filename\n@@ -139,10 +145,13 @@\n         #   Create new empty lists\n         final_date = list()\n         final_pattern = list()\n+        re_together = re_together.replace('[A-Z]', '\\\\[A-Z]')\n         for p,r in zip(pattern_together.split('%')[1:], re_together.split('\\\\')[1:]):\n-            regexp = '\\\\{}'.format(r)\n+            if p == 'e':\n+                continue\n+            regexp = '\\\\{}'.format(r) if not r.startswith('[') else r\n             pattern = '%{}'.format(p)\n-            date_part = re.match(regexp, date_together)\n+            date_part = re.search(regexp, date_together)\n             date_together = date_together[:date_part.start()] + \\\n                             date_together[date_part.end():]\n             if pattern not in final_pattern:\n@@ -182,7 +191,7 @@\n             try:\n                 opn = urlopen(directory)\n                 try:\n-                    soup = BeautifulSoup(opn)\n+                    soup = BeautifulSoup(opn, \"lxml\")\n                     for link in soup.find_all(\"a\"):\n                         href = link.get(\"href\")\n                         if href.endswith(self.pattern.split('.')[-1]):\n", "issue": "get_url_for_time_range function in stereo.py in dataretriever not working correctly.\nThe following query:-\n\n``` python\nfrom sunpy.time.timerange import TimeRange\nfrom sunpy.net.vso.attrs import Time, Instrument\nfrom sunpy.net.dataretriever.client import QueryResponse\nimport sunpy.net.dataretriever.sources.stereo as stereo\n\nLCClient = stereo.HETClient()\nurls = LCClient._get_url_for_timerange(TimeRange('2008/12/01','2010/12/01'),'ahead', 15*u.min)\n\n```\n\nShould return a non-empty list of urls but instead returns an empty list. Possible problem stems from the implementation of scraper.py in sunpy.util. The scraper doesn't work as intended on http://www.srl.caltech.edu/STEREO/DATA/HET.\n\n", "before_files": [{"content": "from __future__ import absolute_import, division, print_function\n\nimport os\nimport datetime\nimport re\n\nfrom bs4 import BeautifulSoup\nfrom sunpy.extern import six\nfrom sunpy.extern.six.moves import range, zip\nfrom sunpy.extern.six.moves.urllib.request import urlopen\n\n__all__ = ['Scraper']\n\n# regular expressions to convert datetime format\nTIME_CONVERSIONS = {'%Y': '\\d{4}', '%y': '\\d{2}',\n                    '%b': '[A-Z]..', '%B': '\\W', '%m': '\\d{2}',\n                    '%d': '\\d{2}', '%j': '\\d{3}',\n                    '%H': '\\d{2}', '%I': '\\d{2}',\n                    '%M': '\\d{2}',\n                    '%S': '\\d{2}'}\n\nclass Scraper(object):\n    \"\"\"\n    A Scraper to scrap web data archives based on dates.\n\n    Parameters\n    ----------\n    pattern : string\n        A string containing the url with the date encoded as\n        datetime formats, and any other parameter as kwargs\n        as string format.\n\n    Attributes\n    ----------\n    pattern : string\n        A converted string with the kwargs.\n    now : datetime.datetime\n        The pattern with the actual date.\n\n    Examples\n    --------\n    >>> # Downloading data from SolarMonitor.org\n    >>> from sunpy.util.scraper import Scraper\n    >>> solmon_pattern = ('http://solarmonitor.org/data/'\n                          '%Y/%m/%d/fits/{instrument}/'\n                          '{instrument}_{wave:05d}_fd_%Y%m%d_%H%M%S.fts.gz')\n    >>> solmon = Scraper(solmon_pattern, instrument = 'swap', wave = 174)\n    >>> print(solmon.pattern)\n    http://solarmonitor.org/data/%Y/%m/%d/fits/swap/swap_00174_fd_%Y%m%d_%H%M%S.fts.gz\n    >>> print(solmon.now)\n    http://solarmonitor.org/data/2012/01/25/fits/swap/swap_00174_fd_20120125_173301.fts.gz\n\n    Notes\n    -----\n    The now attribute does not return an existent file, but just how the\n    pattern looks with the actual time.\n    \"\"\"\n    def __init__(self, pattern, **kwargs):\n        self.pattern = pattern.format(**kwargs)\n        self.now = datetime.datetime.now().strftime(self.pattern)\n\n    def matches(self, filepath, date):\n        return date.strftime(self.pattern) == filepath\n\n    def range(self, timerange):\n        \"\"\"\n        Gets the directories for a certain range of time\n        (i.e. using `~sunpy.time.TimeRange`).\n\n        Parameters\n        ----------\n\n        timerange : `~sunpy.time.timerange.TimeRange`\n            Time interval where to find the directories for a given\n            pattern.\n\n        Returns\n        -------\n\n        directories : list of strings\n            List of all the possible directories valid for the time\n            range given. Notice that these directories may not exist\n            in the archive.\n        \"\"\"\n        #find directory structure - without file names\n        directorypattern = os.path.dirname(self.pattern) + '/'\n        #TODO what if there's not slashes?\n        rangedelta = timerange.dt\n        timestep = self._smallerPattern(directorypattern)\n        if timestep is None:\n            return [directorypattern]\n        else:\n            # Number of elements in the time range (including end)\n            n_steps = rangedelta.total_seconds()/timestep.total_seconds()\n            TotalTimeElements = int(round(n_steps)) + 1\n            directories = [(timerange.start + n * timestep).strftime(directorypattern)\n                        for n in range(TotalTimeElements)] #todo if date <= endate\n            return directories\n\n    def _URL_followsPattern(self, url):\n        \"\"\"Check whether the url provided follows the pattern\"\"\"\n        pattern = self.pattern\n        for k,v in six.iteritems(TIME_CONVERSIONS):\n            pattern = pattern.replace(k, v)\n        matches = re.match(pattern, url)\n        if matches:\n            return matches.end() == matches.endpos == len(self.now)\n        return False\n\n    def _extractDateURL(self, url):\n        \"\"\"Extracts the date from a particular url following the pattern\"\"\"\n        # url_to_list substitutes '.' and '_' for '/' to then create\n        # a list of all the blocks in times - assuming they are all\n        # separated with either '.', '_' or '/'\n        url_to_list = lambda txt: re.sub(r'\\.|_', '/', txt).split('/')\n        pattern_list = url_to_list(self.pattern)\n        url_list = url_to_list(url)\n\n        time_order = ['%Y', '%y', '%b', '%B', '%m', '%d', '%j',\n                      '%H', '%I', '%M', '%S']\n        final_date = []\n        final_pattern = []\n        # Find in directory and filename\n        for pattern_elem, url_elem in zip(pattern_list, url_list):\n            time_formats = [x for x in time_order if x in pattern_elem]\n            if len(time_formats) > 0:\n                final_date.append(url_elem)\n                final_pattern.append(pattern_elem)\n                for time_bit in time_formats:\n                    time_order.remove(time_bit)\n        # Find and remove repeated elements eg: %Y in ['%Y', '%Y%m%d']\n        #   Make all as single strings\n        date_together = ''.join(final_date)\n        pattern_together = ''.join(final_pattern)\n        re_together = pattern_together\n        for k, v in six.iteritems(TIME_CONVERSIONS):\n            re_together = re_together.replace(k, v)\n\n        #   Create new empty lists\n        final_date = list()\n        final_pattern = list()\n        for p,r in zip(pattern_together.split('%')[1:], re_together.split('\\\\')[1:]):\n            regexp = '\\\\{}'.format(r)\n            pattern = '%{}'.format(p)\n            date_part = re.match(regexp, date_together)\n            date_together = date_together[:date_part.start()] + \\\n                            date_together[date_part.end():]\n            if pattern not in final_pattern:\n                final_pattern.append('%{}'.format(p))\n                final_date.append(date_part.group())\n        return datetime.datetime.strptime(' '.join(final_date),\n                                          ' '.join(final_pattern))\n\n    def filelist(self, timerange):\n        \"\"\"\n        Returns the list of existent files in the archive for the\n        given time range.\n\n        Parameters\n        ----------\n\n        timerange : `~sunpy.time.TimeRange`\n            Time interval where to find the directories for a given\n            pattern.\n\n        Returns\n        -------\n\n        filesurls : list of strings\n            List of all the files found between the time range given.\n\n        Examples\n        --------\n        >>> from sunpy.time import TimeRange\n        >>> timerange = TimeRange('2015-01-01','2015-01-01T16:00:00')\n        >>> print(solmon.filelist(timerange))\n        ['http://solarmonitor.org/data/2015/01/01/fits/swap/swap_00174_fd_20150101_025423.fts.gz']\n        \"\"\"\n        directories = self.range(timerange)\n        filesurls = []\n        for directory in directories:\n            try:\n                opn = urlopen(directory)\n                try:\n                    soup = BeautifulSoup(opn)\n                    for link in soup.find_all(\"a\"):\n                        href = link.get(\"href\")\n                        if href.endswith(self.pattern.split('.')[-1]):\n                            fullpath = directory + href\n                            if self._URL_followsPattern(fullpath):\n                                datehref = self._extractDateURL(fullpath)\n                                if (datehref >= timerange.start and\n                                    datehref <= timerange.end):\n                                    filesurls.append(fullpath)\n                finally:\n                    opn.close()\n            except:\n                pass\n        return filesurls\n\n    def _smallerPattern(self, directoryPattern):\n        \"\"\"Obtain the smaller time step for the given pattern\"\"\"\n        try:\n            if \"%S\" in directoryPattern:\n                return datetime.timedelta(seconds=1)\n            elif \"%M\" in directoryPattern:\n                return datetime.timedelta(minutes=1)\n            elif any(hour in directoryPattern for hour in [\"%H\", \"%I\"]):\n                return datetime.timedelta(hours=1)\n            elif any(day in directoryPattern for day in [\"%d\", \"%j\"]):\n                return datetime.timedelta(days=1)\n            elif any(month in directoryPattern for month in [\"%b\",\"%B\",\"%m\"]):\n                return datetime.timedelta(days=31)\n            elif any(year in directoryPattern for year in [\"%Y\", \"%y\"]):\n                return datetime.timedelta(days=365)\n            else:\n                return None\n        except:\n            raise\n\n", "path": "sunpy/util/scraper.py"}], "after_files": [{"content": "from __future__ import absolute_import, division, print_function\n\nimport os\nimport datetime\nimport re\n\nfrom bs4 import BeautifulSoup\nfrom sunpy.extern import six\nfrom sunpy.extern.six.moves import range, zip\nfrom sunpy.extern.six.moves.urllib.request import urlopen\n\n__all__ = ['Scraper']\n\n# regular expressions to convert datetime format\n# added `%e` as for milliseconds `%f/1000`\nTIME_CONVERSIONS = {'%Y': '\\d{4}', '%y': '\\d{2}',\n                    '%b': '[A-Z][a-z]{2}', '%B': '\\W', '%m': '\\d{2}',\n                    '%d': '\\d{2}', '%j': '\\d{3}',\n                    '%H': '\\d{2}', '%I': '\\d{2}',\n                    '%M': '\\d{2}',\n                    '%S': '\\d{2}', '%e': '\\d{3}', '%f': '\\d{6}'}\n\nclass Scraper(object):\n    \"\"\"\n    A Scraper to scrap web data archives based on dates.\n\n    Parameters\n    ----------\n    pattern : string\n        A string containing the url with the date encoded as\n        datetime formats, and any other parameter as kwargs\n        as string format.\n\n    Attributes\n    ----------\n    pattern : string\n        A converted string with the kwargs.\n    now : datetime.datetime\n        The pattern with the actual date.\n\n    Examples\n    --------\n    >>> # Downloading data from SolarMonitor.org\n    >>> from sunpy.util.scraper import Scraper\n    >>> solmon_pattern = ('http://solarmonitor.org/data/'\n                          '%Y/%m/%d/fits/{instrument}/'\n                          '{instrument}_{wave:05d}_fd_%Y%m%d_%H%M%S.fts.gz')\n    >>> solmon = Scraper(solmon_pattern, instrument = 'swap', wave = 174)\n    >>> print(solmon.pattern)\n    http://solarmonitor.org/data/%Y/%m/%d/fits/swap/swap_00174_fd_%Y%m%d_%H%M%S.fts.gz\n    >>> print(solmon.now)\n    http://solarmonitor.org/data/2012/01/25/fits/swap/swap_00174_fd_20120125_173301.fts.gz\n\n    Notes\n    -----\n    The now attribute does not return an existent file, but just how the\n    pattern looks with the actual time.\n    \"\"\"\n    def __init__(self, pattern, **kwargs):\n        self.pattern = pattern.format(**kwargs)\n        milliseconds = re.search('\\%e', self.pattern)\n        if not milliseconds:\n            self.now = datetime.datetime.now().strftime(self.pattern)\n        else:\n            now = datetime.datetime.now()\n            milliseconds_ = int(now.microsecond / 1000.)\n            self.now = now.strftime(self.pattern[0:milliseconds.start()] + str(milliseconds_) + self.pattern[milliseconds.end():])\n\n    def matches(self, filepath, date):\n        return date.strftime(self.pattern) == filepath\n\n    def range(self, timerange):\n        \"\"\"\n        Gets the directories for a certain range of time\n        (i.e. using `~sunpy.time.TimeRange`).\n\n        Parameters\n        ----------\n\n        timerange : `~sunpy.time.timerange.TimeRange`\n            Time interval where to find the directories for a given\n            pattern.\n\n        Returns\n        -------\n\n        directories : list of strings\n            List of all the possible directories valid for the time\n            range given. Notice that these directories may not exist\n            in the archive.\n        \"\"\"\n        #find directory structure - without file names\n        directorypattern = os.path.dirname(self.pattern) + '/'\n        #TODO what if there's not slashes?\n        rangedelta = timerange.dt\n        timestep = self._smallerPattern(directorypattern)\n        if timestep is None:\n            return [directorypattern]\n        else:\n            # Number of elements in the time range (including end)\n            n_steps = rangedelta.total_seconds()/timestep.total_seconds()\n            TotalTimeElements = int(round(n_steps)) + 1\n            directories = [(timerange.start + n * timestep).strftime(directorypattern)\n                        for n in range(TotalTimeElements)] #todo if date <= endate\n            return directories\n\n    def _URL_followsPattern(self, url):\n        \"\"\"Check whether the url provided follows the pattern\"\"\"\n        pattern = self.pattern\n        for k,v in six.iteritems(TIME_CONVERSIONS):\n            pattern = pattern.replace(k, v)\n        matches = re.match(pattern, url)\n        if matches:\n            return matches.end() == matches.endpos == len(self.now)\n        return False\n\n    def _extractDateURL(self, url):\n        \"\"\"Extracts the date from a particular url following the pattern\"\"\"\n        # url_to_list substitutes '.' and '_' for '/' to then create\n        # a list of all the blocks in times - assuming they are all\n        # separated with either '.', '_' or '/'\n        url_to_list = lambda txt: re.sub(r'\\.|_', '/', txt).split('/')\n        pattern_list = url_to_list(self.pattern)\n        url_list = url_to_list(url)\n        time_order = ['%Y', '%y', '%b', '%B', '%m', '%d', '%j',\n                      '%H', '%I', '%M', '%S', '%e', '%f']\n        final_date = []\n        final_pattern = []\n        # Find in directory and filename\n        for pattern_elem, url_elem in zip(pattern_list, url_list):\n            time_formats = [x for x in time_order if x in pattern_elem]\n            if len(time_formats) > 0:\n                final_date.append(url_elem)\n                final_pattern.append(pattern_elem)\n                for time_bit in time_formats:\n                    time_order.remove(time_bit)\n        # Find and remove repeated elements eg: %Y in ['%Y', '%Y%m%d']\n        #   Make all as single strings\n        date_together = ''.join(final_date)\n        pattern_together = ''.join(final_pattern)\n        re_together = pattern_together\n        for k, v in six.iteritems(TIME_CONVERSIONS):\n            re_together = re_together.replace(k, v)\n\n        #   Create new empty lists\n        final_date = list()\n        final_pattern = list()\n        re_together = re_together.replace('[A-Z]', '\\\\[A-Z]')\n        for p,r in zip(pattern_together.split('%')[1:], re_together.split('\\\\')[1:]):\n            if p == 'e':\n                continue\n            regexp = '\\\\{}'.format(r) if not r.startswith('[') else r\n            pattern = '%{}'.format(p)\n            date_part = re.search(regexp, date_together)\n            date_together = date_together[:date_part.start()] + \\\n                            date_together[date_part.end():]\n            if pattern not in final_pattern:\n                final_pattern.append('%{}'.format(p))\n                final_date.append(date_part.group())\n        return datetime.datetime.strptime(' '.join(final_date),\n                                          ' '.join(final_pattern))\n\n    def filelist(self, timerange):\n        \"\"\"\n        Returns the list of existent files in the archive for the\n        given time range.\n\n        Parameters\n        ----------\n\n        timerange : `~sunpy.time.TimeRange`\n            Time interval where to find the directories for a given\n            pattern.\n\n        Returns\n        -------\n\n        filesurls : list of strings\n            List of all the files found between the time range given.\n\n        Examples\n        --------\n        >>> from sunpy.time import TimeRange\n        >>> timerange = TimeRange('2015-01-01','2015-01-01T16:00:00')\n        >>> print(solmon.filelist(timerange))\n        ['http://solarmonitor.org/data/2015/01/01/fits/swap/swap_00174_fd_20150101_025423.fts.gz']\n        \"\"\"\n        directories = self.range(timerange)\n        filesurls = []\n        for directory in directories:\n            try:\n                opn = urlopen(directory)\n                try:\n                    soup = BeautifulSoup(opn, \"lxml\")\n                    for link in soup.find_all(\"a\"):\n                        href = link.get(\"href\")\n                        if href.endswith(self.pattern.split('.')[-1]):\n                            fullpath = directory + href\n                            if self._URL_followsPattern(fullpath):\n                                datehref = self._extractDateURL(fullpath)\n                                if (datehref >= timerange.start and\n                                    datehref <= timerange.end):\n                                    filesurls.append(fullpath)\n                finally:\n                    opn.close()\n            except:\n                pass\n        return filesurls\n\n    def _smallerPattern(self, directoryPattern):\n        \"\"\"Obtain the smaller time step for the given pattern\"\"\"\n        try:\n            if \"%S\" in directoryPattern:\n                return datetime.timedelta(seconds=1)\n            elif \"%M\" in directoryPattern:\n                return datetime.timedelta(minutes=1)\n            elif any(hour in directoryPattern for hour in [\"%H\", \"%I\"]):\n                return datetime.timedelta(hours=1)\n            elif any(day in directoryPattern for day in [\"%d\", \"%j\"]):\n                return datetime.timedelta(days=1)\n            elif any(month in directoryPattern for month in [\"%b\",\"%B\",\"%m\"]):\n                return datetime.timedelta(days=31)\n            elif any(year in directoryPattern for year in [\"%Y\", \"%y\"]):\n                return datetime.timedelta(days=365)\n            else:\n                return None\n        except:\n            raise\n\n", "path": "sunpy/util/scraper.py"}]}