We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Sopel's web lookups should not work with internal addresses like http://192.168.1.1
Example (also reproduced in freenode.net's official sopel bot via /msg command):
[01:51pm] <Ant> http://192.168.1.1
01:51PM <URL> [ 401 Unauthorized ] - 192.168.1.1
[01:51pm] <Ant> http://192.168.100.1
01:51PM <URL> [ Touchstone Status ] - 192.168.100.1

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `sopel/modules/url.py`
Content:
```
1 # coding=utf-8
2 """
3 url.py - Sopel URL Title Module
4 Copyright 2010-2011, Michael Yanovich (yanovich.net) & Kenneth Sham
5 Copyright 2012-2013, Elsie Powell
6 Copyright 2013, Lior Ramati <firerogue517@gmail.com>
7 Copyright 2014, Elad Alfassa <elad@fedoraproject.org>
8 Licensed under the Eiffel Forum License 2.
9 
10 https://sopel.chat
11 """
12 from __future__ import unicode_literals, absolute_import, print_function, division
13 
14 import re
15 
16 import requests
17 
18 from sopel import __version__, module, tools, web
19 from sopel.config.types import ListAttribute, StaticSection, ValidatedAttribute
20 
21 USER_AGENT = 'Sopel/{} (https://sopel.chat)'.format(__version__)
22 default_headers = {'User-Agent': USER_AGENT}
23 # These are used to clean up the title tag before actually parsing it. Not the
24 # world's best way to do this, but it'll do for now.
25 title_tag_data = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)
26 quoted_title = re.compile('[\'"]<title>[\'"]', re.IGNORECASE)
27 # This is another regex that presumably does something important.
28 re_dcc = re.compile(r'(?i)dcc\ssend')
29 # This sets the maximum number of bytes that should be read in order to find
30 # the title. We don't want it too high, or a link to a big file/stream will
31 # just keep downloading until there's no more memory. 640k ought to be enough
32 # for anybody.
33 max_bytes = 655360
34 
35 
36 class UrlSection(StaticSection):
37     # TODO some validation rules maybe?
38     exclude = ListAttribute('exclude')
39     """A list of regular expressions to match URLs for which the title should not be shown."""
40     exclusion_char = ValidatedAttribute('exclusion_char', default='!')
41     """A character (or string) which, when immediately preceding a URL, will stop that URL's title from being shown."""
42     shorten_url_length = ValidatedAttribute(
43         'shorten_url_length', int, default=0)
44     """If greater than 0, the title fetcher will include a TinyURL version of links longer than this many characters."""
45 
46 
47 def configure(config):
48     """
49     | name | example | purpose |
50     | ---- | ------- | ------- |
51     | exclude | https?://git\\\\.io/.* | A list of regular expressions for URLs for which the title should not be shown. |
52     | exclusion\\_char | ! | A character (or string) which, when immediately preceding a URL, will stop the URL's title from being shown. |
53     | shorten\\_url\\_length | 72 | If greater than 0, the title fetcher will include a TinyURL version of links longer than this many characters. |
54     """
55     config.define_section('url', UrlSection)
56     config.url.configure_setting(
57         'exclude',
58         'Enter regular expressions for each URL you would like to exclude.'
59     )
60     config.url.configure_setting(
61         'exclusion_char',
62         'Enter a character which can be prefixed to suppress URL titling'
63     )
64     config.url.configure_setting(
65         'shorten_url_length',
66         'Enter how many characters a URL should be before the bot puts a'
67         ' shorter version of the URL in the title as a TinyURL link'
68         ' (0 to disable)'
69     )
70 
71 
72 def setup(bot):
73     bot.config.define_section('url', UrlSection)
74 
75     if bot.config.url.exclude:
76         regexes = [re.compile(s) for s in bot.config.url.exclude]
77     else:
78         regexes = []
79 
80     # We're keeping these in their own list, rather than putting then in the
81     # callbacks list because 1, it's easier to deal with modules that are still
82     # using this list, and not the newer callbacks list and 2, having a lambda
83     # just to pass is kinda ugly.
84     if 'url_exclude' not in bot.memory:
85         bot.memory['url_exclude'] = regexes
86     else:
87         exclude = bot.memory['url_exclude']
88         if regexes:
89             exclude.extend(regexes)
90         bot.memory['url_exclude'] = exclude
91 
92     # Ensure last_seen_url is in memory
93     if 'last_seen_url' not in bot.memory:
94         bot.memory['last_seen_url'] = tools.SopelMemory()
95 
96     # Initialize shortened_urls as a dict if it doesn't exist.
97     if 'shortened_urls' not in bot.memory:
98         bot.memory['shortened_urls'] = tools.SopelMemory()
99 
100 
101 @module.commands('title')
102 @module.example(
103     '.title https://www.google.com',
104     '[ Google ] - www.google.com',
105     online=True)
106 def title_command(bot, trigger):
107     """
108     Show the title or URL information for the given URL, or the last URL seen
109     in this channel.
110     """
111     if not trigger.group(2):
112         if trigger.sender not in bot.memory['last_seen_url']:
113             return
114         matched = check_callbacks(
115             bot, bot.memory['last_seen_url'][trigger.sender])
116         if matched:
117             return
118         else:
119             urls = [bot.memory['last_seen_url'][trigger.sender]]
120     else:
121         urls = web.search_urls(
122             trigger,
123             exclusion_char=bot.config.url.exclusion_char)
124 
125     for url, title, domain, tinyurl in process_urls(bot, trigger, urls):
126         message = '[ %s ] - %s' % (title, domain)
127         if tinyurl:
128             message += ' ( %s )' % tinyurl
129         bot.reply(message)
130         bot.memory['last_seen_url'][trigger.sender] = url
131 
132 
133 @module.rule(r'(?u).*(https?://\S+).*')
134 def title_auto(bot, trigger):
135     """
136     Automatically show titles for URLs. For shortened URLs/redirects, find
137     where the URL redirects to and show the title for that (or call a function
138     from another module to give more information).
139     """
140     if re.match(bot.config.core.prefix + 'title', trigger):
141         return
142 
143     # Avoid fetching known malicious links
144     if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']:
145         if bot.memory['safety_cache'][trigger]['positives'] > 1:
146             return
147 
148     urls = web.search_urls(
149         trigger, exclusion_char=bot.config.url.exclusion_char, clean=True)
150 
151     for url, title, domain, tinyurl in process_urls(bot, trigger, urls):
152         message = '[ %s ] - %s' % (title, domain)
153         if tinyurl:
154             message += ' ( %s )' % tinyurl
155         # Guard against responding to other instances of this bot.
156         if message != trigger:
157             bot.say(message)
158             bot.memory['last_seen_url'][trigger.sender] = url
159 
160 
161 def process_urls(bot, trigger, urls):
162     """
163     For each URL in the list, ensure that it isn't handled by another module.
164     If not, find where it redirects to, if anywhere. If that redirected URL
165     should be handled by another module, dispatch the callback for it.
166     Return a list of (title, hostname) tuples for each URL which is not handled
167     by another module.
168     """
169     shorten_url_length = bot.config.url.shorten_url_length
170     for url in urls:
171         # Exclude URLs that start with the exclusion char
172         if url.startswith(bot.config.url.exclusion_char):
173             continue
174 
175         # Check the URL does not match an existing URL callback
176         if check_callbacks(bot, url):
177             continue
178 
179         # Call the URL to get a title, if possible
180         title = find_title(url, verify=bot.config.core.verify_ssl)
181         if not title:
182             # No title found: don't handle this URL
183             continue
184 
185         # If the URL is over bot.config.url.shorten_url_length, shorten the URL
186         tinyurl = None
187         if (shorten_url_length > 0) and (len(url) > shorten_url_length):
188             tinyurl = get_or_create_shorturl(bot, url)
189 
190         yield (url, title, get_hostname(url), tinyurl)
191 
192 
193 def check_callbacks(bot, url):
194     """Check if ``url`` is excluded or matches any URL callback patterns.
195 
196     :param bot: Sopel instance
197     :param str url: URL to check
198     :return: True if ``url`` is excluded or matches any URL Callback pattern
199 
200     This function looks at the ``bot.memory`` for ``url_exclude`` patterns and
201     it returns ``True`` if any matches the given ``url``. Otherwise, it looks
202     at the ``bot``'s URL Callback patterns, and it returns ``True`` if any
203     matches, ``False`` otherwise.
204 
205     .. seealso::
206 
207         The :func:`~sopel.modules.url.setup` function that defines the
208         ``url_exclude`` in ``bot.memory``.
209 
210     .. versionchanged:: 7.0
211 
212         This function **does not** trigger URL callbacks anymore when ``url``
213         matches a pattern.
214 
215     """
216     # Check if it matches the exclusion list first
217     matched = any(regex.search(url) for regex in bot.memory['url_exclude'])
218     return matched or any(bot.search_url_callbacks(url))
219 
220 
221 def find_title(url, verify=True):
222     """Return the title for the given URL."""
223     try:
224         response = requests.get(url, stream=True, verify=verify,
225                                 headers=default_headers)
226         content = b''
227         for byte in response.iter_content(chunk_size=512):
228             content += byte
229             if b'</title>' in content or len(content) > max_bytes:
230                 break
231         content = content.decode('utf-8', errors='ignore')
232         # Need to close the connection because we have not read all
233         # the data
234         response.close()
235     except requests.exceptions.ConnectionError:
236         return None
237 
238     # Some cleanup that I don't really grok, but was in the original, so
239     # we'll keep it (with the compiled regexes made global) for now.
240     content = title_tag_data.sub(r'<\1title>', content)
241     content = quoted_title.sub('', content)
242 
243     start = content.rfind('<title>')
244     end = content.rfind('</title>')
245     if start == -1 or end == -1:
246         return
247     title = web.decode(content[start + 7:end])
248     title = title.strip()[:200]
249 
250     title = ' '.join(title.split())  # cleanly remove multiple spaces
251 
252     # More cryptic regex substitutions. This one looks to be myano's invention.
253     title = re_dcc.sub('', title)
254 
255     return title or None
256 
257 
258 def get_hostname(url):
259     idx = 7
260     if url.startswith('https://'):
261         idx = 8
262     elif url.startswith('ftp://'):
263         idx = 6
264     hostname = url[idx:]
265     slash = hostname.find('/')
266     if slash != -1:
267         hostname = hostname[:slash]
268     return hostname
269 
270 
271 def get_or_create_shorturl(bot, url):
272     """Get or create a short URL for ``url``
273 
274     :param bot: Sopel instance
275     :param str url: URL to get or create a short URL for
276     :return: A short URL
277     :rtype: str
278 
279     It gets the short URL for ``url`` from the bot's memory if it exists.
280     Otherwise, it creates a short URL (see :func:`get_tinyurl`), stores it
281     into the bot's memory, then returns it.
282     """
283     # Check bot memory to see if the shortened URL is already in
284     # memory
285     if url in bot.memory['shortened_urls']:
286         return bot.memory['shortened_urls'][url]
287 
288     tinyurl = get_tinyurl(url)
289     bot.memory['shortened_urls'][url] = tinyurl
290     return tinyurl
291 
292 
293 def get_tinyurl(url):
294     """Returns a shortened tinyURL link of the URL"""
295     base_url = "https://tinyurl.com/api-create.php"
296     tinyurl = "%s?%s" % (base_url, web.urlencode({'url': url}))
297     try:
298         res = requests.get(tinyurl)
299         res.raise_for_status()
300     except requests.exceptions.RequestException:
301         return None
302     # Replace text output with https instead of http to make the
303     # result an HTTPS link.
304     return res.text.replace("http://", "https://")
305 
306 
307 if __name__ == "__main__":
308     from sopel.test_tools import run_example_tests
309     run_example_tests(__file__)
310
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/sopel/modules/url.py b/sopel/modules/url.py
--- a/sopel/modules/url.py
+++ b/sopel/modules/url.py
@@ -13,11 +13,19 @@
 
 import re
 
+import dns.resolver
+import ipaddress
 import requests
 
 from sopel import __version__, module, tools, web
 from sopel.config.types import ListAttribute, StaticSection, ValidatedAttribute
 
+# Python3 vs Python2
+try:
+    from urllib.parse import urlparse
+except ImportError:
+    from urlparse import urlparse
+
 USER_AGENT = 'Sopel/{} (https://sopel.chat)'.format(__version__)
 default_headers = {'User-Agent': USER_AGENT}
 # These are used to clean up the title tag before actually parsing it. Not the
@@ -42,6 +50,10 @@
     shorten_url_length = ValidatedAttribute(
         'shorten_url_length', int, default=0)
     """If greater than 0, the title fetcher will include a TinyURL version of links longer than this many characters."""
+    enable_private_resolution = ValidatedAttribute(
+        'enable_private_resolution', bool, default=False)
+    enable_dns_resolution = ValidatedAttribute(
+        'enable_dns_resolution', bool, default=False)
 
 
 def configure(config):
@@ -67,6 +79,14 @@
         ' shorter version of the URL in the title as a TinyURL link'
         ' (0 to disable)'
     )
+    config.url.configure_setting(
+        'enable_private_resolution',
+        'Enable URL lookups for RFC1918 addresses?'
+    )
+    config.url.configure_setting(
+        'enable_dns_resolution',
+        'Enable DNS resolution for all domains to validate if there are RFC1918 resolutions?'
+    )
 
 
 def setup(bot):
@@ -173,6 +193,26 @@
         if check_callbacks(bot, url):
             continue
 
+        # Prevent private addresses form being queried if enable_private_resolution is False
+        if not bot.config.url.enable_private_resolution:
+            parsed = urlparse(url)
+            # Check if it's an address like http://192.168.1.1
+            try:
+                if ipaddress.ip_address(parsed.hostname).is_private or ipaddress.ip_address(parsed.hostname).is_loopback:
+                    continue
+            except ValueError:
+                pass
+
+            # Check if domains are RFC1918 addresses if enable_dns_resolutions is set
+            if bot.config.url.enable_dns_resolution:
+                private = False
+                for result in dns.resolver.query(parsed.hostname):
+                    if ipaddress.ip_address(result).is_private:
+                        private = True
+                        break
+                if private:
+                    continue
+
         # Call the URL to get a title, if possible
         title = find_title(url, verify=bot.config.core.verify_ssl)
         if not title:

{"golden_diff": "diff --git a/sopel/modules/url.py b/sopel/modules/url.py\n--- a/sopel/modules/url.py\n+++ b/sopel/modules/url.py\n@@ -13,11 +13,19 @@\n \n import re\n \n+import dns.resolver\n+import ipaddress\n import requests\n \n from sopel import __version__, module, tools, web\n from sopel.config.types import ListAttribute, StaticSection, ValidatedAttribute\n \n+# Python3 vs Python2\n+try:\n+    from urllib.parse import urlparse\n+except ImportError:\n+    from urlparse import urlparse\n+\n USER_AGENT = 'Sopel/{} (https://sopel.chat)'.format(__version__)\n default_headers = {'User-Agent': USER_AGENT}\n # These are used to clean up the title tag before actually parsing it. Not the\n@@ -42,6 +50,10 @@\n     shorten_url_length = ValidatedAttribute(\n         'shorten_url_length', int, default=0)\n     \"\"\"If greater than 0, the title fetcher will include a TinyURL version of links longer than this many characters.\"\"\"\n+    enable_private_resolution = ValidatedAttribute(\n+        'enable_private_resolution', bool, default=False)\n+    enable_dns_resolution = ValidatedAttribute(\n+        'enable_dns_resolution', bool, default=False)\n \n \n def configure(config):\n@@ -67,6 +79,14 @@\n         ' shorter version of the URL in the title as a TinyURL link'\n         ' (0 to disable)'\n     )\n+    config.url.configure_setting(\n+        'enable_private_resolution',\n+        'Enable URL lookups for RFC1918 addresses?'\n+    )\n+    config.url.configure_setting(\n+        'enable_dns_resolution',\n+        'Enable DNS resolution for all domains to validate if there are RFC1918 resolutions?'\n+    )\n \n \n def setup(bot):\n@@ -173,6 +193,26 @@\n         if check_callbacks(bot, url):\n             continue\n \n+        # Prevent private addresses form being queried if enable_private_resolution is False\n+        if not bot.config.url.enable_private_resolution:\n+            parsed = urlparse(url)\n+            # Check if it's an address like http://192.168.1.1\n+            try:\n+                if ipaddress.ip_address(parsed.hostname).is_private or ipaddress.ip_address(parsed.hostname).is_loopback:\n+                    continue\n+            except ValueError:\n+                pass\n+\n+            # Check if domains are RFC1918 addresses if enable_dns_resolutions is set\n+            if bot.config.url.enable_dns_resolution:\n+                private = False\n+                for result in dns.resolver.query(parsed.hostname):\n+                    if ipaddress.ip_address(result).is_private:\n+                        private = True\n+                        break\n+                if private:\n+                    continue\n+\n         # Call the URL to get a title, if possible\n         title = find_title(url, verify=bot.config.core.verify_ssl)\n         if not title:\n", "issue": "Sopel's web lookups should not work with internal addresses like http://192.168.1.1\nExample (also reproduced in freenode.net's official sopel bot via /msg command):\r\n[01:51pm] <Ant> http://192.168.1.1\r\n01:51PM <URL> [ 401 Unauthorized ] - 192.168.1.1\r\n[01:51pm] <Ant> http://192.168.100.1\r\n01:51PM <URL> [ Touchstone Status ] - 192.168.100.1\n", "before_files": [{"content": "# coding=utf-8\n\"\"\"\nurl.py - Sopel URL Title Module\nCopyright 2010-2011, Michael Yanovich (yanovich.net) & Kenneth Sham\nCopyright 2012-2013, Elsie Powell\nCopyright 2013, Lior Ramati <firerogue517@gmail.com>\nCopyright 2014, Elad Alfassa <elad@fedoraproject.org>\nLicensed under the Eiffel Forum License 2.\n\nhttps://sopel.chat\n\"\"\"\nfrom __future__ import unicode_literals, absolute_import, print_function, division\n\nimport re\n\nimport requests\n\nfrom sopel import __version__, module, tools, web\nfrom sopel.config.types import ListAttribute, StaticSection, ValidatedAttribute\n\nUSER_AGENT = 'Sopel/{} (https://sopel.chat)'.format(__version__)\ndefault_headers = {'User-Agent': USER_AGENT}\n# These are used to clean up the title tag before actually parsing it. Not the\n# world's best way to do this, but it'll do for now.\ntitle_tag_data = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)\nquoted_title = re.compile('[\\'\"]<title>[\\'\"]', re.IGNORECASE)\n# This is another regex that presumably does something important.\nre_dcc = re.compile(r'(?i)dcc\\ssend')\n# This sets the maximum number of bytes that should be read in order to find\n# the title. We don't want it too high, or a link to a big file/stream will\n# just keep downloading until there's no more memory. 640k ought to be enough\n# for anybody.\nmax_bytes = 655360\n\n\nclass UrlSection(StaticSection):\n    # TODO some validation rules maybe?\n    exclude = ListAttribute('exclude')\n    \"\"\"A list of regular expressions to match URLs for which the title should not be shown.\"\"\"\n    exclusion_char = ValidatedAttribute('exclusion_char', default='!')\n    \"\"\"A character (or string) which, when immediately preceding a URL, will stop that URL's title from being shown.\"\"\"\n    shorten_url_length = ValidatedAttribute(\n        'shorten_url_length', int, default=0)\n    \"\"\"If greater than 0, the title fetcher will include a TinyURL version of links longer than this many characters.\"\"\"\n\n\ndef configure(config):\n    \"\"\"\n    | name | example | purpose |\n    | ---- | ------- | ------- |\n    | exclude | https?://git\\\\\\\\.io/.* | A list of regular expressions for URLs for which the title should not be shown. |\n    | exclusion\\\\_char | ! | A character (or string) which, when immediately preceding a URL, will stop the URL's title from being shown. |\n    | shorten\\\\_url\\\\_length | 72 | If greater than 0, the title fetcher will include a TinyURL version of links longer than this many characters. |\n    \"\"\"\n    config.define_section('url', UrlSection)\n    config.url.configure_setting(\n        'exclude',\n        'Enter regular expressions for each URL you would like to exclude.'\n    )\n    config.url.configure_setting(\n        'exclusion_char',\n        'Enter a character which can be prefixed to suppress URL titling'\n    )\n    config.url.configure_setting(\n        'shorten_url_length',\n        'Enter how many characters a URL should be before the bot puts a'\n        ' shorter version of the URL in the title as a TinyURL link'\n        ' (0 to disable)'\n    )\n\n\ndef setup(bot):\n    bot.config.define_section('url', UrlSection)\n\n    if bot.config.url.exclude:\n        regexes = [re.compile(s) for s in bot.config.url.exclude]\n    else:\n        regexes = []\n\n    # We're keeping these in their own list, rather than putting then in the\n    # callbacks list because 1, it's easier to deal with modules that are still\n    # using this list, and not the newer callbacks list and 2, having a lambda\n    # just to pass is kinda ugly.\n    if 'url_exclude' not in bot.memory:\n        bot.memory['url_exclude'] = regexes\n    else:\n        exclude = bot.memory['url_exclude']\n        if regexes:\n            exclude.extend(regexes)\n        bot.memory['url_exclude'] = exclude\n\n    # Ensure last_seen_url is in memory\n    if 'last_seen_url' not in bot.memory:\n        bot.memory['last_seen_url'] = tools.SopelMemory()\n\n    # Initialize shortened_urls as a dict if it doesn't exist.\n    if 'shortened_urls' not in bot.memory:\n        bot.memory['shortened_urls'] = tools.SopelMemory()\n\n\n@module.commands('title')\n@module.example(\n    '.title https://www.google.com',\n    '[ Google ] - www.google.com',\n    online=True)\ndef title_command(bot, trigger):\n    \"\"\"\n    Show the title or URL information for the given URL, or the last URL seen\n    in this channel.\n    \"\"\"\n    if not trigger.group(2):\n        if trigger.sender not in bot.memory['last_seen_url']:\n            return\n        matched = check_callbacks(\n            bot, bot.memory['last_seen_url'][trigger.sender])\n        if matched:\n            return\n        else:\n            urls = [bot.memory['last_seen_url'][trigger.sender]]\n    else:\n        urls = web.search_urls(\n            trigger,\n            exclusion_char=bot.config.url.exclusion_char)\n\n    for url, title, domain, tinyurl in process_urls(bot, trigger, urls):\n        message = '[ %s ] - %s' % (title, domain)\n        if tinyurl:\n            message += ' ( %s )' % tinyurl\n        bot.reply(message)\n        bot.memory['last_seen_url'][trigger.sender] = url\n\n\n@module.rule(r'(?u).*(https?://\\S+).*')\ndef title_auto(bot, trigger):\n    \"\"\"\n    Automatically show titles for URLs. For shortened URLs/redirects, find\n    where the URL redirects to and show the title for that (or call a function\n    from another module to give more information).\n    \"\"\"\n    if re.match(bot.config.core.prefix + 'title', trigger):\n        return\n\n    # Avoid fetching known malicious links\n    if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']:\n        if bot.memory['safety_cache'][trigger]['positives'] > 1:\n            return\n\n    urls = web.search_urls(\n        trigger, exclusion_char=bot.config.url.exclusion_char, clean=True)\n\n    for url, title, domain, tinyurl in process_urls(bot, trigger, urls):\n        message = '[ %s ] - %s' % (title, domain)\n        if tinyurl:\n            message += ' ( %s )' % tinyurl\n        # Guard against responding to other instances of this bot.\n        if message != trigger:\n            bot.say(message)\n            bot.memory['last_seen_url'][trigger.sender] = url\n\n\ndef process_urls(bot, trigger, urls):\n    \"\"\"\n    For each URL in the list, ensure that it isn't handled by another module.\n    If not, find where it redirects to, if anywhere. If that redirected URL\n    should be handled by another module, dispatch the callback for it.\n    Return a list of (title, hostname) tuples for each URL which is not handled\n    by another module.\n    \"\"\"\n    shorten_url_length = bot.config.url.shorten_url_length\n    for url in urls:\n        # Exclude URLs that start with the exclusion char\n        if url.startswith(bot.config.url.exclusion_char):\n            continue\n\n        # Check the URL does not match an existing URL callback\n        if check_callbacks(bot, url):\n            continue\n\n        # Call the URL to get a title, if possible\n        title = find_title(url, verify=bot.config.core.verify_ssl)\n        if not title:\n            # No title found: don't handle this URL\n            continue\n\n        # If the URL is over bot.config.url.shorten_url_length, shorten the URL\n        tinyurl = None\n        if (shorten_url_length > 0) and (len(url) > shorten_url_length):\n            tinyurl = get_or_create_shorturl(bot, url)\n\n        yield (url, title, get_hostname(url), tinyurl)\n\n\ndef check_callbacks(bot, url):\n    \"\"\"Check if ``url`` is excluded or matches any URL callback patterns.\n\n    :param bot: Sopel instance\n    :param str url: URL to check\n    :return: True if ``url`` is excluded or matches any URL Callback pattern\n\n    This function looks at the ``bot.memory`` for ``url_exclude`` patterns and\n    it returns ``True`` if any matches the given ``url``. Otherwise, it looks\n    at the ``bot``'s URL Callback patterns, and it returns ``True`` if any\n    matches, ``False`` otherwise.\n\n    .. seealso::\n\n        The :func:`~sopel.modules.url.setup` function that defines the\n        ``url_exclude`` in ``bot.memory``.\n\n    .. versionchanged:: 7.0\n\n        This function **does not** trigger URL callbacks anymore when ``url``\n        matches a pattern.\n\n    \"\"\"\n    # Check if it matches the exclusion list first\n    matched = any(regex.search(url) for regex in bot.memory['url_exclude'])\n    return matched or any(bot.search_url_callbacks(url))\n\n\ndef find_title(url, verify=True):\n    \"\"\"Return the title for the given URL.\"\"\"\n    try:\n        response = requests.get(url, stream=True, verify=verify,\n                                headers=default_headers)\n        content = b''\n        for byte in response.iter_content(chunk_size=512):\n            content += byte\n            if b'</title>' in content or len(content) > max_bytes:\n                break\n        content = content.decode('utf-8', errors='ignore')\n        # Need to close the connection because we have not read all\n        # the data\n        response.close()\n    except requests.exceptions.ConnectionError:\n        return None\n\n    # Some cleanup that I don't really grok, but was in the original, so\n    # we'll keep it (with the compiled regexes made global) for now.\n    content = title_tag_data.sub(r'<\\1title>', content)\n    content = quoted_title.sub('', content)\n\n    start = content.rfind('<title>')\n    end = content.rfind('</title>')\n    if start == -1 or end == -1:\n        return\n    title = web.decode(content[start + 7:end])\n    title = title.strip()[:200]\n\n    title = ' '.join(title.split())  # cleanly remove multiple spaces\n\n    # More cryptic regex substitutions. This one looks to be myano's invention.\n    title = re_dcc.sub('', title)\n\n    return title or None\n\n\ndef get_hostname(url):\n    idx = 7\n    if url.startswith('https://'):\n        idx = 8\n    elif url.startswith('ftp://'):\n        idx = 6\n    hostname = url[idx:]\n    slash = hostname.find('/')\n    if slash != -1:\n        hostname = hostname[:slash]\n    return hostname\n\n\ndef get_or_create_shorturl(bot, url):\n    \"\"\"Get or create a short URL for ``url``\n\n    :param bot: Sopel instance\n    :param str url: URL to get or create a short URL for\n    :return: A short URL\n    :rtype: str\n\n    It gets the short URL for ``url`` from the bot's memory if it exists.\n    Otherwise, it creates a short URL (see :func:`get_tinyurl`), stores it\n    into the bot's memory, then returns it.\n    \"\"\"\n    # Check bot memory to see if the shortened URL is already in\n    # memory\n    if url in bot.memory['shortened_urls']:\n        return bot.memory['shortened_urls'][url]\n\n    tinyurl = get_tinyurl(url)\n    bot.memory['shortened_urls'][url] = tinyurl\n    return tinyurl\n\n\ndef get_tinyurl(url):\n    \"\"\"Returns a shortened tinyURL link of the URL\"\"\"\n    base_url = \"https://tinyurl.com/api-create.php\"\n    tinyurl = \"%s?%s\" % (base_url, web.urlencode({'url': url}))\n    try:\n        res = requests.get(tinyurl)\n        res.raise_for_status()\n    except requests.exceptions.RequestException:\n        return None\n    # Replace text output with https instead of http to make the\n    # result an HTTPS link.\n    return res.text.replace(\"http://\", \"https://\")\n\n\nif __name__ == \"__main__\":\n    from sopel.test_tools import run_example_tests\n    run_example_tests(__file__)\n", "path": "sopel/modules/url.py"}], "after_files": [{"content": "# coding=utf-8\n\"\"\"\nurl.py - Sopel URL Title Module\nCopyright 2010-2011, Michael Yanovich (yanovich.net) & Kenneth Sham\nCopyright 2012-2013, Elsie Powell\nCopyright 2013, Lior Ramati <firerogue517@gmail.com>\nCopyright 2014, Elad Alfassa <elad@fedoraproject.org>\nLicensed under the Eiffel Forum License 2.\n\nhttps://sopel.chat\n\"\"\"\nfrom __future__ import unicode_literals, absolute_import, print_function, division\n\nimport re\n\nimport dns.resolver\nimport ipaddress\nimport requests\n\nfrom sopel import __version__, module, tools, web\nfrom sopel.config.types import ListAttribute, StaticSection, ValidatedAttribute\n\n# Python3 vs Python2\ntry:\n    from urllib.parse import urlparse\nexcept ImportError:\n    from urlparse import urlparse\n\nUSER_AGENT = 'Sopel/{} (https://sopel.chat)'.format(__version__)\ndefault_headers = {'User-Agent': USER_AGENT}\n# These are used to clean up the title tag before actually parsing it. Not the\n# world's best way to do this, but it'll do for now.\ntitle_tag_data = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)\nquoted_title = re.compile('[\\'\"]<title>[\\'\"]', re.IGNORECASE)\n# This is another regex that presumably does something important.\nre_dcc = re.compile(r'(?i)dcc\\ssend')\n# This sets the maximum number of bytes that should be read in order to find\n# the title. We don't want it too high, or a link to a big file/stream will\n# just keep downloading until there's no more memory. 640k ought to be enough\n# for anybody.\nmax_bytes = 655360\n\n\nclass UrlSection(StaticSection):\n    # TODO some validation rules maybe?\n    exclude = ListAttribute('exclude')\n    \"\"\"A list of regular expressions to match URLs for which the title should not be shown.\"\"\"\n    exclusion_char = ValidatedAttribute('exclusion_char', default='!')\n    \"\"\"A character (or string) which, when immediately preceding a URL, will stop that URL's title from being shown.\"\"\"\n    shorten_url_length = ValidatedAttribute(\n        'shorten_url_length', int, default=0)\n    \"\"\"If greater than 0, the title fetcher will include a TinyURL version of links longer than this many characters.\"\"\"\n    enable_private_resolution = ValidatedAttribute(\n        'enable_private_resolution', bool, default=False)\n    enable_dns_resolution = ValidatedAttribute(\n        'enable_dns_resolution', bool, default=False)\n\n\ndef configure(config):\n    \"\"\"\n    | name | example | purpose |\n    | ---- | ------- | ------- |\n    | exclude | https?://git\\\\\\\\.io/.* | A list of regular expressions for URLs for which the title should not be shown. |\n    | exclusion\\\\_char | ! | A character (or string) which, when immediately preceding a URL, will stop the URL's title from being shown. |\n    | shorten\\\\_url\\\\_length | 72 | If greater than 0, the title fetcher will include a TinyURL version of links longer than this many characters. |\n    \"\"\"\n    config.define_section('url', UrlSection)\n    config.url.configure_setting(\n        'exclude',\n        'Enter regular expressions for each URL you would like to exclude.'\n    )\n    config.url.configure_setting(\n        'exclusion_char',\n        'Enter a character which can be prefixed to suppress URL titling'\n    )\n    config.url.configure_setting(\n        'shorten_url_length',\n        'Enter how many characters a URL should be before the bot puts a'\n        ' shorter version of the URL in the title as a TinyURL link'\n        ' (0 to disable)'\n    )\n    config.url.configure_setting(\n        'enable_private_resolution',\n        'Enable URL lookups for RFC1918 addresses?'\n    )\n    config.url.configure_setting(\n        'enable_dns_resolution',\n        'Enable DNS resolution for all domains to validate if there are RFC1918 resolutions?'\n    )\n\n\ndef setup(bot):\n    bot.config.define_section('url', UrlSection)\n\n    if bot.config.url.exclude:\n        regexes = [re.compile(s) for s in bot.config.url.exclude]\n    else:\n        regexes = []\n\n    # We're keeping these in their own list, rather than putting then in the\n    # callbacks list because 1, it's easier to deal with modules that are still\n    # using this list, and not the newer callbacks list and 2, having a lambda\n    # just to pass is kinda ugly.\n    if 'url_exclude' not in bot.memory:\n        bot.memory['url_exclude'] = regexes\n    else:\n        exclude = bot.memory['url_exclude']\n        if regexes:\n            exclude.extend(regexes)\n        bot.memory['url_exclude'] = exclude\n\n    # Ensure last_seen_url is in memory\n    if 'last_seen_url' not in bot.memory:\n        bot.memory['last_seen_url'] = tools.SopelMemory()\n\n    # Initialize shortened_urls as a dict if it doesn't exist.\n    if 'shortened_urls' not in bot.memory:\n        bot.memory['shortened_urls'] = tools.SopelMemory()\n\n\n@module.commands('title')\n@module.example('.title https://www.google.com', '[ Google ] - www.google.com')\ndef title_command(bot, trigger):\n    \"\"\"\n    Show the title or URL information for the given URL, or the last URL seen\n    in this channel.\n    \"\"\"\n    if not trigger.group(2):\n        if trigger.sender not in bot.memory['last_seen_url']:\n            return\n        matched = check_callbacks(\n            bot, bot.memory['last_seen_url'][trigger.sender])\n        if matched:\n            return\n        else:\n            urls = [bot.memory['last_seen_url'][trigger.sender]]\n    else:\n        urls = web.search_urls(\n            trigger,\n            exclusion_char=bot.config.url.exclusion_char)\n\n    for url, title, domain, tinyurl in process_urls(bot, trigger, urls):\n        message = '[ %s ] - %s' % (title, domain)\n        if tinyurl:\n            message += ' ( %s )' % tinyurl\n        bot.reply(message)\n        bot.memory['last_seen_url'][trigger.sender] = url\n\n\n@module.rule(r'(?u).*(https?://\\S+).*')\ndef title_auto(bot, trigger):\n    \"\"\"\n    Automatically show titles for URLs. For shortened URLs/redirects, find\n    where the URL redirects to and show the title for that (or call a function\n    from another module to give more information).\n    \"\"\"\n    if re.match(bot.config.core.prefix + 'title', trigger):\n        return\n\n    # Avoid fetching known malicious links\n    if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']:\n        if bot.memory['safety_cache'][trigger]['positives'] > 1:\n            return\n\n    urls = web.search_urls(\n        trigger, exclusion_char=bot.config.url.exclusion_char, clean=True)\n\n    for url, title, domain, tinyurl in process_urls(bot, trigger, urls):\n        message = '[ %s ] - %s' % (title, domain)\n        if tinyurl:\n            message += ' ( %s )' % tinyurl\n        # Guard against responding to other instances of this bot.\n        if message != trigger:\n            bot.say(message)\n            bot.memory['last_seen_url'][trigger.sender] = url\n\n\ndef process_urls(bot, trigger, urls):\n    \"\"\"\n    For each URL in the list, ensure that it isn't handled by another module.\n    If not, find where it redirects to, if anywhere. If that redirected URL\n    should be handled by another module, dispatch the callback for it.\n    Return a list of (title, hostname) tuples for each URL which is not handled\n    by another module.\n    \"\"\"\n    shorten_url_length = bot.config.url.shorten_url_length\n    for url in urls:\n        # Exclude URLs that start with the exclusion char\n        if url.startswith(bot.config.url.exclusion_char):\n            continue\n\n        # Check the URL does not match an existing URL callback\n        if check_callbacks(bot, url):\n            continue\n\n        # Prevent private addresses form being queried if enable_private_resolution is False\n        if not bot.config.url.enable_private_resolution:\n            parsed = urlparse(url)\n            # Check if it's an address like http://192.168.1.1\n            try:\n                if ipaddress.ip_address(parsed.hostname).is_private or ipaddress.ip_address(parsed.hostname).is_loopback:\n                    continue\n            except ValueError:\n                pass\n\n            # Check if domains are RFC1918 addresses if enable_dns_resolutions is set\n            if bot.config.url.enable_dns_resolution:\n                private = False\n                for result in dns.resolver.query(parsed.hostname):\n                    if ipaddress.ip_address(result).is_private:\n                        private = True\n                        break\n                if private:\n                    continue\n\n        # Call the URL to get a title, if possible\n        title = find_title(url, verify=bot.config.core.verify_ssl)\n        if not title:\n            # No title found: don't handle this URL\n            continue\n\n        # If the URL is over bot.config.url.shorten_url_length, shorten the URL\n        tinyurl = None\n        if (shorten_url_length > 0) and (len(url) > shorten_url_length):\n            tinyurl = get_or_create_shorturl(bot, url)\n\n        yield (url, title, get_hostname(url), tinyurl)\n\n\ndef check_callbacks(bot, url):\n    \"\"\"Check if ``url`` is excluded or matches any URL callback patterns.\n\n    :param bot: Sopel instance\n    :param str url: URL to check\n    :return: True if ``url`` is excluded or matches any URL Callback pattern\n\n    This function looks at the ``bot.memory`` for ``url_exclude`` patterns and\n    it returns ``True`` if any matches the given ``url``. Otherwise, it looks\n    at the ``bot``'s URL Callback patterns, and it returns ``True`` if any\n    matches, ``False`` otherwise.\n\n    .. seealso::\n\n        The :func:`~sopel.modules.url.setup` function that defines the\n        ``url_exclude`` in ``bot.memory``.\n\n    .. versionchanged:: 7.0\n\n        This function **does not** trigger URL callbacks anymore when ``url``\n        matches a pattern.\n\n    \"\"\"\n    # Check if it matches the exclusion list first\n    matched = any(regex.search(url) for regex in bot.memory['url_exclude'])\n    return matched or any(bot.search_url_callbacks(url))\n\n\ndef find_title(url, verify=True):\n    \"\"\"Return the title for the given URL.\"\"\"\n    try:\n        response = requests.get(url, stream=True, verify=verify,\n                                headers=default_headers)\n        content = b''\n        for byte in response.iter_content(chunk_size=512):\n            content += byte\n            if b'</title>' in content or len(content) > max_bytes:\n                break\n        content = content.decode('utf-8', errors='ignore')\n        # Need to close the connection because we have not read all\n        # the data\n        response.close()\n    except requests.exceptions.ConnectionError:\n        return None\n\n    # Some cleanup that I don't really grok, but was in the original, so\n    # we'll keep it (with the compiled regexes made global) for now.\n    content = title_tag_data.sub(r'<\\1title>', content)\n    content = quoted_title.sub('', content)\n\n    start = content.rfind('<title>')\n    end = content.rfind('</title>')\n    if start == -1 or end == -1:\n        return\n    title = web.decode(content[start + 7:end])\n    title = title.strip()[:200]\n\n    title = ' '.join(title.split())  # cleanly remove multiple spaces\n\n    # More cryptic regex substitutions. This one looks to be myano's invention.\n    title = re_dcc.sub('', title)\n\n    return title or None\n\n\ndef get_hostname(url):\n    idx = 7\n    if url.startswith('https://'):\n        idx = 8\n    elif url.startswith('ftp://'):\n        idx = 6\n    hostname = url[idx:]\n    slash = hostname.find('/')\n    if slash != -1:\n        hostname = hostname[:slash]\n    return hostname\n\n\ndef get_or_create_shorturl(bot, url):\n    \"\"\"Get or create a short URL for ``url``\n\n    :param bot: Sopel instance\n    :param str url: URL to get or create a short URL for\n    :return: A short URL\n    :rtype: str\n\n    It gets the short URL for ``url`` from the bot's memory if it exists.\n    Otherwise, it creates a short URL (see :func:`get_tinyurl`), stores it\n    into the bot's memory, then returns it.\n    \"\"\"\n    # Check bot memory to see if the shortened URL is already in\n    # memory\n    if url in bot.memory['shortened_urls']:\n        return bot.memory['shortened_urls'][url]\n\n    tinyurl = get_tinyurl(url)\n    bot.memory['shortened_urls'][url] = tinyurl\n    return tinyurl\n\n\ndef get_tinyurl(url):\n    \"\"\"Returns a shortened tinyURL link of the URL\"\"\"\n    base_url = \"https://tinyurl.com/api-create.php\"\n    tinyurl = \"%s?%s\" % (base_url, web.urlencode({'url': url}))\n    try:\n        res = requests.get(tinyurl)\n        res.raise_for_status()\n    except requests.exceptions.RequestException:\n        return None\n    # Replace text output with https instead of http to make the\n    # result an HTTPS link.\n    return res.text.replace(\"http://\", \"https://\")\n\n\nif __name__ == \"__main__\":\n    from sopel.test_tools import run_example_tests\n    run_example_tests(__file__)\n", "path": "sopel/modules/url.py"}]}