We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
URLs with ending commas get included when Sopel v6.5.x parse them.
```
Examples:

[02:46pm] <Ant> http://www.tv.com/shows/family-guy/da-boom-25450/,
02:46PM <Crushinator> [ Not found - TV.com ] - www.tv.com
02:47PM <URL> [ Not found - TV.com ] - www.tv.com

Periods had no problems:
[02:48pm] <Ant> http://www.tv.com/shows/family-guy/da-boom-25450/.
02:48PM <URL> [ Family Guy - Season 2, Episode 3: Da Boom - TV.com ] -
          www.tv.com
02:48PM <Crushinator> [ Family Guy - Season 2, Episode 3: Da Boom - TV.com ] -
          www.tv.com

```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `sopel/modules/url.py`
Content:
```
1 # coding=utf-8
2 """URL title module"""
3 # Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham
4 # Copyright 2012-2013 Elsie Powell
5 # Copyright 2013      Lior Ramati (firerogue517@gmail.com)
6 # Copyright © 2014 Elad Alfassa <elad@fedoraproject.org>
7 # Licensed under the Eiffel Forum License 2.
8 from __future__ import unicode_literals, absolute_import, print_function, division
9 
10 import re
11 from sopel import web, tools, __version__
12 from sopel.module import commands, rule, example
13 from sopel.config.types import ValidatedAttribute, ListAttribute, StaticSection
14 
15 import requests
16 
17 USER_AGENT = 'Sopel/{} (https://sopel.chat)'.format(__version__)
18 default_headers = {'User-Agent': USER_AGENT}
19 find_urls = None
20 # These are used to clean up the title tag before actually parsing it. Not the
21 # world's best way to do this, but it'll do for now.
22 title_tag_data = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)
23 quoted_title = re.compile('[\'"]<title>[\'"]', re.IGNORECASE)
24 # This is another regex that presumably does something important.
25 re_dcc = re.compile(r'(?i)dcc\ssend')
26 # This sets the maximum number of bytes that should be read in order to find
27 # the title. We don't want it too high, or a link to a big file/stream will
28 # just keep downloading until there's no more memory. 640k ought to be enough
29 # for anybody.
30 max_bytes = 655360
31 
32 
33 class UrlSection(StaticSection):
34     # TODO some validation rules maybe?
35     exclude = ListAttribute('exclude')
36     exclusion_char = ValidatedAttribute('exclusion_char', default='!')
37     shorten_url_length = ValidatedAttribute(
38         'shorten_url_length', int, default=0)
39 
40 
41 def configure(config):
42     config.define_section('url', UrlSection)
43     config.url.configure_setting(
44         'exclude',
45         'Enter regular expressions for each URL you would like to exclude.'
46     )
47     config.url.configure_setting(
48         'exclusion_char',
49         'Enter a character which can be prefixed to suppress URL titling'
50     )
51     config.url.configure_setting(
52         'shorten_url_length',
53         'Enter how many characters a URL should be before the bot puts a'
54         ' shorter version of the URL in the title as a TinyURL link'
55         ' (0 to disable)'
56     )
57 
58 
59 def setup(bot):
60     global find_urls
61 
62     bot.config.define_section('url', UrlSection)
63 
64     if bot.config.url.exclude:
65         regexes = [re.compile(s) for s in bot.config.url.exclude]
66     else:
67         regexes = []
68 
69     # We're keeping these in their own list, rather than putting then in the
70     # callbacks list because 1, it's easier to deal with modules that are still
71     # using this list, and not the newer callbacks list and 2, having a lambda
72     # just to pass is kinda ugly.
73     if not bot.memory.contains('url_exclude'):
74         bot.memory['url_exclude'] = regexes
75     else:
76         exclude = bot.memory['url_exclude']
77         if regexes:
78             exclude.extend(regexes)
79         bot.memory['url_exclude'] = exclude
80 
81     # Ensure that url_callbacks and last_seen_url are in memory
82     if not bot.memory.contains('url_callbacks'):
83         bot.memory['url_callbacks'] = tools.SopelMemory()
84     if not bot.memory.contains('last_seen_url'):
85         bot.memory['last_seen_url'] = tools.SopelMemory()
86 
87     def find_func(text):
88         re_url = r'(?u)((?<!%s)(?:http|https|ftp)(?::\/\/\S+))'\
89             % (bot.config.url.exclusion_char)
90         r = re.compile(re_url, re.IGNORECASE)
91 
92         urls = re.findall(r, text)
93         return urls
94 
95     find_urls = find_func
96 
97 
98 @commands('title')
99 @example('.title http://google.com', '[ Google ] - google.com')
100 def title_command(bot, trigger):
101     """
102     Show the title or URL information for the given URL, or the last URL seen
103     in this channel.
104     """
105     if not trigger.group(2):
106         if trigger.sender not in bot.memory['last_seen_url']:
107             return
108         matched = check_callbacks(bot, trigger,
109                                   bot.memory['last_seen_url'][trigger.sender],
110                                   True)
111         if matched:
112             return
113         else:
114             urls = [bot.memory['last_seen_url'][trigger.sender]]
115     else:
116         urls = find_urls(trigger)
117 
118     results = process_urls(bot, trigger, urls)
119     for title, domain, tinyurl in results[:4]:
120         message = '[ %s ] - %s' % (title, domain)
121         if tinyurl:
122             message += ' ( %s )' % tinyurl
123         bot.reply(message)
124 
125 
126 @rule(r'(?u).*(https?://\S+).*')
127 def title_auto(bot, trigger):
128     """
129     Automatically show titles for URLs. For shortened URLs/redirects, find
130     where the URL redirects to and show the title for that (or call a function
131     from another module to give more information).
132     """
133     if re.match(bot.config.core.prefix + 'title', trigger):
134         return
135 
136     # Avoid fetching known malicious links
137     if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']:
138         if bot.memory['safety_cache'][trigger]['positives'] > 1:
139             return
140 
141     urls = find_urls(trigger)
142     if len(urls) == 0:
143         return
144 
145     results = process_urls(bot, trigger, urls)
146     bot.memory['last_seen_url'][trigger.sender] = urls[-1]
147 
148     for title, domain, tinyurl in results[:4]:
149         message = '[ %s ] - %s' % (title, domain)
150         if tinyurl:
151             message += ' ( %s )' % tinyurl
152         # Guard against responding to other instances of this bot.
153         if message != trigger:
154             bot.say(message)
155 
156 
157 def process_urls(bot, trigger, urls):
158     """
159     For each URL in the list, ensure that it isn't handled by another module.
160     If not, find where it redirects to, if anywhere. If that redirected URL
161     should be handled by another module, dispatch the callback for it.
162     Return a list of (title, hostname) tuples for each URL which is not handled by
163     another module.
164     """
165 
166     results = []
167     shorten_url_length = bot.config.url.shorten_url_length
168     for url in urls:
169         if not url.startswith(bot.config.url.exclusion_char):
170             # Magic stuff to account for international domain names
171             try:
172                 url = web.iri_to_uri(url)
173             except Exception:  # TODO: Be specific
174                 pass
175             # First, check that the URL we got doesn't match
176             matched = check_callbacks(bot, trigger, url, False)
177             if matched:
178                 continue
179             # If the URL is over bot.config.url.shorten_url_length,
180             # shorten the URL
181             tinyurl = None
182             if (shorten_url_length > 0) and (len(url) > shorten_url_length):
183                 # Check bot memory to see if the shortened URL is already in
184                 # memory
185                 if not bot.memory.contains('shortened_urls'):
186                     # Initialize shortened_urls as a dict if it doesn't exist.
187                     bot.memory['shortened_urls'] = tools.SopelMemory()
188                 if bot.memory['shortened_urls'].contains(url):
189                     tinyurl = bot.memory['shortened_urls'][url]
190                 else:
191                     tinyurl = get_tinyurl(url)
192                     bot.memory['shortened_urls'][url] = tinyurl
193             # Finally, actually show the URL
194             title = find_title(url, verify=bot.config.core.verify_ssl)
195             if title:
196                 results.append((title, get_hostname(url), tinyurl))
197     return results
198 
199 
200 def check_callbacks(bot, trigger, url, run=True):
201     """
202     Check the given URL against the callbacks list. If it matches, and ``run``
203     is given as ``True``, run the callback function, otherwise pass. Returns
204     ``True`` if the url matched anything in the callbacks list.
205     """
206     # Check if it matches the exclusion list first
207     matched = any(regex.search(url) for regex in bot.memory['url_exclude'])
208     # Then, check if there's anything in the callback list
209     for regex, function in tools.iteritems(bot.memory['url_callbacks']):
210         match = regex.search(url)
211         if match:
212             # Always run ones from @url; they don't run on their own.
213             if run or hasattr(function, 'url_regex'):
214                 function(bot, trigger, match)
215             matched = True
216     return matched
217 
218 
219 def find_title(url, verify=True):
220     """Return the title for the given URL."""
221     try:
222         response = requests.get(url, stream=True, verify=verify,
223                                 headers=default_headers)
224         content = b''
225         for byte in response.iter_content(chunk_size=512):
226             content += byte
227             if b'</title>' in content or len(content) > max_bytes:
228                 break
229         content = content.decode('utf-8', errors='ignore')
230         # Need to close the connection because we have not read all
231         # the data
232         response.close()
233     except requests.exceptions.ConnectionError:
234         return None
235 
236     # Some cleanup that I don't really grok, but was in the original, so
237     # we'll keep it (with the compiled regexes made global) for now.
238     content = title_tag_data.sub(r'<\1title>', content)
239     content = quoted_title.sub('', content)
240 
241     start = content.rfind('<title>')
242     end = content.rfind('</title>')
243     if start == -1 or end == -1:
244         return
245     title = web.decode(content[start + 7:end])
246     title = title.strip()[:200]
247 
248     title = ' '.join(title.split())  # cleanly remove multiple spaces
249 
250     # More cryptic regex substitutions. This one looks to be myano's invention.
251     title = re_dcc.sub('', title)
252 
253     return title or None
254 
255 
256 def get_hostname(url):
257     idx = 7
258     if url.startswith('https://'):
259         idx = 8
260     elif url.startswith('ftp://'):
261         idx = 6
262     hostname = url[idx:]
263     slash = hostname.find('/')
264     if slash != -1:
265         hostname = hostname[:slash]
266     return hostname
267 
268 
269 def get_tinyurl(url):
270     """ Returns a shortened tinyURL link of the URL. """
271     tinyurl = "https://tinyurl.com/api-create.php?url=%s" % url
272     try:
273         res = requests.get(tinyurl)
274         res.raise_for_status()
275     except requests.exceptions.RequestException:
276         return None
277     # Replace text output with https instead of http to make the
278     # result an HTTPS link.
279     return res.text.replace("http://", "https://")
280 
281 
282 if __name__ == "__main__":
283     from sopel.test_tools import run_example_tests
284     run_example_tests(__file__)
285
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/sopel/modules/url.py b/sopel/modules/url.py
--- a/sopel/modules/url.py
+++ b/sopel/modules/url.py
@@ -84,12 +84,26 @@
     if not bot.memory.contains('last_seen_url'):
         bot.memory['last_seen_url'] = tools.SopelMemory()
 
-    def find_func(text):
+    def find_func(text, clean=False):
+        def trim_url(url):
+            # clean trailing sentence- or clause-ending punctuation
+            while url[-1] in '.,?!\'":;':
+                url = url[:-1]
+
+            # clean unmatched parentheses/braces/brackets
+            for (opener, closer) in [('(', ')'), ('[', ']'), ('{', '}'), ('<', '>')]:
+                if url[-1] is closer and url.count(opener) < url.count(closer):
+                    url = url[:-1]
+
+            return url
+
         re_url = r'(?u)((?<!%s)(?:http|https|ftp)(?::\/\/\S+))'\
             % (bot.config.url.exclusion_char)
         r = re.compile(re_url, re.IGNORECASE)
 
         urls = re.findall(r, text)
+        if clean:
+            urls = [trim_url(url) for url in urls]
         return urls
 
     find_urls = find_func

{"golden_diff": "diff --git a/sopel/modules/url.py b/sopel/modules/url.py\n--- a/sopel/modules/url.py\n+++ b/sopel/modules/url.py\n@@ -84,12 +84,26 @@\n     if not bot.memory.contains('last_seen_url'):\n         bot.memory['last_seen_url'] = tools.SopelMemory()\n \n-    def find_func(text):\n+    def find_func(text, clean=False):\n+        def trim_url(url):\n+            # clean trailing sentence- or clause-ending punctuation\n+            while url[-1] in '.,?!\\'\":;':\n+                url = url[:-1]\n+\n+            # clean unmatched parentheses/braces/brackets\n+            for (opener, closer) in [('(', ')'), ('[', ']'), ('{', '}'), ('<', '>')]:\n+                if url[-1] is closer and url.count(opener) < url.count(closer):\n+                    url = url[:-1]\n+\n+            return url\n+\n         re_url = r'(?u)((?<!%s)(?:http|https|ftp)(?::\\/\\/\\S+))'\\\n             % (bot.config.url.exclusion_char)\n         r = re.compile(re_url, re.IGNORECASE)\n \n         urls = re.findall(r, text)\n+        if clean:\n+            urls = [trim_url(url) for url in urls]\n         return urls\n \n     find_urls = find_func\n", "issue": "URLs with ending commas get included when Sopel v6.5.x parse them.\n```\r\nExamples:\r\n\r\n[02:46pm] <Ant> http://www.tv.com/shows/family-guy/da-boom-25450/,\r\n02:46PM <Crushinator> [ Not found - TV.com ] - www.tv.com\r\n02:47PM <URL> [ Not found - TV.com ] - www.tv.com\r\n\r\nPeriods had no problems:\r\n[02:48pm] <Ant> http://www.tv.com/shows/family-guy/da-boom-25450/.\r\n02:48PM <URL> [ Family Guy - Season 2, Episode 3: Da Boom - TV.com ] -\r\n          www.tv.com\r\n02:48PM <Crushinator> [ Family Guy - Season 2, Episode 3: Da Boom - TV.com ] -\r\n          www.tv.com\r\n\r\n```\n", "before_files": [{"content": "# coding=utf-8\n\"\"\"URL title module\"\"\"\n# Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham\n# Copyright 2012-2013 Elsie Powell\n# Copyright 2013      Lior Ramati (firerogue517@gmail.com)\n# Copyright \u00a9 2014 Elad Alfassa <elad@fedoraproject.org>\n# Licensed under the Eiffel Forum License 2.\nfrom __future__ import unicode_literals, absolute_import, print_function, division\n\nimport re\nfrom sopel import web, tools, __version__\nfrom sopel.module import commands, rule, example\nfrom sopel.config.types import ValidatedAttribute, ListAttribute, StaticSection\n\nimport requests\n\nUSER_AGENT = 'Sopel/{} (https://sopel.chat)'.format(__version__)\ndefault_headers = {'User-Agent': USER_AGENT}\nfind_urls = None\n# These are used to clean up the title tag before actually parsing it. Not the\n# world's best way to do this, but it'll do for now.\ntitle_tag_data = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)\nquoted_title = re.compile('[\\'\"]<title>[\\'\"]', re.IGNORECASE)\n# This is another regex that presumably does something important.\nre_dcc = re.compile(r'(?i)dcc\\ssend')\n# This sets the maximum number of bytes that should be read in order to find\n# the title. We don't want it too high, or a link to a big file/stream will\n# just keep downloading until there's no more memory. 640k ought to be enough\n# for anybody.\nmax_bytes = 655360\n\n\nclass UrlSection(StaticSection):\n    # TODO some validation rules maybe?\n    exclude = ListAttribute('exclude')\n    exclusion_char = ValidatedAttribute('exclusion_char', default='!')\n    shorten_url_length = ValidatedAttribute(\n        'shorten_url_length', int, default=0)\n\n\ndef configure(config):\n    config.define_section('url', UrlSection)\n    config.url.configure_setting(\n        'exclude',\n        'Enter regular expressions for each URL you would like to exclude.'\n    )\n    config.url.configure_setting(\n        'exclusion_char',\n        'Enter a character which can be prefixed to suppress URL titling'\n    )\n    config.url.configure_setting(\n        'shorten_url_length',\n        'Enter how many characters a URL should be before the bot puts a'\n        ' shorter version of the URL in the title as a TinyURL link'\n        ' (0 to disable)'\n    )\n\n\ndef setup(bot):\n    global find_urls\n\n    bot.config.define_section('url', UrlSection)\n\n    if bot.config.url.exclude:\n        regexes = [re.compile(s) for s in bot.config.url.exclude]\n    else:\n        regexes = []\n\n    # We're keeping these in their own list, rather than putting then in the\n    # callbacks list because 1, it's easier to deal with modules that are still\n    # using this list, and not the newer callbacks list and 2, having a lambda\n    # just to pass is kinda ugly.\n    if not bot.memory.contains('url_exclude'):\n        bot.memory['url_exclude'] = regexes\n    else:\n        exclude = bot.memory['url_exclude']\n        if regexes:\n            exclude.extend(regexes)\n        bot.memory['url_exclude'] = exclude\n\n    # Ensure that url_callbacks and last_seen_url are in memory\n    if not bot.memory.contains('url_callbacks'):\n        bot.memory['url_callbacks'] = tools.SopelMemory()\n    if not bot.memory.contains('last_seen_url'):\n        bot.memory['last_seen_url'] = tools.SopelMemory()\n\n    def find_func(text):\n        re_url = r'(?u)((?<!%s)(?:http|https|ftp)(?::\\/\\/\\S+))'\\\n            % (bot.config.url.exclusion_char)\n        r = re.compile(re_url, re.IGNORECASE)\n\n        urls = re.findall(r, text)\n        return urls\n\n    find_urls = find_func\n\n\n@commands('title')\n@example('.title http://google.com', '[ Google ] - google.com')\ndef title_command(bot, trigger):\n    \"\"\"\n    Show the title or URL information for the given URL, or the last URL seen\n    in this channel.\n    \"\"\"\n    if not trigger.group(2):\n        if trigger.sender not in bot.memory['last_seen_url']:\n            return\n        matched = check_callbacks(bot, trigger,\n                                  bot.memory['last_seen_url'][trigger.sender],\n                                  True)\n        if matched:\n            return\n        else:\n            urls = [bot.memory['last_seen_url'][trigger.sender]]\n    else:\n        urls = find_urls(trigger)\n\n    results = process_urls(bot, trigger, urls)\n    for title, domain, tinyurl in results[:4]:\n        message = '[ %s ] - %s' % (title, domain)\n        if tinyurl:\n            message += ' ( %s )' % tinyurl\n        bot.reply(message)\n\n\n@rule(r'(?u).*(https?://\\S+).*')\ndef title_auto(bot, trigger):\n    \"\"\"\n    Automatically show titles for URLs. For shortened URLs/redirects, find\n    where the URL redirects to and show the title for that (or call a function\n    from another module to give more information).\n    \"\"\"\n    if re.match(bot.config.core.prefix + 'title', trigger):\n        return\n\n    # Avoid fetching known malicious links\n    if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']:\n        if bot.memory['safety_cache'][trigger]['positives'] > 1:\n            return\n\n    urls = find_urls(trigger)\n    if len(urls) == 0:\n        return\n\n    results = process_urls(bot, trigger, urls)\n    bot.memory['last_seen_url'][trigger.sender] = urls[-1]\n\n    for title, domain, tinyurl in results[:4]:\n        message = '[ %s ] - %s' % (title, domain)\n        if tinyurl:\n            message += ' ( %s )' % tinyurl\n        # Guard against responding to other instances of this bot.\n        if message != trigger:\n            bot.say(message)\n\n\ndef process_urls(bot, trigger, urls):\n    \"\"\"\n    For each URL in the list, ensure that it isn't handled by another module.\n    If not, find where it redirects to, if anywhere. If that redirected URL\n    should be handled by another module, dispatch the callback for it.\n    Return a list of (title, hostname) tuples for each URL which is not handled by\n    another module.\n    \"\"\"\n\n    results = []\n    shorten_url_length = bot.config.url.shorten_url_length\n    for url in urls:\n        if not url.startswith(bot.config.url.exclusion_char):\n            # Magic stuff to account for international domain names\n            try:\n                url = web.iri_to_uri(url)\n            except Exception:  # TODO: Be specific\n                pass\n            # First, check that the URL we got doesn't match\n            matched = check_callbacks(bot, trigger, url, False)\n            if matched:\n                continue\n            # If the URL is over bot.config.url.shorten_url_length,\n            # shorten the URL\n            tinyurl = None\n            if (shorten_url_length > 0) and (len(url) > shorten_url_length):\n                # Check bot memory to see if the shortened URL is already in\n                # memory\n                if not bot.memory.contains('shortened_urls'):\n                    # Initialize shortened_urls as a dict if it doesn't exist.\n                    bot.memory['shortened_urls'] = tools.SopelMemory()\n                if bot.memory['shortened_urls'].contains(url):\n                    tinyurl = bot.memory['shortened_urls'][url]\n                else:\n                    tinyurl = get_tinyurl(url)\n                    bot.memory['shortened_urls'][url] = tinyurl\n            # Finally, actually show the URL\n            title = find_title(url, verify=bot.config.core.verify_ssl)\n            if title:\n                results.append((title, get_hostname(url), tinyurl))\n    return results\n\n\ndef check_callbacks(bot, trigger, url, run=True):\n    \"\"\"\n    Check the given URL against the callbacks list. If it matches, and ``run``\n    is given as ``True``, run the callback function, otherwise pass. Returns\n    ``True`` if the url matched anything in the callbacks list.\n    \"\"\"\n    # Check if it matches the exclusion list first\n    matched = any(regex.search(url) for regex in bot.memory['url_exclude'])\n    # Then, check if there's anything in the callback list\n    for regex, function in tools.iteritems(bot.memory['url_callbacks']):\n        match = regex.search(url)\n        if match:\n            # Always run ones from @url; they don't run on their own.\n            if run or hasattr(function, 'url_regex'):\n                function(bot, trigger, match)\n            matched = True\n    return matched\n\n\ndef find_title(url, verify=True):\n    \"\"\"Return the title for the given URL.\"\"\"\n    try:\n        response = requests.get(url, stream=True, verify=verify,\n                                headers=default_headers)\n        content = b''\n        for byte in response.iter_content(chunk_size=512):\n            content += byte\n            if b'</title>' in content or len(content) > max_bytes:\n                break\n        content = content.decode('utf-8', errors='ignore')\n        # Need to close the connection because we have not read all\n        # the data\n        response.close()\n    except requests.exceptions.ConnectionError:\n        return None\n\n    # Some cleanup that I don't really grok, but was in the original, so\n    # we'll keep it (with the compiled regexes made global) for now.\n    content = title_tag_data.sub(r'<\\1title>', content)\n    content = quoted_title.sub('', content)\n\n    start = content.rfind('<title>')\n    end = content.rfind('</title>')\n    if start == -1 or end == -1:\n        return\n    title = web.decode(content[start + 7:end])\n    title = title.strip()[:200]\n\n    title = ' '.join(title.split())  # cleanly remove multiple spaces\n\n    # More cryptic regex substitutions. This one looks to be myano's invention.\n    title = re_dcc.sub('', title)\n\n    return title or None\n\n\ndef get_hostname(url):\n    idx = 7\n    if url.startswith('https://'):\n        idx = 8\n    elif url.startswith('ftp://'):\n        idx = 6\n    hostname = url[idx:]\n    slash = hostname.find('/')\n    if slash != -1:\n        hostname = hostname[:slash]\n    return hostname\n\n\ndef get_tinyurl(url):\n    \"\"\" Returns a shortened tinyURL link of the URL. \"\"\"\n    tinyurl = \"https://tinyurl.com/api-create.php?url=%s\" % url\n    try:\n        res = requests.get(tinyurl)\n        res.raise_for_status()\n    except requests.exceptions.RequestException:\n        return None\n    # Replace text output with https instead of http to make the\n    # result an HTTPS link.\n    return res.text.replace(\"http://\", \"https://\")\n\n\nif __name__ == \"__main__\":\n    from sopel.test_tools import run_example_tests\n    run_example_tests(__file__)\n", "path": "sopel/modules/url.py"}], "after_files": [{"content": "# coding=utf-8\n\"\"\"URL title module\"\"\"\n# Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham\n# Copyright 2012-2013 Elsie Powell\n# Copyright 2013      Lior Ramati (firerogue517@gmail.com)\n# Copyright \u00a9 2014 Elad Alfassa <elad@fedoraproject.org>\n# Licensed under the Eiffel Forum License 2.\nfrom __future__ import unicode_literals, absolute_import, print_function, division\n\nimport re\nfrom sopel import web, tools, __version__\nfrom sopel.module import commands, rule, example\nfrom sopel.config.types import ValidatedAttribute, ListAttribute, StaticSection\n\nimport requests\n\nUSER_AGENT = 'Sopel/{} (https://sopel.chat)'.format(__version__)\ndefault_headers = {'User-Agent': USER_AGENT}\nfind_urls = None\n# These are used to clean up the title tag before actually parsing it. Not the\n# world's best way to do this, but it'll do for now.\ntitle_tag_data = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)\nquoted_title = re.compile('[\\'\"]<title>[\\'\"]', re.IGNORECASE)\n# This is another regex that presumably does something important.\nre_dcc = re.compile(r'(?i)dcc\\ssend')\n# This sets the maximum number of bytes that should be read in order to find\n# the title. We don't want it too high, or a link to a big file/stream will\n# just keep downloading until there's no more memory. 640k ought to be enough\n# for anybody.\nmax_bytes = 655360\n\n\nclass UrlSection(StaticSection):\n    # TODO some validation rules maybe?\n    exclude = ListAttribute('exclude')\n    exclusion_char = ValidatedAttribute('exclusion_char', default='!')\n    shorten_url_length = ValidatedAttribute(\n        'shorten_url_length', int, default=0)\n\n\ndef configure(config):\n    config.define_section('url', UrlSection)\n    config.url.configure_setting(\n        'exclude',\n        'Enter regular expressions for each URL you would like to exclude.'\n    )\n    config.url.configure_setting(\n        'exclusion_char',\n        'Enter a character which can be prefixed to suppress URL titling'\n    )\n    config.url.configure_setting(\n        'shorten_url_length',\n        'Enter how many characters a URL should be before the bot puts a'\n        ' shorter version of the URL in the title as a TinyURL link'\n        ' (0 to disable)'\n    )\n\n\ndef setup(bot):\n    global find_urls\n\n    bot.config.define_section('url', UrlSection)\n\n    if bot.config.url.exclude:\n        regexes = [re.compile(s) for s in bot.config.url.exclude]\n    else:\n        regexes = []\n\n    # We're keeping these in their own list, rather than putting then in the\n    # callbacks list because 1, it's easier to deal with modules that are still\n    # using this list, and not the newer callbacks list and 2, having a lambda\n    # just to pass is kinda ugly.\n    if not bot.memory.contains('url_exclude'):\n        bot.memory['url_exclude'] = regexes\n    else:\n        exclude = bot.memory['url_exclude']\n        if regexes:\n            exclude.extend(regexes)\n        bot.memory['url_exclude'] = exclude\n\n    # Ensure that url_callbacks and last_seen_url are in memory\n    if not bot.memory.contains('url_callbacks'):\n        bot.memory['url_callbacks'] = tools.SopelMemory()\n    if not bot.memory.contains('last_seen_url'):\n        bot.memory['last_seen_url'] = tools.SopelMemory()\n\n    def find_func(text, clean=False):\n        def trim_url(url):\n            # clean trailing sentence- or clause-ending punctuation\n            while url[-1] in '.,?!\\'\":;':\n                url = url[:-1]\n\n            # clean unmatched parentheses/braces/brackets\n            for (opener, closer) in [('(', ')'), ('[', ']'), ('{', '}'), ('<', '>')]:\n                if url[-1] is closer and url.count(opener) < url.count(closer):\n                    url = url[:-1]\n\n            return url\n\n        re_url = r'(?u)((?<!%s)(?:http|https|ftp)(?::\\/\\/\\S+))'\\\n            % (bot.config.url.exclusion_char)\n        r = re.compile(re_url, re.IGNORECASE)\n\n        urls = re.findall(r, text)\n        if clean:\n            urls = [trim_url(url) for url in urls]\n        return urls\n\n    find_urls = find_func\n\n\n@commands('title')\n@example('.title http://google.com', '[ Google ] - google.com')\ndef title_command(bot, trigger):\n    \"\"\"\n    Show the title or URL information for the given URL, or the last URL seen\n    in this channel.\n    \"\"\"\n    if not trigger.group(2):\n        if trigger.sender not in bot.memory['last_seen_url']:\n            return\n        matched = check_callbacks(bot, trigger,\n                                  bot.memory['last_seen_url'][trigger.sender],\n                                  True)\n        if matched:\n            return\n        else:\n            urls = [bot.memory['last_seen_url'][trigger.sender]]\n    else:\n        urls = find_urls(trigger)\n\n    results = process_urls(bot, trigger, urls)\n    for title, domain, tinyurl in results[:4]:\n        message = '[ %s ] - %s' % (title, domain)\n        if tinyurl:\n            message += ' ( %s )' % tinyurl\n        bot.reply(message)\n\n\n@rule(r'(?u).*(https?://\\S+).*')\ndef title_auto(bot, trigger):\n    \"\"\"\n    Automatically show titles for URLs. For shortened URLs/redirects, find\n    where the URL redirects to and show the title for that (or call a function\n    from another module to give more information).\n    \"\"\"\n    if re.match(bot.config.core.prefix + 'title', trigger):\n        return\n\n    # Avoid fetching known malicious links\n    if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']:\n        if bot.memory['safety_cache'][trigger]['positives'] > 1:\n            return\n\n    urls = find_urls(trigger)\n    if len(urls) == 0:\n        return\n\n    results = process_urls(bot, trigger, urls)\n    bot.memory['last_seen_url'][trigger.sender] = urls[-1]\n\n    for title, domain, tinyurl in results[:4]:\n        message = '[ %s ] - %s' % (title, domain)\n        if tinyurl:\n            message += ' ( %s )' % tinyurl\n        # Guard against responding to other instances of this bot.\n        if message != trigger:\n            bot.say(message)\n\n\ndef process_urls(bot, trigger, urls):\n    \"\"\"\n    For each URL in the list, ensure that it isn't handled by another module.\n    If not, find where it redirects to, if anywhere. If that redirected URL\n    should be handled by another module, dispatch the callback for it.\n    Return a list of (title, hostname) tuples for each URL which is not handled by\n    another module.\n    \"\"\"\n\n    results = []\n    shorten_url_length = bot.config.url.shorten_url_length\n    for url in urls:\n        if not url.startswith(bot.config.url.exclusion_char):\n            # Magic stuff to account for international domain names\n            try:\n                url = web.iri_to_uri(url)\n            except Exception:  # TODO: Be specific\n                pass\n            # First, check that the URL we got doesn't match\n            matched = check_callbacks(bot, trigger, url, False)\n            if matched:\n                continue\n            # If the URL is over bot.config.url.shorten_url_length,\n            # shorten the URL\n            tinyurl = None\n            if (shorten_url_length > 0) and (len(url) > shorten_url_length):\n                # Check bot memory to see if the shortened URL is already in\n                # memory\n                if not bot.memory.contains('shortened_urls'):\n                    # Initialize shortened_urls as a dict if it doesn't exist.\n                    bot.memory['shortened_urls'] = tools.SopelMemory()\n                if bot.memory['shortened_urls'].contains(url):\n                    tinyurl = bot.memory['shortened_urls'][url]\n                else:\n                    tinyurl = get_tinyurl(url)\n                    bot.memory['shortened_urls'][url] = tinyurl\n            # Finally, actually show the URL\n            title = find_title(url, verify=bot.config.core.verify_ssl)\n            if title:\n                results.append((title, get_hostname(url), tinyurl))\n    return results\n\n\ndef check_callbacks(bot, trigger, url, run=True):\n    \"\"\"\n    Check the given URL against the callbacks list. If it matches, and ``run``\n    is given as ``True``, run the callback function, otherwise pass. Returns\n    ``True`` if the url matched anything in the callbacks list.\n    \"\"\"\n    # Check if it matches the exclusion list first\n    matched = any(regex.search(url) for regex in bot.memory['url_exclude'])\n    # Then, check if there's anything in the callback list\n    for regex, function in tools.iteritems(bot.memory['url_callbacks']):\n        match = regex.search(url)\n        if match:\n            # Always run ones from @url; they don't run on their own.\n            if run or hasattr(function, 'url_regex'):\n                function(bot, trigger, match)\n            matched = True\n    return matched\n\n\ndef find_title(url, verify=True):\n    \"\"\"Return the title for the given URL.\"\"\"\n    try:\n        response = requests.get(url, stream=True, verify=verify,\n                                headers=default_headers)\n        content = b''\n        for byte in response.iter_content(chunk_size=512):\n            content += byte\n            if b'</title>' in content or len(content) > max_bytes:\n                break\n        content = content.decode('utf-8', errors='ignore')\n        # Need to close the connection because we have not read all\n        # the data\n        response.close()\n    except requests.exceptions.ConnectionError:\n        return None\n\n    # Some cleanup that I don't really grok, but was in the original, so\n    # we'll keep it (with the compiled regexes made global) for now.\n    content = title_tag_data.sub(r'<\\1title>', content)\n    content = quoted_title.sub('', content)\n\n    start = content.rfind('<title>')\n    end = content.rfind('</title>')\n    if start == -1 or end == -1:\n        return\n    title = web.decode(content[start + 7:end])\n    title = title.strip()[:200]\n\n    title = ' '.join(title.split())  # cleanly remove multiple spaces\n\n    # More cryptic regex substitutions. This one looks to be myano's invention.\n    title = re_dcc.sub('', title)\n\n    return title or None\n\n\ndef get_hostname(url):\n    idx = 7\n    if url.startswith('https://'):\n        idx = 8\n    elif url.startswith('ftp://'):\n        idx = 6\n    hostname = url[idx:]\n    slash = hostname.find('/')\n    if slash != -1:\n        hostname = hostname[:slash]\n    return hostname\n\n\ndef get_tinyurl(url):\n    \"\"\" Returns a shortened tinyURL link of the URL. \"\"\"\n    tinyurl = \"https://tinyurl.com/api-create.php?url=%s\" % url\n    try:\n        res = requests.get(tinyurl)\n        res.raise_for_status()\n    except requests.exceptions.RequestException:\n        return None\n    # Replace text output with https instead of http to make the\n    # result an HTTPS link.\n    return res.text.replace(\"http://\", \"https://\")\n\n\nif __name__ == \"__main__\":\n    from sopel.test_tools import run_example_tests\n    run_example_tests(__file__)\n", "path": "sopel/modules/url.py"}]}