'Return a set of canonical representations of ``url``.  The default
implementation uses just the server hostname and path of the URL,
ignoring any query parameters, fragments, etc. The canonical
representations must be unique across all URLs for this news source. If
they are not, then internal links may be resolved incorrectly.
:param is_link: Is True if the URL is coming from an internal link in
an HTML file. False if the URL is the URL used to
download an article.'

def canonicalize_internal_url(self, url, is_link=True):

    try:
        parts = urlparse.urlparse(url)
    except Exception:
        self.log.error(('Failed    to    parse    url:    %r,    ignoring' % url))
        return frozenset()
    return frozenset([(parts.netloc, (parts.path or '').rstrip('/'))])
