'All per-URL processing comes together here, regardless of Input.
Here we run filters, remove duplicates, spill to disk as needed, etc.'

def ConsumeURL(self, url, allow_fragment):

    if (not url):
        return
    if (not url.Validate(self._base_url, allow_fragment)):
        return
    accept = None
    for filter in self._filters:
        accept = filter.Apply(url)
        if (accept != None):
            break
    if (not (accept or (accept == None))):
        url.Log(prefix='FILTERED', level=2)
        return
    if (fnmatch.fnmatchcase(url.loc, self._wildurl1) or fnmatch.fnmatchcase(url.loc, self._wildurl2)):
        url.Log(prefix='IGNORED    (output    file)', level=2)
        return
    hash = url.MakeHash()
    if self._urls.has_key(hash):
        dup = self._urls[hash]
        if (dup > 0):
            dup = (dup + 1)
            self._urls[hash] = dup
            if (self._dup_max < dup):
                self._dup_max = dup
        url.Log(prefix='DUPLICATE')
        return
    self._urls[hash] = 1
    self._set.append(url)
    self._stat.Consume(url)
    url.Log()
    if (len(self._set) >= MAXURLS_PER_SITEMAP):
        self.FlushSet()
