'This function is the callback function called from os.path.walk, python\'s
help says:
walk(top, func, arg)
Directory tree walk with callback function.
For each directory in the directory tree rooted at top (including top
itself, but excluding \'.\' and \'..\'), call func(arg, dirname, fnames).
dirname is the name of the directory, and fnames a list of the names of
the files and subdirectories in dirname (excluding \'.\' and \'..\').  func
may modify the fnames list in-place (e.g. via del or slice assignment),
and walk will only recurse into the subdirectories whose names remain in
fnames; this can be used to implement a filter, or to impose a specific
order of visiting.  No semantics are defined for, or required of, arg,
beyond that arg is always passed to func.  It can be used, e.g., to pass
a filename pattern, or a mutable object designed to accumulate
statistics.  Passing None for arg is common.'

def _compare_dir(self, arg, directory, flist):

    if self._first:
        self._first = False
        self._start_path = directory
    relative_dir = directory.replace(self._start_path, '')
    if (relative_dir and (not relative_dir.endswith('/'))):
        relative_dir += '/'
    remote_root = self._remote_url_path
    remote_root_with_local_path = remote_root.url_join(relative_dir)
    for fname in flist:
        if os.path.isfile(((directory + os.path.sep) + fname)):
            url = remote_root_with_local_path.url_join(fname)
            response = self._uri_opener.GET(url, cache=True)
            if (not is_404(response)):
                if response.is_text_or_html():
                    fr = FuzzableRequest(response.get_url())
                    self.output_queue.put(fr)
                path = ('%s%s%s' % (directory, os.path.sep, fname))
                self._check_content(response, path)
                self._exist_remote.append(url)
            else:
                self._not_exist_remote.append(url)
