archiver/util.py
def wget_output_path(link, look_in=None):
    """calculate the path to the wgetted .html file, since wget may
    adjust some paths to be different than the base_url path.

    See docs on wget --adjust-extension (-E)
    """

    # if we have it stored, always prefer the actual output path to computed one
    if link.get('latest', {}).get('wget'):
        return link['latest']['wget']

    urlencode = lambda s: quote(s, encoding='utf-8', errors='replace')

    if link['type'] in ('PDF', 'image'):
        return urlencode(link['base_url'])

    # Since the wget algorithm to for -E (appending .html) is incredibly complex
    # instead of trying to emulate it here, we just look in the output folder
    # to see what html file wget actually created as the output
    wget_folder = link['base_url'].rsplit('/', 1)[0].split('/')
    look_in = os.path.join(ARCHIVE_DIR, link['timestamp'], *wget_folder)

    if look_in and os.path.exists(look_in):
        html_files = [
            f for f in os.listdir(look_in)
            if re.search(".+\\.[Hh][Tt][Mm][Ll]?$", f, re.I | re.M)
        ]
        if html_files:
            return urlencode(os.path.join(*wget_folder, html_files[0]))

    return None

    # If finding the actual output file didn't work, fall back to the buggy
    # implementation of the wget .html appending algorithm
    # split_url = link['url'].split('#', 1)
    # query = ('%3F' + link['url'].split('?', 1)[-1]) if '?' in link['url'] else ''

    # if re.search(".+\\.[Hh][Tt][Mm][Ll]?$", split_url[0], re.I | re.M):
    #     # already ends in .html
    #     return urlencode(link['base_url'])
    # else:
    #     # .html needs to be appended
    #     without_scheme = split_url[0].split('://', 1)[-1].split('?', 1)[0]
    #     if without_scheme.endswith('/'):
    #         if query:
    #             return urlencode('#'.join([without_scheme + 'index.html' + query + '.html', *split_url[1:]]))
    #         return urlencode('#'.join([without_scheme + 'index.html', *split_url[1:]]))
    #     else:
    #         if query:
    #             return urlencode('#'.join([without_scheme + '/index.html' + query + '.html', *split_url[1:]]))
    #         elif '/' in without_scheme:
    #             return urlencode('#'.join([without_scheme + '.html', *split_url[1:]]))
    #         return urlencode(link['base_url'] + '/index.html')
calculate the path to the wgetted .html file, since wget may
    adjust some paths to be different than the base_url path.

    See docs on wget --adjust-extension (-E)
    
calculate the path to the wgetted .html file, since wget may
adjust some paths to be different than the base_url path.

See docs on wget --adjust-extension (-E)
calculate the path to the wgetted . html file since wget may adjust some paths to be different than the base_url path . See docs on wget -- adjust - extension ( - E )
def wget_output_path(link, look_in=None):
    if link.get('latest', {}).get('wget'):
        return link['latest']['wget']
    urlencode = lambda s: quote(s, encoding='utf-8', errors='replace')
    if link['type'] in ('PDF', 'image'):
        return urlencode(link['base_url'])
    wget_folder = link['base_url'].rsplit('/', 1)[0].split('/')
    look_in = os.path.join(ARCHIVE_DIR, link['timestamp'], *wget_folder)
    if look_in and os.path.exists(look_in):
        html_files = [
            f for f in os.listdir(look_in)
            if re.search(".+\\.[Hh][Tt][Mm][Ll]?$", f, re.I | re.M)
        ]
        if html_files:
            return urlencode(os.path.join(*wget_folder, html_files[0]))
    return None
def wget_output_path ( link , look_in = None ) : if link . get ( 'latest' , { } ) . get ( 'wget' ) : return link [ 'latest' ] [ 'wget' ] urlencode = lambda s : quote ( s , encoding = 'utf-8' , errors = 'replace' ) if link [ 'type' ] in ( 'PDF' , 'image' ) : return urlencode ( link [ 'base_url' ] ) wget_folder = link [ 'base_url' ] . rsplit ( '/' , 1 ) [ 0 ] . split ( '/' ) look_in = os . path . join ( ARCHIVE_DIR , link [ 'timestamp' ] , * wget_folder ) if look_in and os . path . exists ( look_in ) : html_files = [ f for f in os . listdir ( look_in ) if re . search ( ".+\\.[Hh][Tt][Mm][Ll]?$" , f , re . I | re . M ) ] if html_files : return urlencode ( os . path . join ( * wget_folder , html_files [ 0 ] ) ) return None
calculate the path to the wgetted .html file, since wget may
adjust some paths to be different than the base_url path.
calculate the path to the wgetted . html file since wget may adjust some paths to be different than the base_url path .
"""calculate the path to the wgetted .html file, since wget may
    adjust some paths to be different than the base_url path.

    See docs on wget --adjust-extension (-E)
    """ # if we have it stored, always prefer the actual output path to computed one # Since the wget algorithm to for -E (appending .html) is incredibly complex # instead of trying to emulate it here, we just look in the output folder # to see what html file wget actually created as the output # If finding the actual output file didn't work, fall back to the buggy # implementation of the wget .html appending algorithm # split_url = link['url'].split('#', 1) # query = ('%3F' + link['url'].split('?', 1)[-1]) if '?' in link['url'] else '' # if re.search(".+\\.[Hh][Tt][Mm][Ll]?$", split_url[0], re.I | re.M): #     # already ends in .html #     return urlencode(link['base_url']) # else: #     # .html needs to be appended #     without_scheme = split_url[0].split('://', 1)[-1].split('?', 1)[0] #     if without_scheme.endswith('/'): #         if query: #             return urlencode('#'.join([without_scheme + 'index.html' + query + '.html', *split_url[1:]])) #         return urlencode('#'.join([without_scheme + 'index.html', *split_url[1:]])) #     else: #         if query: #             return urlencode('#'.join([without_scheme + '/index.html' + query + '.html', *split_url[1:]])) #         elif '/' in without_scheme: #             return urlencode('#'.join([without_scheme + '.html', *split_url[1:]])) #         return urlencode(link['base_url'] + '/index.html')
link
look_in