adhoc_scraper_core.py
def fetch_image_urls(urls_file, fetch_page_image_urls):
    """
    fetch_page_image_urls should be a callback that handles image fetching for each URL in 'urls_file'.
    We assume a flow where we need to paginate through the user's feed and that relevant URLs are already present in the HTML.
    """
    urls = []
    with open(urls_file) as f:
        for url in f:
            urls.append(url)
    print(urls[0])
    print("------Synchronous step. Scrape all image URLs from webpages.--------")

    images = []
    session = HTMLSession()

    for url in urls:
        response = session.get(url, headers=__headers)
        page_img_urls = fetch_page_image_urls(response)

        images.extend(page_img_urls)

    print(f"Synchronous step done. {len(images)} images were found on urls given. Saving image urls to text file.")
    with open('images.txt', 'w') as f:
        for img_url in images:
            f.write(f"{img_url}\n")
    return images

    fetch_page_image_urls should be a callback that handles image fetching for each URL in 'urls_file'.
    We assume a flow where we need to paginate through the user's feed and that relevant URLs are already present in the HTML.
    
fetch_page_image_urls should be a callback that handles image fetching for each URL in 'urls_file'.
We assume a flow where we need to paginate through the user's feed and that relevant URLs are already present in the HTML.
fetch_page_image_urls should be a callback that handles image fetching for each URL in ' urls_file ' . We assume a flow where we need to paginate through the user ' s feed and that relevant URLs are already present in the HTML .
def fetch_image_urls(urls_file, fetch_page_image_urls):
    urls = []
    with open(urls_file) as f:
        for url in f:
            urls.append(url)
    print(urls[0])
    print("------Synchronous step. Scrape all image URLs from webpages.--------")
    images = []
    session = HTMLSession()
    for url in urls:
        response = session.get(url, headers=__headers)
        page_img_urls = fetch_page_image_urls(response)
        images.extend(page_img_urls)
    print(f"Synchronous step done. {len(images)} images were found on urls given. Saving image urls to text file.")
    with open('images.txt', 'w') as f:
        for img_url in images:
            f.write(f"{img_url}\n")
    return images
def fetch_image_urls ( urls_file , fetch_page_image_urls ) : urls = [ ] with open ( urls_file ) as f : for url in f : urls . append ( url ) print ( urls [ 0 ] ) print ( "------Synchronous step. Scrape all image URLs from webpages.--------" ) images = [ ] session = HTMLSession ( ) for url in urls : response = session . get ( url , headers = __headers ) page_img_urls = fetch_page_image_urls ( response ) images . extend ( page_img_urls ) print ( f"Synchronous step done. {len(images)} images were found on urls given. Saving image urls to text file." ) with open ( 'images.txt' , 'w' ) as f : for img_url in images : f . write ( f"{img_url}\n" ) return images
fetch_page_image_urls should be a callback that handles image fetching for each URL in 'urls_file'.
fetch_page_image_urls should be a callback that handles image fetching for each URL in ' urls_file ' .
"""
    fetch_page_image_urls should be a callback that handles image fetching for each URL in 'urls_file'.
    We assume a flow where we need to paginate through the user's feed and that relevant URLs are already present in the HTML.
    """
urls_file
fetch_page_image_urls