Image hosting site image downloader using requests and BeautifulSoup I went about this the way I have because Selenium is slow and inconvenient (opens browser) and I was unable to find the href attribute in the link element using BeautifulSoup. Html included below. On the downside, this appears to only ever find 25 images.

#! python3

# Saves photos to file from flickr.com using specified search term

import bs4
import logging
import os
import re
import requests
import shutil
import sys
import time


def find_link(element):
    """Finds link using a regular expression"""

    link_regex = re.compile(r"//c\d+.staticflickr.com/\d+/\d+/\w+\.jpg")
    # dictionary of element attributes
    element_attr_dict = element.attrs

    # get list of element attribute values wherein image link lies   
    attr_list = [element_attr_dict[key] for key in element_attr_dict.keys()]
    attr_string = ""

    # link exists within a string list element
    for element in attr_list:
        if type(element) == str:
            attr_string += element
    match = link_regex.search(attr_string)
    if match:
        link = "https:" + match.group()
    else:
        link = None
    return link


def main():
    """Downloads specified type/number of 'flickr' images to folder

    Takes three command line arguments: "filename, search_term, number_
    images". Saves images to folder. Number of images saved based upon
    number requested by user or number found during search. Whichever is
    lower.

    Arguments:
    search_term   -- search image site using this term
    number_images -- maximum number of images to save to folder
    """

    try:
        search_term, number_images = sys.argv[1:]
        number_images = int(number_images)
    except ValueError:
        print("Something went wrong. Command line input must be of \
format: 'filename searchterm numberimages'")
        return

    links = []
    # make folder to store photos and name using search term
    html_path = r"C:\Users\Dave\Desktop\2016Coding\AutomateBoring\11" + \
                r"-WebScraping\flickrhtml.txt"
    path = \
    r"C:\Users\Dave\Desktop\2016Coding\AutomateBoring\11-WebScraping" + \
    r"\gimages\requests"
    folder_path = os.path.join(path, search_term)
    if os.path.exists(folder_path):
        shutil.rmtree(folder_path)
    os.makedirs(folder_path)

    print("Finding photos...")

    # get links to photos
    res = requests.get("https://www.flickr.com/search/?text=" + search_term)
    res.raise_for_status()
    soup = bs4.BeautifulSoup(res.text, "html.parser")
    found_elems = soup.select(".photo-list-photo-view")

    # incase number found images &lt; requested
    number_save_images = min(number_images, len(found_elems))

    print("Found {} images".format(number_save_images))

    for found_elem in found_elems[:number_save_images]:
        link = find_link(found_elem)
        links.append(link)

    # write images to file
    print("Writing images to folder...")
    for image_link in links:
        basename = os.path.basename(image_link)
        save_file_name = os.path.join(folder_path, basename)
        res = requests.get(image_link)
        res.raise_for_status()

        with open(save_file_name, "wb") as f:
            for chunk in res.iter_content(100000):
                f.write(chunk)

    print("Images saved at: {}".format(folder_path))
    print("*****Done*****")

if __name__ == "__main__":
    main()
According to PEP8 standard library import should be first.

Function find_link(element):


The docstring of the function find_link should probably say something about the argument type.
Why is the argument element (and not element_attributes) when it looks like all you use is element.attrs?
The list comprehension could be written as: attr_list = list(element_attr_dict.values()) 
But, you don't need it at all. You can simply write for element in attr_dict.values().


To summarize, you can delete most lines in the beginning. Instead start with

def find_link(element_attributes):
    link_regex = re.compile(r"//c\d+.staticflickr.com/\d+/\d+/\w+\.jpg")
    attr_string = ""
    for element in element_attributes.values():
        ...


As for the main function:


Investigate argparse.
Use logging instead of print.
Split the main function to sub-functions. At least saving an image from a link should probably be one.
The for loop where you append a link to a list can be expressed as a list comprehension.


I hope that helps!