Scraping Instagram followers page using selenium and python
Question: I have a question related to scraping the instagram followers page. I have a
code but it displays only 9 followers. Kindly help me.

    
    
    from selenium import webdriver
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.webdriver.support import expected_conditions as EC
    
    
    def login(driver):
        username = "xxxx@yahoo.com"  # <username here>
        password = "xxxx"  # <password here>
    
        # Load page
        driver.get("https://www.instagram.com/accounts/login/")
    
        # Login
        driver.find_element_by_xpath("//div/input[@name='username']").send_keys(username)
        driver.find_element_by_xpath("//div/input[@name='password']").send_keys(password)
        driver.find_element_by_xpath("//span/button").click()
    
        # Wait for the login page to load
        WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.LINK_TEXT, "See All")))
    
    
    def scrape_followers(driver, account):
        # Load account page
        driver.get("https://www.instagram.com/{0}/".format(account))
    
        # Click the 'Follower(s)' link
        driver.find_element_by_partial_link_text("follower").click()
    
        # Wait for the followers modal to load
        xpath = "//div[@style='position: relative; z-index: 1;']/div/div[2]/div/div[1]"
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.XPATH, xpath)))
    
        # You'll need to figure out some scrolling magic here. Something that can
        # scroll to the bottom of the followers modal, and know when its reached
        # the bottom. This is pretty impractical for people with a lot of followers
    
        # Finally, scrape the followers
        xpath = "//div[@style='position: relative; z-index: 1;']//ul/li/div/div/div/div/a"
        followers_elems = driver.find_elements_by_xpath(xpath)
    
        return [e.text for e in followers_elems]
    
    
    if __name__ == "__main__":
        driver = webdriver.Firefox()
        try:
            login(driver)
            followers = scrape_followers(driver, "instagram")
            print(followers)
        finally:
            driver.quit()
    

This code was taken from another page. I dont understand how to scroll down
the followers page.


Answer: You can easily scroll down using javascript by increasing the scrollTop. You
run this scroll until the amount of users in the list no longer changes.

The difference in the amount of users can be checked using the following
function

    
    
    count = 0
    
    def check_difference_in_count(driver):
        global count
    
        new_count = len(driver.find_elements_by_xpath("//div[@role='dialog']//li"))
    
        if count != new_count:
            count = new_count
            return True
        else:
            return False
    

And the following script scrolls down the user container until it has reached
the bottom

    
    
    while 1:
        # scroll down
        driver.execute_script("document.querySelector('div[role=dialog] ul').parentNode.scrollTop=1e100")
    
        try:
            WebDriverWait(driver, 5).until(check_difference_in_count)
        except:
            break
    

