title: Puppeteer: Scrolling down twitter timeline stops
body: <p>I am having trouble with scraping all tweet URLs on a user timeline with puppeteer. </p>
  
  <p>With puppeteer, the script is supposed to scroll down the timeline on each iteration of the while loop in the <code>scrollToEnd</code> function until it hits the bottom. In order to monitor the progress, I made the script output the value of the <code>previousHeight</code> variable, which is the current <code>scrollheight</code> of <code>document.body</code> evaluated everytime before the scrolling is executed. </p>
  
  <p>However the scrolling stops once the output value turns 285,834. What's puzzling is that the script neither does break out of the while loop nor does the <code>page.waitForFunction</code> method throw a timeout error. </p>
  
  <p>How should I rewrite the <code>scrollToEnd</code> function or any other part of the script so that the function ends properly? </p>
  
  <p>Here is a snippet of my code. Irrelevant functions are left out for brevity. </p>
  
  <pre><code>const puppeteer = require('puppeteer');
  
  var UserUrls = ['https://twitter.com/someuser'];
  
  // more functions here
  
  async function scrollToEnd(
      page,
      ScrollDelay = 1000
  ) {
      try {
          let previousHeight = 0;
          let notEnd = await page.waitForFunction(`document.body.scrollHeight &gt; ${previousHeight}`);
          while (notEnd) {
              previousHeight = await page.evaluate('document.body.scrollHeight');
              await page.evaluate('window.scrollBy(0, document.body.scrollHeight)');
              await page.waitFor(ScrollDelay);
  
              notEnd = await page.waitForFunction(`document.body.scrollHeight &gt; ${previousHeight}`);
              console.log(previousHeight)
          };
          return;
      } catch (e) {
          return;
      };
  };
  
  (async () =&gt; {
      const browser = await puppeteer.launch();
      const page = await browser.newPage();
      var tweetUrls = [];
      for (let UserUrl of UserUrls) {
          await page.goto(UserUrl);
          await page.evaluate((async () =&gt; {
              await scrollToEnd(page);
          })());
          await page.screenshot({ path: 'PageEnd.png' });
          tweetUrls = await getTweetUrls(page, extractItems, 100);
      };
      await browser.close();
      console.log(tweetUrls);
  })();
  </code></pre>