title: Webscraping paginated website with different page layout using Puppeteer
body: <p>I'm trying to paginate through 200+ pages on this website and not all of them have the same layout. For example: GPA breakdown and SAT/ACT(in testing policy row) Super Scores are different across these schools. And for the harvard college page, SAT/ACT Super Scores just flat out don't show up. I'm having problems trying to format this for the csv because these data show up for one page but not for some other ones. </p>
  
  <p>Links:
  <a href="https://www.princetonreview.com/college/georgia-institute-technology-1022905" rel="nofollow noreferrer">https://www.princetonreview.com/college/georgia-institute-technology-1022905</a>
  <a href="https://www.princetonreview.com/college/princeton-university-1024041" rel="nofollow noreferrer">https://www.princetonreview.com/college/princeton-university-1024041</a>
  <a href="https://www.princetonreview.com/college/harvard-college-1022984" rel="nofollow noreferrer">https://www.princetonreview.com/college/harvard-college-1022984</a></p>
  
  <p>CSV file I currently have: <a href="https://ibb.co/Tc3DyFR" rel="nofollow noreferrer">https://ibb.co/Tc3DyFR</a> This sample only shows the difference in Super Scores because I have not scraped GPA breakdown yet. However, both layouts are different across different pages. </p>
  
  <p>Code:</p>
  
  <pre><code>const puppeteer = require('puppeteer');
  const fs = require('fs-extra');
  
  (async function main() {
  try{
      var names = await (fs.readFileSync('names.csv', 'utf8')).split('\n');
  
      const browser = await puppeteer.launch({ headless: false });
      const page = await browser.newPage();
      page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36');
  
      await page.goto('https://www.princetonreview.com/college/harvard-college-1022984#!admissions');
      //await fs.writeFile('out.csv', 'School Name,Applicants,Acceptance Rate,Average HS GPA,GPA: Over 3.75,GPA: 3.50-3.74,GPA: 3.25 - 3.49,GPA: 3.00 - 3.24,GPA: 2.50 - 2.99,GPA: 2.00 - 2.49,SAT Reading and Writing,SAT Math,ACT\n');
  
      await fs.appendFile('out.csv', `"${names[1]}",`);
      const numbers = await page.evaluate(() =&gt; {
          let nums = document.querySelectorAll('.number-callout');
  
          let arr = Array.prototype.slice.call(nums);
          let text_arr = [];
          for(let i = 0; i &lt; arr.length; i++){
              if(arr[i].innerText == "")
                  continue;
              text_arr.push(arr[i].innerText.trim());
          }
          return text_arr;
      });
  
      for(var e of numbers){
          await fs.appendFile('out.csv', `"${e}",`);
      }
      await fs.appendFile('out.csv', `\n`);
      //console.log(numbers);
      await browser.close();
  }catch(e){
      console.log('our error', e);
  }
  })();
  </code></pre>