text: import requests
  from bs4 import BeautifulSoup
  from multiprocessing.dummy import Pool
  import time
  import shutil
  import re
  import os
  import json
  import string
  import glob
  
  BASE_URL = 'https://www.pro-football-reference.com{0}'
  PLAYER_LIST_URL = 'https://www.pro-football-reference.com/players/{0}'
  PLAYER_PROFILE_URL = 'https://www.pro-football-reference.com/players/{0}/{1}'
  PLAYER_GAMELOG_URL = 'https://www.pro-football-reference.com/players/{0}/{1}/gamelog/{2}'
  
  HEADERS = {
      'user-agent': ('Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 '
                     '(KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36')
  }
  
  PROFILE_DIR = 'profile_data'
  STATS_DIR = 'stats_data'
  
  class Scraper():
      """Scraper for pro-football-reference.com to collect NFL player stats"""
  
      def __init__(self, letters_to_scrape=['A'], num_jobs=1, clear_old_data=True, first_player_id=1):
          """Initialize the scraper to get player stats
  
                  Args:
                      - letters_to_scrape (str[]): The site sorts players by the first letter of their
                        last name. This array tells the scraper which letters to scrape data for.
                      - num_jobs (int): Number of concurrent jobs the scraper should run. While Python
                        can't multi-thread, it can manage multiple processes at once, which allows it to
                        utilize time spent waiting for the server to respond.
                      - clear_old_data (boolean): Whether or not the data file should be wiped before
                        starting the scrape.
                      - first_player_id (int): The first ID for a player (set if you are rerunning to avoid duplicates)
  
                  Returns:
                      None
          """
          self.letters_to_scrape = [letter.upper() for letter in letters_to_scrape]
          self.num_jobs = num_jobs
          self.clear_old_data = clear_old_data
          self.session = requests.Session()
          self.start_time = time.time()
          self.cross_process_player_count = 0
          self.first_player_id = first_player_id
  
          if num_jobs > 1:
              self.multiprocessing = True
              self.worker_pool = Pool(num_jobs)
          else:
              self.multiprocessing = False
  
      def scrape_site(self):
          """Pool workers to scrape players by first letter of last name"""
          if self.clear_old_data:
              self.clear_data()
          player_id = self.first_player_id
          for letter in self.letters_to_scrape:
              player_profile_urls = self.get_players_for_letter(letter)
              for player_profile_url in player_profile_urls:
                  player = Player(player_id, player_profile_url, self)
                  try:
                      player.scrape_profile()
                      player.scrape_player_stats()
                  except (KeyboardInterrupt, SystemExit):
                      raise
                  except:
                      # raise
                      print('There was a problem parsing stats for {}'.format(player_profile_url))
                      break
                      # continue
                  self.save_player_profile(player.profile)
                  self.save_player_game_stats(player.game_stats, player.player_id, player.profile['name'])
                  player_id += 1
          self.condense_data()
  
      def condense_data(self):
          """Condense data into two files, a profile file and a stats file"""
          print('Condensing Data...')
          # current_time = time.strftime('%y-%m-%d')
          current_time = time.time()
          condensed_profile_data = []
          all_profile_files = glob.glob('{}/*.json'.format(PROFILE_DIR))
          for file in all_profile_files:
              with open(file, 'rb') as fin:
                  condensed_profile_data.append(json.load(fin))
          print('{} player profiles condensed'.format(len(condensed_profile_data)))
          filename = 'profiles_{}.json'.format(current_time)
          with open(filename, 'w') as fout:
              json.dump(condensed_profile_data, fout)
  
          condensed_game_data = []
          all_game_files = glob.glob('{}/*.json'.format(STATS_DIR))
          for file in all_game_files:
              with open(file, 'rb') as fin:
                  condensed_game_data += json.load(fin)
          print('{} player seasons condensed'.format(len(condensed_game_data)))
          filename = 'games_{}.json'.format(current_time)
          with open(filename, 'w') as fout:
              json.dump(condensed_game_data, fout)
  
      def save_player_profile(self, profile):
          """Save a player's profile as JSON
  
              Args:
                  - profile (dict): Player profile data
  
              Return:
                  None
          """
          filename = '{}/{}_{}.json'.format(PROFILE_DIR, profile['player_id'], profile['name'].replace(' ', '-'))
          try:
              os.makedirs(PROFILE_DIR)
          except OSError:
              pass
          with open(filename, 'w') as fout:
              json.dump(profile, fout)
  
      def save_player_game_stats(self, games, player_id, player_name):
          """Save a list of player games with stats info
  
              Args:
                  - games (dict[]): List of game stats
                  - player_id (int): ID of the player the games belong to
                  - player_name (str): Name of the player the game stats belong to
  
              Return:
                  None
          """
          filename = '{}/{}_{}.json'.format(STATS_DIR, player_id, player_name.replace(' ', '-'))
          try:
              os.makedirs(STATS_DIR)
          except OSError:
              pass
          with open(filename, 'w') as fout:
              json.dump(games, fout)
  
      def get_players_for_letter(self, letter):
          """Get a list of player links for a letter of the alphabet.
              Site organizes players by first letter of last name.
  
              Args:
                  - letter (str): letter of the alphabet uppercased
  
              Returns:
                  - player_links (str[]): the URLs to get player profiles
          """
          response = self.get_page(PLAYER_LIST_URL.format(letter))
          soup = BeautifulSoup(response.content, 'html.parser')
  
          players = soup.find('div', {'id': 'div_players'}).find_all('a')
          return [BASE_URL.format(player['href']) for player in players]
  
      def get_page(self, url, retry_count=0):
          """Use requests to get a page; retry when failures occur
  
              Args:
                  - url (str): The URL of the page to make a GET request to
                  - retry_count (int): Number of times the URL has already been requests
  
              Returns:
                  - response (obj): The Requests response object
          """
          try:
              return self.session.get(url, headers=HEADERS)
          except (KeyboardInterrupt, SystemExit):
              raise
          except:
              retry_count += 1
              if retry_count <= 3:
                  self.session = requests.Session()
                  return self.get_page(url, retry_count)
              else:
                  raise
  
      def clear_data(self):
          """Clear the data directories"""
          try:
              shutil.rmtree(PROFILE_DIR)
          except FileNotFoundError:
              pass
          try:
              shutil.rmtree(STATS_DIR)
          except FileNotFoundError:
              pass
  
  
  class Player():
      """An NFL player"""
  
      def __init__(self, player_id, profile_url, scraper):
          """
              Args:
                  - player_id (int): Unique ID for player
                  - profile_url (str): URL to the player's profile
                  - scraper (obj): instance of Scraper class
  
              Returns:
                  None
          """
          self.player_id = player_id
          self.profile_url = profile_url
          self.scraper = scraper
          self.profile = {
              'player_id': player_id,
              'name': None,
              'position': None,
              'height': None,
              'weight': None,
              'current_team': None,
              'birth_date': None,
              'birth_place': None,
              'death_date': None,
              'college': None,
              'high_school': None,
              'draft_team': None,
              'draft_round': None,
              'draft_position': None,
              'draft_year': None,
              'current_salary': None,
              'hof_induction_year': None
          }
          self.seasons_with_stats = []
          self.game_stats = []
  
      def scrape_profile(self):
          """Scrape profile info for player"""
          print(f"scraping profile_url: {self.profile_url}")
          response = self.scraper.get_page(self.profile_url)
          soup = BeautifulSoup(response.content, 'html.parser')
  
          profile_section = soup.find('div', {'id': 'meta'})
          self.profile['name'] = profile_section.find('h1', {'itemprop': 'name'}).contents[1].contents[0]
          print('scraping {}'.format(self.profile['name']))
  
          profile_attributes = profile_section.find_all('p')
          current_attribute = 1
          num_attributes = len(profile_attributes)
  
          self.profile['position'] = profile_attributes[current_attribute].contents[2].split('\n')[0].split(' ')[1]
          current_attribute += 1
  
          height = profile_attributes[current_attribute].find('span', {'itemprop': 'height'})
          if height is not None:
              self.profile['height'] = height.contents[0]
          weight = profile_attributes[current_attribute].find('span', {'itemprop': 'weight'})
          if weight is not None:
              self.profile['weight'] = weight.contents[0].split('lb')[0]
          if height is not None or weight is not None:
              current_attribute += 1
  
          affiliation_section = profile_section.find('span', {'itemprop': 'affiliation'})
          if affiliation_section is not None:
              self.profile['current_team'] = affiliation_section.contents[0].contents[0]
              current_attribute += 1
  
          birth_date = profile_attributes[current_attribute].find('span', {'itemprop': 'birthDate'})
          if birth_date is not None:
              self.profile['birth_date'] = birth_date['data-birth']
          birth_place_section = profile_attributes[current_attribute].find('span', {'itemprop': 'birthPlace'}).contents
          try:
              self.profile['birth_place'] = re.split('\xa0', birth_place_section[0])[1] + ' ' + birth_place_section[1].contents[0]
          except IndexError:
              pass
          if birth_date is not None or len(birth_place_section) > 0:
              current_attribute += 1
  
          death_section = profile_section.find('span', {'itemprop': 'deathDate'})
          if death_section is not None:
              self.profile['death_date'] = death_section['data-death']
              current_attribute += 1
  
          if profile_attributes[current_attribute].contents[0].contents[0] == 'College':
              self.profile['college'] = profile_attributes[current_attribute].contents[2].contents[0]
              current_attribute += 1
  
          # Skip weighted career AV
          current_attribute += 1
  
          if ((current_attribute + 1) <= num_attributes) and profile_attributes[current_attribute].contents[0].contents[0] == 'High School':
              self.profile['high_school'] = profile_attributes[current_attribute].contents[2].contents[0] + ', ' + profile_attributes[current_attribute].contents[4].contents[0]
              current_attribute += 1
  
          if ((current_attribute + 1) <= num_attributes) and profile_attributes[current_attribute].contents[0].contents[0] == 'Draft':
              self.profile['draft_team'] = profile_attributes[current_attribute].contents[2].contents[0]
              draft_info = profile_attributes[current_attribute].contents[3].split(' ')
              self.profile['draft_round'] = re.findall(r'\d+', draft_info[3])[0]
              self.profile['draft_position'] = re.findall(r'\d+', draft_info[5])[0]
              self.profile['draft_year'] = re.findall(r'\d+', profile_attributes[current_attribute].contents[4].contents[0])[0]
              current_attribute += 1
  
          if ((current_attribute + 1) <= num_attributes) and profile_attributes[current_attribute].contents[0].contents[0] == 'Current cap hit':
              profile_attributes[current_attribute].contents
              self.profile['current_salary'] = profile_attributes[current_attribute].contents[2].contents[0]
              current_attribute += 1
  
          if ((current_attribute + 1) <= num_attributes) and profile_attributes[current_attribute].contents[0].contents[0] == 'Hall of fame':
              self.profile['hof_induction_year'] = profile_attributes[current_attribute].contents[2].contents[0]
              current_attribute += 1
  
          self.seasons_with_stats = self.get_seasons_with_stats(soup)
  
      def scrape_player_stats(self):
          """Scrape the stats for all available games for a player"""
          for season in self.seasons_with_stats:
              if season['year'] == 'Career' or season['year'] == 'Postseason':
                  continue
              self.scrape_season_gamelog(season['gamelog_url'], season['year'])
  
      def scrape_season_gamelog(self, gamelog_url, year):
          """Scrape player stats for a given year
  
              Args:
                  - gamelog_url (str): URL to the stats for a given year
                  - year (int): The year the stats are for
  
              Returns:
                  - stats (dict): All of the player's stats for that year
          """
          response = self.scraper.get_page(gamelog_url)
          soup = BeautifulSoup(response.content, 'html.parser')
          regular_season_table = soup.find('table', {'id': 'stats'})
          if regular_season_table is None:
              return False
          games = regular_season_table.find('tbody').find_all('tr')
  
          playoff_table = soup.find('table', {'id': 'stats_playoffs'})
          if playoff_table is not None:
              games += playoff_table.find('tbody').find_all('tr')
  
          for game in games:
              stats = self.make_player_game_stats(self.player_id, year)
              stats['game_id'] = game.find('td', {'data-stat': 'game_date'}).find('a', href=True)['href'].replace('/boxscores/', '').replace('.htm', '')
              stats['date'] = game.find('td', {'data-stat': 'game_date'}).contents[0].contents[0]
              stats['game_number'] = game.find('td', {'data-stat': 'game_num'}).contents[0]
              stats['age'] = game.find('td', {'data-stat': 'age'}).contents[0]
              stats['team'] = game.find('td', {'data-stat': 'team'}).contents[0].contents[0]
              if game.find('td', {'data-stat': 'game_location'}).contents == ['@']:
                  stats['game_location'] = 'A'
              elif game.find('td', {'data-stat': 'game_location'}).contents == ['N']:
                  stats['game_location'] = 'N'
              else:
                  stats['game_location'] = 'H'
              stats['opponent'] = game.find('td', {'data-stat': 'opp'}).contents[0].contents[0]
              result = game.find('td', {'data-stat': 'game_result'}).contents[0].contents[0]
              stats['game_won'] = (result.split(' ')[0] == 'W')
              stats['player_team_score'] = result.split(' ')[1].split('-')[0]
              stats['opponent_score'] = result.split(' ')[1].split('-')[1]
  
              # Collect passing stats
              pass_attempts = game.find('td', {'data-stat': 'pass_cmp'})
              if pass_attempts is not None and len(pass_attempts) > 0:
                  stats['passing_attempts'] = int(pass_attempts.contents[0])
  
              pass_completions = game.find('td', {'data-stat': 'pass_att'})
              if pass_completions is not None and len(pass_completions) > 0:
                  stats['passing_completions'] = int(pass_completions.contents[0])
  
              pass_yards = game.find('td', {'data-stat': 'pass_yds'})
              if pass_yards is not None and len(pass_yards) > 0:
                  stats['passing_yards'] = int(pass_yards.contents[0])
  
              pass_touchdowns = game.find('td', {'data-stat': 'pass_td'})
              if pass_touchdowns is not None and len(pass_touchdowns) > 0:
                  stats['passing_touchdowns'] = int(pass_touchdowns.contents[0])
  
              pass_interceptions = game.find('td', {'data-stat': 'pass_int'})
              if pass_interceptions is not None and len(pass_interceptions) > 0:
                  stats['passing_interceptions'] = int(pass_interceptions.contents[0])
  
              pass_rating = game.find('td', {'data-stat': 'pass_rating'})
              if pass_rating is not None and len(pass_rating) > 0:
                  stats['passing_rating'] = float(pass_rating.contents[0])
  
              pass_sacks = game.find('td', {'data-stat': 'pass_sacked'})
              if pass_sacks is not None and len(pass_sacks) > 0:
                  stats['passing_sacks'] = int(pass_sacks.contents[0])
  
              pass_sacks_yards_lost = game.find('td', {'data-stat': 'pass_sacked_yds'})
              if pass_sacks_yards_lost is not None and len(pass_sacks_yards_lost) > 0:
                  stats['passing_sacks_yards_lost'] = int(pass_sacks_yards_lost.contents[0])
  
              # Collect rushing stats
              rushing_attempts = game.find('td', {'data-stat': 'rush_att'})
              if rushing_attempts is not None and len(rushing_attempts) > 0:
                  stats['rushing_attempts'] = int(rushing_attempts.contents[0])
  
              rushing_yards = game.find('td', {'data-stat': 'rush_yds'})
              if rushing_yards is not None and len(rushing_yards) > 0:
                  stats['rushing_yards'] = int(rushing_yards.contents[0])
  
              rushing_touchdowns = game.find('td', {'data-stat': 'rush_td'})
              if rushing_touchdowns is not None and len(rushing_touchdowns) > 0:
                  stats['rushing_touchdowns'] = int(rushing_touchdowns.contents[0])
  
              # Collect receiving stats
              receiving_targets = game.find('td', {'data-stat': 'targets'})
              if receiving_targets is not None and len(receiving_targets) > 0:
                  stats['receiving_targets'] = int(receiving_targets.contents[0])
  
              receiving_receptions = game.find('td', {'data-stat': 'rec'})
              if receiving_receptions is not None and len(receiving_receptions) > 0:
                  stats['receiving_receptions'] = int(receiving_receptions.contents[0])
  
              receiving_yards = game.find('td', {'data-stat': 'rec_yds'})
              if receiving_yards is not None and len(receiving_yards) > 0:
                  stats['receiving_yards'] = int(receiving_yards.contents[0])
  
              receiving_touchdowns = game.find('td', {'data-stat': 'rec_td'})
              if receiving_touchdowns is not None and len(receiving_touchdowns) > 0:
                  stats['receiving_touchdowns'] = int(receiving_touchdowns.contents[0])
  
              # Collect kick return stats
              kick_return_attempts = game.find('td', {'data-stat': 'kick_ret'})
              if kick_return_attempts is not None and len(kick_return_attempts) > 0:
                  stats['kick_return_attempts'] = int(kick_return_attempts.contents[0])
  
              kick_return_yards = game.find('td', {'data-stat': 'kick_ret_yds'})
              if kick_return_yards is not None and len(kick_return_yards) > 0:
                  stats['kick_return_yards'] = int(kick_return_yards.contents[0])
  
              kick_return_touchdowns = game.find('td', {'data-stat': 'kick_ret_td'})
              if kick_return_touchdowns is not None and len(kick_return_touchdowns) > 0:
                  stats['kick_return_touchdowns'] = int(kick_return_touchdowns.contents[0])
  
              # Collect punt return stats
              punt_return_attempts = game.find('td', {'data-stat': 'punt_ret'})
              if punt_return_attempts is not None and len(punt_return_attempts) > 0:
                  stats['punt_return_attempts'] = int(punt_return_attempts.contents[0])
  
              punt_return_yards = game.find('td', {'data-stat': 'punt_ret_yds'})
              if punt_return_yards is not None and len(punt_return_yards) > 0:
                  stats['punt_return_yards'] = int(punt_return_yards.contents[0])
  
              punt_return_touchdowns = game.find('td', {'data-stat': 'punt_ret_td'})
              if punt_return_touchdowns is not None and len(punt_return_touchdowns) > 0:
                  stats['punt_return_touchdowns'] = int(punt_return_touchdowns.contents[0])
  
              # Collect defensive stats
              defense_sacks = game.find('td', {'data-stat': 'sacks'})
              if defense_sacks is not None and len(defense_sacks) > 0:
                  stats['defense_sacks'] = float(defense_sacks.contents[0])
  
              defense_tackles = game.find('td', {'data-stat': 'tackles_solo'})
              if defense_tackles is not None and len(defense_tackles) > 0:
                  stats['defense_tackles'] = int(defense_tackles.contents[0])
  
              defense_tackle_assists = game.find('td', {'data-stat': 'tackles_assists'})
              if defense_tackle_assists is not None and len(defense_tackle_assists) > 0:
                  stats['defense_tackle_assists'] = int(defense_tackle_assists.contents[0])
  
              defense_interceptions = game.find('td', {'data-stat': 'def_int'})
              if defense_interceptions is not None and len(defense_interceptions) > 0:
                  stats['defense_interceptions'] = int(defense_interceptions.contents[0])
  
              defense_interception_yards = game.find('td', {'data-stat': 'def_int_yds'})
              if defense_interception_yards is not None and len(defense_interception_yards) > 0:
                  stats['defense_interception_yards'] = int(defense_interception_yards.contents[0])
  
              defense_safeties = game.find('td', {'data-stat': 'safety_md'})
              if defense_safeties is not None and len(defense_safeties) > 0:
                  stats['defense_safeties'] = int(defense_safeties.contents[0])
  
              # Collect kicking stats
              point_after_attemps = game.find('td', {'data-stat': 'xpm'})
              if point_after_attemps is not None and len(point_after_attemps) > 0:
                  stats['point_after_attemps'] = int(point_after_attemps.contents[0])
  
              point_after_makes = game.find('td', {'data-stat': 'xpa'})
              if point_after_makes is not None and len(point_after_makes) > 0:
                  stats['point_after_makes'] = int(point_after_makes.contents[0])
  
              field_goal_attempts = game.find('td', {'data-stat': 'fga'})
              if field_goal_attempts is not None and len(field_goal_attempts) > 0:
                  stats['field_goal_attempts'] = int(field_goal_attempts.contents[0])
  
              field_goal_makes = game.find('td', {'data-stat': 'fgm'})
              if field_goal_makes is not None and len(field_goal_makes) > 0:
                  stats['field_goal_makes'] = int(field_goal_makes.contents[0])
  
              # Collect punting stats
              punting_attempts = game.find('td', {'data-stat': 'punt'})
              if punting_attempts is not None and len(punting_attempts) > 0:
                  stats['punting_attempts'] = int(punting_attempts.contents[0])
  
              punting_yards = game.find('td', {'data-stat': 'punt_yds'})
              if punting_yards is not None and len(punting_yards) > 0:
                  stats['punting_yards'] = int(punting_yards.contents[0])
  
              punting_blocked = game.find('td', {'data-stat': 'punt_blocked'})
              if punting_blocked is not None and len(punting_blocked) > 0:
                  stats['punting_blocked'] = int(punting_blocked.contents[0])
  
              self.game_stats.append(stats)
  
      @staticmethod
      def make_player_game_stats(player_id, year):
          """Factory method to return possible stats to collect for a player in a game
  
              Args:
                  - player_id (int): unique Id for the player
                  - year (int): The year the stats are for
  
              Returns:
                  - game_stats (dict): dictionary with game stats initialized
          """
          return {
              'player_id': player_id,
              'year': year,
              # General stats
              'game_id': None,
              'date': None,
              'game_number': None,
              'age': None,
              'team': None,
              'game_location': None,
              'opponent': None,
              'game_won': None,
              'player_team_score': 0,
              'opponent_score': 0,
              # Passing stats
              'passing_attempts': 0,
              'passing_completions': 0,
              'passing_yards': 0,
              'passing_rating': 0,
              'passing_touchdowns': 0,
              'passing_interceptions': 0,
              'passing_sacks': 0,
              'passing_sacks_yards_lost': 0,
              # Rushing stats
              'rushing_attempts': 0,
              'rushing_yards': 0,
              'rushing_touchdowns': 0,
              # Receiving stats
              'receiving_targets': 0,
              'receiving_receptions': 0,
              'receiving_yards': 0,
              'receiving_touchdowns': 0,
              # Kick return stats
              'kick_return_attempts': 0,
              'kick_return_yards': 0,
              'kick_return_touchdowns': 0,
              # PUnt return stats
              'punt_return_attempts': 0,
              'punt_return_yards': 0,
              'punt_return_touchdowns': 0,
              # Defense
              'defense_sacks': 0,
              'defense_tackles': 0,
              'defense_tackle_assists': 0,
              'defense_interceptions': 0,
              'defense_interception_yards': 0,
              'defense_interception_touchdowns': 0,
              'defense_safeties': 0,
              # Kicking
              'point_after_attemps': 0,
              'point_after_makes': 0,
              'field_goal_attempts': 0,
              'field_goal_makes': 0,
              # Punting
              'punting_attempts': 0,
              'punting_yards': 0,
              'punting_blocked': 0
          }
  
      def get_seasons_with_stats(self, profile_soup):
          """Scrape a list of seasons that has stats for the player
  
              Args:
                  - profile_soup (obj): The BeautifulSoup object for the player profile page
  
              Returns:
                  - seasons (dict[]): List of dictionaries with meta information about season stats
          """
          seasons = []
          gamelog_list = profile_soup.find('div', {'id': 'inner_nav'}).find_all('li')[1].find_all('li')
          if len(gamelog_list) > 0 and gamelog_list[0].contents[0].contents[0] == 'Career':
              for season in gamelog_list:
                  if season.contents[0].contents[0] == 'Player Game Finder':
                      # skip the player game finder
                      continue
                  seasons.append({
                      'year': season.contents[0].contents[0],
                      'gamelog_url': BASE_URL.format(season.contents[0]['href'])
                  })
          return seasons
  
  
  if __name__ == '__main__':
      letters_to_scrape = list(string.ascii_uppercase)
      nfl_scraper = Scraper(letters_to_scrape=letters_to_scrape, num_jobs=10, clear_old_data=False)
  
      nfl_scraper.scrape_site()
  