code: import pandas as pd
  import pandas_gbq as gbq
  import json
  from google.oauth2 import service_account
  from IPython.core.debugger import set_trace
  from pathlib import Path
  import time
  from . import accounts
  
  '''
  Configuration
  '''
  proj = 'global-news-crawl'
  table_downloaded = 'news_dataset.downloaded'
  table_trashed = 'news_dataset.trashed'
  credentials = service_account.Credentials.from_service_account_info(accounts.bigquery_account)
  #credentials = service_account.Credentials.from_service_account_file('global-news-crawl-c48d7cd9aa81.json')
  
  localpath_to_downloaded = 'newsdata/downloaded'
  localpath_to_trashed = 'newsdata/trashed'
  
  
  class Recorder:
      def __init__(self, storage='local'):
          self.storage = storage
          self.ids = self._get_ids(storage)
  
      def _query_ids_from_bigquery(self, tb):
          qry = 'SELECT id FROM `{}`'.format(proj + '.' + tb)
          return gbq.read_gbq(qry, credentials=credentials).id
  
      def _retreive_ids_from_local(self, path):
          return [p.stem for p in Path(path).glob('**/*.json')]
  
  
      def _get_ids(self, storage):
          start = time.time()
          print('checking ' + storage + ' storage... ', end='')
  
          if storage == 'bigquery':
              ids_downloaded = self._query_ids_from_bigquery(table_downloaded)
              ids_trashed = self._query_ids_from_bigquery(table_trashed)
  
          elif storage == 'local':
              ids_downloaded = self._retreive_ids_from_local(localpath_to_downloaded)
              ids_trashed = self._retreive_ids_from_local(localpath_to_trashed)
  
          ids_downloaded_set = set(ids_downloaded)
          ids_trashed_set = set(ids_trashed)
  
          if len(ids_downloaded) != len(ids_downloaded_set):
              '''downloaded articles의 uniqueness'''
              raise self.DuplicatesInSingleTable('duplicated in downloaded')
  
          if len(ids_trashed) != len(ids_trashed_set):
              '''trashed articles의 uniqueness'''
              raise self.DuplicatesInSingleTable('duplicated in trashed')
  
          if len(ids_downloaded_set & ids_trashed_set) != 0:
              '''downloaded와 trashed 간의 uniqueness'''
              raise self.DuplicatesBetweenTwoTables('duplicated between downloaded and trashed')
  
          ids = ids_downloaded_set | ids_trashed_set
  
          print('done ({howlong:.2f} seconds)'.format(howlong=time.time()-start))
          print('we have total {} articles ({} downloaded, {} trashed)'.format(len(ids), len(ids_downloaded_set), len(ids_trashed_set)))
          return ids
  
  
      def has(self, id):
          return id in self.ids
  
  
      def update(self, downloaded=None, trashed=None, chunksize=1000, subdir_len=3):
          '''
          downloaded or trashed = {
              id0: {...},
              id1: {...},
              ...
          }
          '''
          if self.storage == 'bigquery':
              self._update_bigquery('downloaded', downloaded, chunksize)
              self._update_bigquery('trashed', trashed, chunksize)
  
          elif self.storage == 'local':
              self._update_local('downloaded', downloaded, subdir_len)
              self._update_local('trashed', trashed, subdir_len)
  
  
      def _update_local(self, newstype, newsdict, subdir_len):
          if newsdict is not None:
              if newstype == 'downloaded':
                  path = localpath_to_downloaded
              elif newstype == 'trashed':
                  path = localpath_to_trashed
  
              for id, article in newsdict.items():
  #                 '''
  #                 local storage의 경우,
  #                 downloaded는 downloaded 폴더에,
  #                 trashed는 trashed/id[:3] 폴더에 저장했다
  #                 나중에 혹시 local에 저장할 일이 있다면, 저장방식을 통일하는 것이 좋겠다 (2019.10.31)
  #                 '''
  #                 if newstype == 'downloaded':
  #                     _dir = Path(path)
  #                 elif newstype == 'trashed':
  #                     _dir = Path(path + '/' + id[:subdir_len])
  
                  _dir = Path(path + '/' + id[:subdir_len])
                  _dir.mkdir(parents=True, exist_ok=True)
                  fname = id + '.json'
                  fpath = _dir / fname
                  with fpath.open('w') as f:
                      json.dump(article, f)
  
  
      def _update_bigquery(self, newstype, newsdict, chunksize):
          if newsdict is not None:
              if newstype == 'downloaded':
                  tb = table_downloaded #+ '2'
              elif newstype == 'trashed':
                  tb = table_trashed #+ '2'
  
              df = pd.DataFrame.from_dict(newsdict, orient='index')
              df.index.name = 'id'
              df = df.reset_index()
              gbq.to_gbq(df, tb, project_id=proj, if_exists='append', chunksize=chunksize, credentials=credentials, progress_bar=False)
  
      class DuplicatesInSingleTable(Exception):
          pass
  
      class DuplicatesBetweenTwoTables(Exception):
          pass
  
apis: pandas.DataFrame.from_dict