# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import pandas as pd
import gzip
import os

def parse(path):
  g = gzip.open(path, 'rb')
  for l in g:
    yield eval(l)

def getDF(path):
  i = 0
  df = {}
  for d in parse(path):
    df[i] = d
    i += 1
  return pd.DataFrame.from_dict(df, orient='index')

df = getDF('c:/Users/Owen/Downloads/reviews_Books_5.json.gz')
# -

df.shape

df.head()

df1 = df.sample(1000000)

df1.to_csv('books_small.csv')

df.to_csv('books_big.csv')

StarcoderdataJupyterScriptsDedupFiltered