%matplotlib inline
import matplotlib.pyplot as plt
import pandas as pd
import operator
import sys
import os
from utils import zillow_helpers
sys.path.insert(0,'../')
%load_ext autoreload
%autoreload 2

"""
Explanation: Data-Fixing with Zillow Dataset
Useful Links:
- Dataframe merging
- Pipe-separated values to df
- Storing and reading pickles for storage of "good" Dataframes
End of explanation
"""


datadir1 = 'data/46/ZAsmt/'
datadir2 = 'data/46/ZTrans/'
datadir3 = 'data/50/ZAsmt/'
datadir4 = 'data/50/ZTrans/'

"""
Explanation: Set some data folders here. Be sure to "ignore" your actual data folders when pushing to Github.
End of explanation
"""


os.listdir(datadir1)

"""
Explanation: List the text files in the folder. This could be helpful for making iterables later!
End of explanation
"""


f = open(datadir1+'SaleData.txt', 'r')

# Only read a few lines at a time with these files!!!
for lines in range(3):
    line = f.readline()
    print(line)

"""
Explanation: Known syntax for opening and "reading" lines from text files contained below
End of explanation
"""


df = pd.read_csv(datadir1+'SaleData.txt', sep="|", index_col=False, header=None, low_memory=False)
df.head(3)

df2 = zillow_helpers.txt_to_df(datadir1+'Main.txt')
df2.head(3)

"""
Explanation: It's good to note that creation of a dataframe from these massive file-sets does not seem to require a lot of processing power. Attempting to read and print the whole dataset probably will crash your kernel! Be sure to read a few lines of the head each time to check!
End of explanation
"""


df.to_pickle("df.pickle")

df3 = pd.read_pickle("df.pickle")

df3.head(3)

"""
Explanation: If your data frame is good, then pickle it! It will save your dataframe and prevent the need to re-create it later.
End of explanation
"""


chelpMain = 'column_helper/main.csv'
chelpPI = 'column_helper/propertyinfo.csv'

chelpMainOpen = open(chelpMain, 'r')
chelpPIOpen = open(chelpPI, 'r')

for lines in range(3):
    line = chelpMainOpen.readline()
    print(line)

for lines in range(3):
    line = chelpPIOpen.readline()
    print(line)

df4 = zillow_helpers.txt_to_df(datadir2+'Main.txt')

df4.head(3)

chelpMain = 'column_helper/main.csv'
chelpPI = 'column_helper/propertyinfo.csv'

chelpMainOpen = open(chelpMain, 'r')
chelpPIOpen = open(chelpPI, 'r')

chelpMainArray = []
for line in chelpMainOpen:
    chelpMainArray.append(line.rstrip())

print(chelpMainArray)

df4.columns=chelpMainArray

df4

df5 = zillow_helpers.txt_to_column_df('data/46/ZTrans/Main.txt', 'column_helper/main.csv')
df5.head(5)

df6 = zillow_helpers.txt_to_column_df('data/46/ZTrans/PropertyInfo.txt', 'column_helper/propertyinfo.csv')
df6.head(5)

df7 = df5.merge(df6, left_on='TransId', right_on='TransId', how='outer')
df7.head(5)

df5.info()

df6.info()

df7.info()

"""
Explanation: Organizing via Syntax
Time to start looking at organizing/merging files! It seems that the best way to organize is by separating dataframes into ZAsmt and ZTrans first.
First, lets use our column helper files in order to create arrays to be used to name columns later.
End of explanation
"""


from utils import zillow_helpers as zh

datadir2 = 'data/46/ZTrans/'
datadir4 = 'data/50/ZTrans/'
chelpMain = 'column_helper/main.csv'
chelpPI = 'column_helper/propertyinfo.csv'

df8 = zh.txt_to_column_df(datadir2+'Main.txt', chelpMain)
df9 = zh.txt_to_column_df(datadir4+'Main.txt', chelpMain)

df8.info()

df8.head(3)

df9.info()

df9.head(3)

df10 = pd.concat([df8,df9])

df10.info()

"""
Explanation: Concatenating
End of explanation
"""


