import os
import math
import time
import geohash
import geojson
from geojson import MultiLineString
from shapely import geometry
import shapefile
import numpy
import datetime as dt
import pandas as pd
import logging

logger = logging.getLogger(__name__)
source_shape_file_path = "C:/temp/2018/"
threshold = 60*60
cols = ['start', 'end','start_epoch_round','end_epoch_round','start_epoch_round_dt','end_epoch_round_dt']
times = []

for root,dirs,files in os.walk(source_shape_file_path):
    for file in files:
        with open(os.path.join(root,file),"r") as auto:
            if file.endswith(".shp"):
                try:
                    filename = file.replace(".shp","")
                    shape=shapefile.Reader(source_shape_file_path+filename+"/"+file)
                    for r in shape.iterRecords():
                        start_time = dt.datetime.strptime(r[1], '%Y%j %H%M')
                        end_time = dt.datetime.strptime(r[2], '%Y%j %H%M')
                        epoch_s = dt.datetime.timestamp(dt.datetime.strptime(r[1], '%Y%j %H%M'))
                        epoch_e = dt.datetime.timestamp(dt.datetime.strptime(r[2], '%Y%j %H%M'))
                        # sometimes start is later than end time, we'll assume the earlier time is start
                        epoch_end_round = round(max(epoch_s,epoch_e) / threshold) * threshold
                        epoch_start_round = round(min(epoch_s,epoch_e) / threshold) * threshold
                        epoch_end_round_dt = dt.datetime.utcfromtimestamp(3600 * ((max(epoch_s,epoch_e) + 1800) // 3600))
                        epoch_start_round_dt = dt.datetime.utcfromtimestamp(3600 * ((min(epoch_s,epoch_e) + 1800) // 3600))
                        times.append([start_time,end_time,epoch_start_round,epoch_end_round,epoch_start_round_dt,epoch_end_round_dt])
                    break
                except:
                    logger.error('failed to parse file:'+source_shape_file_path+filename+"/")
                continue


df = pd.DataFrame(times, columns=cols)
df.to_csv('noaa_times.csv')
         
[((14, 9, 14, 36), 'logging.getLogger', 'logging.getLogger', ({(14, 27, 14, 35): '__name__'}, {}), '(__name__)', False, 'import logging\n'), ((20, 23, 20, 54), 'os.walk', 'os.walk', ({(20, 31, 20, 53): 'source_shape_file_path'}, {}), '(source_shape_file_path)', False, 'import os\n'), ((44, 5, 44, 38), 'pandas.DataFrame', 'pd.DataFrame', (), '', True, 'import pandas as pd\n'), ((22, 18, 22, 41), 'os.path.join', 'os.path.join', ({(22, 31, 22, 35): 'root', (22, 36, 22, 40): 'file'}, {}), '(root, file)', False, 'import os\n'), ((26, 26, 26, 84), 'shapefile.Reader', 'shapefile.Reader', ({(26, 43, 26, 83): "source_shape_file_path + filename + '/' + file"}, {}), "(source_shape_file_path + filename + '/' + file)", False, 'import shapefile\n'), ((28, 37, 28, 76), 'datetime.datetime.strptime', 'dt.datetime.strptime', ({(28, 58, 28, 62): 'r[1]', (28, 64, 28, 75): '"""%Y%j %H%M"""'}, {}), "(r[1], '%Y%j %H%M')", True, 'import datetime as dt\n'), ((29, 35, 29, 74), 'datetime.datetime.strptime', 'dt.datetime.strptime', ({(29, 56, 29, 60): 'r[2]', (29, 62, 29, 73): '"""%Y%j %H%M"""'}, {}), "(r[2], '%Y%j %H%M')", True, 'import datetime as dt\n'), ((30, 56, 30, 95), 'datetime.datetime.strptime', 'dt.datetime.strptime', ({(30, 77, 30, 81): 'r[1]', (30, 83, 30, 94): '"""%Y%j %H%M"""'}, {}), "(r[1], '%Y%j %H%M')", True, 'import datetime as dt\n'), ((31, 56, 31, 95), 'datetime.datetime.strptime', 'dt.datetime.strptime', ({(31, 77, 31, 81): 'r[2]', (31, 83, 31, 94): '"""%Y%j %H%M"""'}, {}), "(r[2], '%Y%j %H%M')", True, 'import datetime as dt\n')]