# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from datetime import datetime

pd.set_option('display.float_format', lambda x: '%.3f' % x)

sns.set(rc={'figure.figsize':(16,9)})

# # Intro and basic datasets' stats

# These are unique MAIDs seen in a Link viewshed with site id = mn-06-138318 

# +
start_date, end_date = '2020-08-01 15:00:00-0400', '2020-08-07 15:00:00-0400'

begin_dt = datetime.strptime(start_date, '%Y-%m-%d %H:%M:%S%z')
end_dt = datetime.strptime(end_date, '%Y-%m-%d %H:%M:%S%z')
# -

ds_spec = f"mn-06-138318_165_{begin_dt.strftime('%Y%m%d_%H%M%S%z')}_{end_dt.strftime('%Y%m%d_%H%M%S%z')}"

ds_dir = '../../analytics/data'

# These are people/cars/trucks counts from camera with locationId=165 (cameraId=4) ***seen over a period of 60secs***

camera_ts = pd.read_csv(f'../../analytics/data/camera_{ds_spec}.csv')

camera_ts.info()

camera_ts.describe()

camera_ts.head()

camera_ts[camera_ts['people'] == 0].shape

camera_ts[camera_ts['people'] > 0].shape

# Let's make camera data continuous

seconds_range = pd.date_range(begin_dt,end_dt,freq='S')

seconds_df = pd.DataFrame(seconds_range, columns=['full_idx'])

seconds_df

camera_ts['idx'] = camera_ts['timestamp'].apply(lambda ts: datetime.fromtimestamp(ts, tz=begin_dt.tzinfo))

camera_ts['idx']

dataset_df = pd.merge_ordered(seconds_df, camera_ts, how='left', left_on='full_idx', right_on='idx')

dataset_df

dataset_df['people'][dataset_df['people'].isna()] = 0
dataset_df['cars'][dataset_df['cars'].isna()] = 0
dataset_df['trucks'][dataset_df['trucks'].isna()] = 0

dataset_df

# The idea is to align aggregated camera data (new people detected as a change in the number of people between two consecutive timestamped frames) to observational data (# of unique people's devices in 60 sec time cells).
#
# For that we let's consider the difference between the people counts from the current and the previous frames. Positive difference means new people entered a frame. If it is < 0, it means they left the frame (0 means people "swapped", but on a second level resolution it's unlikely)
#
# So if `d` is the difference array and `v` is the people counts array for each of consecutive frames, then `d` over a sequence of frames is calculated as follows:
#
# ```
# d[0] = v[0] - 0      # initial "previous" value is 0 
# d[1] = v[1] - v[0]
# d[2] = v[2] - v[1]
# d[3] = v[3] - v[2]
# ```

dataset_df['total'] = dataset_df['people'] + dataset_df['cars']
dataset_df['ppl_delta_cnt'] = dataset_df['total'] - dataset_df['total'].shift(1, fill_value=0)

dataset_df['ppl_delta_cnt'] = dataset_df['people'] - dataset_df['people'].shift(1, fill_value=0)

dataset_df

dataset_df['timestamp'] = dataset_df['full_idx'].apply(lambda dt: int(dt.timestamp()))

dataset_df['ts_cell'] = (dataset_df['timestamp'] // 60) * 60

dataset_df

# +
grouped_df = dataset_df.groupby('ts_cell')

sum_cnt = grouped_df['ppl_delta_cnt'].agg(lambda df: df[df > 0].sum())

ppl_sum_df = pd.DataFrame(sum_cnt)
# -

ppl_sum_df['ts_cell'] = ppl_sum_df.index.values
ppl_sum_df['t'] = ppl_sum_df['ts_cell'].apply(lambda ts: datetime.fromtimestamp(ts))

ppl_sum_df['ma_cnt_cam'] = ppl_sum_df['ppl_delta_cnt'].ewm(span=20, adjust=False).mean()

ppl_sum_df.head()

ppl_sum_df.iloc[:60*48].plot(x='t', y='ma_cnt_cam', figsize=(16, 9))

ppl_sum_df.plot(x='t', y='ma_cnt_cam', figsize=(16, 9))

observation_ts = pd.read_csv(f'{ds_dir}/observations_{ds_spec}.csv')

observation_ts.head()

observation_ts.info()

observation_ts.describe()

observation_ts['t'] = observation_ts['TS_CELL'].apply(lambda ts: datetime.fromtimestamp(ts))

observation_ts['ma_cnt_obs'] = observation_ts['CNT'].ewm(span=20, adjust=False).mean()

observation_ts.plot(x='t', y='ma_cnt_obs', figsize=(16, 9), color='goldenrod')

obs_cam_df = pd.merge(observation_ts, ppl_sum_df, how='outer', right_index=True, left_on='TS_CELL')

obs_cam_df.head()

obs_cam_df[['ma_cnt_obs', 'ma_cnt_cam']].corr()



plot_df = obs_cam_df.melt(id_vars='t_x', value_vars=['ma_cnt_cam', 'ma_cnt_obs'], var_name='dataset', value_name='people_cnt')

sns.lineplot(data=plot_df, x='t_x', y='people_cnt', hue='dataset')

obs_cam_df['ma_cnt_cam_norm'] = obs_cam_df['ma_cnt_cam'] / obs_cam_df['ma_cnt_cam'].max()
obs_cam_df['ma_cnt_obs_norm'] = obs_cam_df['ma_cnt_obs'] / obs_cam_df['ma_cnt_obs'].max()

plot_df_norm = obs_cam_df.melt(id_vars='t_x', value_vars=['ma_cnt_cam_norm', 'ma_cnt_obs_norm'], var_name='dataset', value_name='people_cnt')

StarcoderdataJupyterScriptsDedupFiltered