def get_result_df(y_true_unscale, y_pred_unscale, ano_index, look_back,target_col='cpu_usage'):
    """
    Add prediction and anomaly value to dataframe.
    """
    result_df = pd.DataFrame({"y_true": y_true_unscale.squeeze(), "y_pred": y_pred_unscale.squeeze()})
    result_df['anomalies'] = 0
    result_df.loc[result_df.index[ano_index], 'anomalies'] = 1
    result_df['anomalies'] = result_df['anomalies'] > 0
    return result_df 

def plot_anomalies_value(date, y_true, y_pred, anomalies):
    """
    plot the anomalies value
    """
    fig, axs = plt.subplots(figsize=(16,6))
    
    axs.plot(date, y_true,color='blue', label='y_true')
    axs.plot(date, y_pred,color='orange', label='y_pred')
    axs.scatter(date[anomalies].tolist(), y_true[anomalies], color='red', label='anomalies value')
    axs.set_title('the anomalies value')
    
    plt.xlabel('datetime')
    plt.legend(loc='upper left')
    plt.show()

"""
Explanation: Unsupervised Anomaly Detection based on Forecasts
Anomaly detection detects data points in data that does not fit well with the rest of data. In this notebook we demonstrate how to do anomaly detection using Chronos's built-in model MTNet
For demonstration, we use the publicly available cluster trace data cluster-trace-v2018 of Alibaba Open Cluster Trace Program. You can find the dataset introduction <a href="https://github.com/alibaba/clusterdata/blob/master/cluster-trace-v2018/trace_2018.md" target="_blank">here</a>. In particular, we use machine usage data to demonstrate anomaly detection, you can download the separate data file directly with <a href="http://clusterdata2018pubcn.oss-cn-beijing.aliyuncs.com/machine_usage.tar.gz" target="_blank">machine_usage</a>.
Helper functions
This section defines some helper functions to be used in the following procedures. You can refer to it later when they're used.
End of explanation
"""


import os 
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline

df_1932 = pd.read_csv("m_1932.csv", header=None, usecols=[1,2,3], names=["time_step", "cpu_usage","mem_usage"])

"""
Explanation: Download raw dataset and load into dataframe
Now we download the dataset and load it into a pandas dataframe.Steps are as below:
* First, download the raw data <a href="http://clusterdata2018pubcn.oss-cn-beijing.aliyuncs.com/machine_usage.tar.gz" target="_blank">machine_usage</a>. Or run the script get_data.sh to download the raw data.It will download the resource usage of each machine from m_1932 to m_2085. 
* Second, run grep m_1932 machine_usage.csv &gt; m_1932.csv to extract records of machine 1932. Or run extract_data.sh.We use machine 1932 as an example in this notebook.You can choose any machines in the similar way.
* Finally, use pandas to load m_1932.csv into a dataframe as shown below.
End of explanation
"""


df_1932.head()

df_1932.sort_values(by="time_step", inplace=True)
df_1932.reset_index(inplace=True)
df_1932.sort_values(by="time_step").plot(y="cpu_usage", x="time_step", figsize=(16,6),title="cpu_usage of machine 1932")

"""
Explanation: Below are some example records of the data
End of explanation
"""


df_1932.reset_index(inplace=True)
df_1932["time_step"] = pd.to_datetime(df_1932["time_step"], unit='s', origin=pd.Timestamp('2018-01-01'))

"""
Explanation: Data pre-processing
Now we need to do data cleaning and preprocessing on the raw data. Note that this part could vary for different dataset. 
For the machine_usage data, the pre-processing convert the time step in seconds to timestamp starting from 2018-01-01.
End of explanation
"""


from zoo.chronos.data import TSDataset
from sklearn.preprocessing import StandardScaler

# we look back one hour data which is of the frequency of 1min.
look_back = 60
horizon = 1

tsdata_train, tsdata_val, tsdata_test = TSDataset.from_pandas(df_1932, dt_col="time_step", target_col="cpu_usage", with_split=True, val_ratio = 0.1, test_ratio=0.1)
standard_scaler = StandardScaler()

for tsdata in [tsdata_train, tsdata_val, tsdata_test]:
    tsdata.resample(interval='1min', merge_mode="mean")\
          .impute(mode="last")\
          .gen_dt_feature()\
          .scale(standard_scaler, fit=(tsdata is tsdata_train))\
          .roll(lookback=look_back, horizon=horizon, feature_col = ["HOUR"])\

x_train, y_train = tsdata_train.to_numpy()
x_val, y_val = tsdata_val.to_numpy()
x_test, y_test = tsdata_test.to_numpy()
y_train, y_val, y_test = y_train[:, 0, :], y_val[:, 0, :], y_test[:, 0, :]
x_train.shape, y_train.shape, x_val.shape, y_val.shape, x_test.shape, y_test.shape

"""
Explanation: Feature Engineering & Data Preperation
For feature engineering, we use hour as feature in addition to the target cpu usage.
For data preperation, we resample the average of cpu_usage in minutes, impute the data to handle missing data and scale the data. At last we generate the sample in numpy ndarray for Forecaster to use.
We generate a built-in TSDataset to complete the whole processing.
End of explanation
"""


from zoo.chronos.forecaster.mtnet_forecaster import MTNetForecaster

"""
Explanation: Time series forecasting
End of explanation
"""


mtnet_forecaster = MTNetForecaster(target_dim=horizon,
                                   feature_dim=x_train.shape[-1],
                                   long_series_num=3,
                                   series_length=15
    )

"""
Explanation: First, we initialize a mtnet_forecaster according to input data shape. Specifcally, look_back should equal (long_series_num+1)*series_length . Details refer to chronos docs <a href="https://analytics-zoo.github.io/master/#Chronos/overview" target="_blank">here</a>.
End of explanation
"""


# mtnet requires reshape of input x before feeding into model.
x_train_mtnet = mtnet_forecaster.preprocess_input(x_train)
x_val_mtnet = mtnet_forecaster.preprocess_input(x_val)
x_test_mtnet = mtnet_forecaster.preprocess_input(x_test)

"""
Explanation: MTNet needs to preprocess the X into another format, so we call MTNetForecaster.preprocess_input on train_x and test_x.
End of explanation
"""


%%time
hist = mtnet_forecaster.fit(x = x_train_mtnet, y = y_train, batch_size=128, epochs=20)

"""
Explanation: Now we train the model and wait till it finished.
End of explanation
"""


y_pred_val = mtnet_forecaster.predict(x_val_mtnet)
y_pred_test = mtnet_forecaster.predict(x_test_mtnet)

y_pred_val_unscale = tsdata_val.unscale_numpy(np.expand_dims(y_pred_val, axis=1))[:, 0, :]
y_pred_test_unscale = tsdata_test.unscale_numpy(np.expand_dims(y_pred_test, axis=1))[:, 0, :]
y_val_unscale = tsdata_val.unscale_numpy(np.expand_dims(y_val, axis=1))[:, 0, :]
y_test_unscale = tsdata_test.unscale_numpy(np.expand_dims(y_test, axis=1))[:, 0, :]

"""
Explanation: Use the model for prediction and inverse the scaling of the prediction results.
End of explanation
"""


# evaluate with sMAPE
from zoo.orca.automl.metrics import Evaluator
smape = Evaluator.evaluate("smape", y_test_unscale, y_pred_test_unscale)
print(f"sMAPE is {'%.2f' % smape}")

"""
Explanation: Calculate the symetric mean absolute percentage error.
End of explanation
"""


from zoo.chronos.detector.anomaly import ThresholdDetector

ratio=0.01

thd=ThresholdDetector()
thd.set_params(ratio=ratio)
thd.fit(y_val_unscale,y_pred_val_unscale)
print("The threshold of validation dataset is:",thd.th)

anomaly_scores_val = thd.score()
val_res_ano_idx = np.where(anomaly_scores_val > 0)[0]
print("The index of anomalies in validation dataset is:",val_res_ano_idx)

anomaly_scores_test = thd.score(y_test_unscale,y_pred_test_unscale)
test_res_ano_idx = np.where(anomaly_scores_test > 0)[0]
print("The index of anoalies in test dataset is:",test_res_ano_idx)

"""
Explanation: Anomaly detection
End of explanation
"""


val_result_df = get_result_df(y_val_unscale, y_pred_val_unscale, val_res_ano_idx, look_back)
test_result_df = get_result_df(y_test_unscale, y_pred_test_unscale, test_res_ano_idx, look_back)

"""
Explanation: Get a new dataframe which contains y_true，y_pred，anomalies value.
End of explanation
"""


plot_anomalies_value(val_result_df.index, val_result_df.y_true, val_result_df.y_pred, val_result_df.anomalies)

plot_anomalies_value(test_result_df.index, test_result_df.y_true, test_result_df.y_pred, test_result_df.anomalies)

"""
Explanation: Draw anomalies in line chart.
End of explanation
"""


