Upload 16 files

app.py

@@ -1,10 +1,10 @@
 # external libraries
 import streamlit as st
 import pandas as pd
-import numpy as np
 import os
 import datetime
 
+import pipeline
 from config import Config
 
 config = vars(Config)
@@ -62,7 +62,7 @@ def main():
                                                                         'planning_method_latest'])
 
             st.write(f'Average demand by category "{category}"')
-            st.bar_chart(st.session_state["predictions_df"].loc[st.session_state["predictions_df"]['product_id'] == pid,:].groupby(category)['demand'].mean())
+            st.bar_chart(st.session_state["predictions_df"].groupby(category)['demand'].mean())
 
             input_save = st.checkbox(config['SAVE_CHECKBOX_TEXT'])
             confirm_params = {
@@ -92,38 +92,16 @@ def save(params):
 
     if params['input_save']:
         today = datetime.datetime.today().strftime("%d-%m-%Y")
-        st.session_state["predictions_df"].to_excel(f'{dir}/predictions_{today}.xlsx', index=False)
+        st.session_state["predictions_df"][['product_id','date','demand']].to_excel(f'{dir}/predictions_{today}.xlsx', index=False)
 
 
 # forecasting
 def predict(input_date):
 
-    data = {
-        'product_id': [f'P{1}' for i in range(5)],
-        'date':['2022-01-01','2022-02-01','2022-03-01','2022-04-01','2022-05-01'],
-        'demand': np.random.randint(1, 100, size=5),
-        'product_application': ['A','A','A','B','B']
-    }
-
-    data2 = {
-        'product_id': [f'P{2}' for i in range(5)],
-        'date':['2022-01-01','2022-02-01','2022-03-01','2022-04-01','2022-05-01'],
-        'demand': np.random.randint(1, 100, size=5),
-        'product_application': ['A','A','A','B','B']
-    }
-
-    df1 = pd.DataFrame(data)
-    df2 = pd.DataFrame(data2)
-
-    # Concatenate the two DataFrames vertically
-    combined_df = pd.concat([df1, df2], ignore_index=True)
-
-
     forecast_start_date = input_date[0].strftime("%Y-%m-%d")
     forecast_end_date = input_date[1].strftime("%Y-%m-%d")
-    print(forecast_start_date, forecast_end_date)
 
-    st.session_state["predictions_df"] = combined_df
+    st.session_state["predictions_df"] = pipeline.run(forecast_start_date, forecast_end_date)
 
 if __name__ == "__main__":
     main()

config.py

@@ -1,9 +1,12 @@
+from sklearn.metrics import r2_score
+
 class Config():
 
     def __init__(self):
         pass
 
     target = 'demand'
+    split_local_test = False
 
     not_include_features = [
         target,
@@ -18,7 +21,27 @@ class Config():
         'planning_method_latest'
     ]
 
-    # production
+    scorer = r2_score
+    model_type = 'CATBOOST'
+    fold = 5
+    fold_models_directory = 'models/date_models_test'
+    fold_input_directory = 'maps/date_models_test'
+
+    catboost_params = {
+        'learning_rate': 0.03,
+        'objective':'RMSE',
+        'depth': 5,
+        'early_stopping_rounds':200,
+        'iterations': 2000,
+        'use_best_model': True,
+    #     'eval_metric': CatBoostEvalMetricSMAPE(),
+        'eval_metric': 'R2',
+        'random_state': 42,
+        'allow_writing_files': False,
+        'thread_count':-1
+    }
+
+    # deployment
     MAIN_TITLE = 'Infineon Product Demand Forecasting System'
     SUB_TITLE = 'Data Analytics in Applications'
     ICON_PATH = 'images/infineon-icon-1.png'

pipeline.py

@@ -0,0 +1,53 @@
+from src.data.load_data import get_data, generate_test_data
+from src.features.build_features import prepare_data
+from src.data.preprocess import get_Xy
+from src.utils.helper_functions import load_models, get_predictions, load_parquet
+from config import Config
+import numpy as np
+
+
+import pandas as pd
+
+config = vars(Config)
+
+def run(forecast_start_date, forecast_end_date):
+
+    print('Script Executing...')
+
+
+    generated_test = generate_test_data(forecast_start_date, 
+                                        forecast_end_date, 
+                                        product_ids=load_parquet(f'{config["fold_input_directory"]}/unique_products.parquet').values)
+    
+    generated_test['date'] = pd.to_datetime(generated_test['date'])
+
+    # merge the fixed columns
+    generated_test = pd.merge(load_parquet(f'{config["fold_input_directory"]}/fixed_columns.parquet'), 
+                              generated_test, on=['product_id'], how='right')
+    
+    dataframe = prepare_data(
+        dataframe=pd.concat([generated_test], axis=0),
+        add_datetime_features=True
+    )
+
+    dataframe[config['target']] = np.nan
+
+    X, X_test, y = get_Xy(
+        dataframe=dataframe,
+        not_include=config['not_include_features'],
+        cat_features=config['cat_features'],
+        cat_encoding='category'
+    )
+
+    models = load_models(config['fold_models_directory'])
+
+    y_test_preds = get_predictions(models, X_test)
+
+    generated_test[config['target']] = y_test_preds
+
+    print('Script Done!')
+
+    return generated_test
+
+# if __name__ == '__main__':
+    # run()

src/data/load_data.py

@@ -0,0 +1,92 @@
+import pandas as pd
+import numpy as np
+from src.utils.helper_functions import save_parquet
+import os
+from config import Config
+
+config = vars(Config)
+
+def get_data(
+        data_dir = '../data/raw/input/',
+        file_name = 'demand_data_IFX.csv',
+        date_columns = ['reporting_month_start'],
+        split_local_test = False,
+        target = 'demand',
+        fixed_columns = [
+            'product_id',
+            'product_application',
+            'product_marketing_name',
+            'product_main_family',
+            'planning_method_latest',
+        ],
+        prediction_interval = ('2023-11-01', '2024-07-01')
+    ):
+
+    print('Loading data...')
+    
+    dataframe = pd.read_csv(os.path.join(data_dir, file_name), parse_dates=date_columns)
+    dataframe['date'] = pd.to_datetime(dataframe['reporting_month_start'].dt.date)
+    dataframe.sort_values(by='date', inplace=True)
+
+    if split_local_test:
+        train, test = split_train_test(
+            dataframe = dataframe
+        )
+
+        test_min_date, test_max_date = test.date.min(), test.date.max()
+    else:
+        train, test = dataframe, None
+
+        test_min_date, test_max_date = prediction_interval
+
+
+    generated_test = generate_test_data(
+        start_date=test_min_date,
+        end_date=test_max_date,
+        product_ids=train.product_id.unique()
+    )
+
+    generated_test['date'] = pd.to_datetime(generated_test['date'])
+
+    # merge the fixed columns
+    generated_test = pd.merge(train[fixed_columns].drop_duplicates(subset=fixed_columns), generated_test, on=['product_id'], how='right')
+
+    save_parquet(
+        dataframe= train[fixed_columns].drop_duplicates(subset=fixed_columns),
+        path=f'{config["fold_input_directory"]}/fixed_columns.parquet'
+    )
+
+    # merge the ground-truth
+    if split_local_test:
+        generated_test = pd.merge(test[[target,'date','product_id']], generated_test, on=['product_id','date'], how='right')
+        generated_test[target] = generated_test[target].fillna(0)
+    else:
+        generated_test[target] = np.nan
+
+    # generate fixed train
+    generated_train = train[[target,'date'] + fixed_columns]
+
+    y_test = generated_test[target]
+    generated_test.drop(target, axis=1, inplace=True)
+
+    return generated_train, generated_test, y_test
+
+def split_train_test(dataframe):
+
+    train = dataframe[dataframe['date'] < pd.to_datetime('2022-11-01')]
+    test = dataframe[(dataframe['date'] >= pd.to_datetime('2022-11-01'))&
+                     (dataframe['date'] <= pd.to_datetime('2023-07-01'))]
+    
+    return train, test
+
+def generate_test_data(start_date, end_date, product_ids):
+
+    # Generate a range of monthly start dates
+    monthly_starts = pd.date_range(start=start_date, end=end_date, freq='MS')
+    monthly_starts = pd.DataFrame(monthly_starts, columns=['date'])
+
+    product_ids = pd.DataFrame(product_ids, columns=['product_id'])
+
+    joined_df = product_ids.merge(monthly_starts, how='cross')
+
+    return joined_df

src/data/preprocess.py

@@ -0,0 +1,23 @@
+import pandas as pd
+
+def get_Xy(
+        dataframe,
+        not_include,
+        cat_features,
+        target='demand',
+        cat_encoding='category'
+    ):
+    
+    print('Preprocessing...')
+
+    tmp_df = dataframe.copy()
+
+    features = [col for col in tmp_df.columns if col not in not_include]
+
+    if cat_encoding == 'category':
+        tmp_df[cat_features] = tmp_df[cat_features].astype('category')
+
+    X, y = tmp_df.loc[~tmp_df[target].isnull(), features], tmp_df.loc[~tmp_df[target].isnull(), target]
+    X_test = tmp_df.loc[tmp_df[target].isnull(), features]
+
+    return X, X_test, y

src/features/build_features.py

@@ -0,0 +1,20 @@
+def prepare_data(
+        dataframe,
+        add_datetime_features=True
+    ):
+
+    print('Building features...')
+    
+    if add_datetime_features:
+        dataframe = datetime_features(dataframe)
+
+    return dataframe
+
+def datetime_features(dataframe, date='date', suffix=''):
+
+    dataframe[f'{suffix}_month'] = dataframe[date].dt.month
+    dataframe[f'{suffix}_year'] = dataframe[date].dt.year
+    dataframe[f'{suffix}_quarter'] = dataframe[date].dt.quarter
+    dataframe[f'{suffix}_weekofyear'] = dataframe[date].dt.isocalendar().week
+
+    return dataframe

src/models/evaluate_model.py

@@ -0,0 +1,68 @@
+from src.models.train_model import train_model
+from src.utils.helper_functions import post_process
+import numpy as np
+
+class MonthlyKFold:
+    def __init__(self, n_splits=3):
+        self.n_splits = n_splits
+        
+    def split(self, X, y=None, groups=None):
+        dates = 12 * X["_year"] + X["_month"]
+        timesteps = sorted(dates.unique().tolist())
+        X = X.reset_index()
+        
+        for t in timesteps[-self.n_splits:]:
+            idx_train = X[dates.values < t].index
+            idx_test = X[dates.values == t].index
+            
+            yield idx_train, idx_test
+            
+    def get_n_splits(self, X, y=None, groups=None):
+        return self.n_splits
+    
+
+def evaluate(
+        X, y,
+        model_params,
+        cat_features,
+        scorer,
+        FOLD=5,
+        model_type='CATBOOST'
+):
+        
+        print('Evaluating...')
+    
+        tscv = MonthlyKFold(FOLD)
+
+
+        scores = []
+        models = []
+        iterations = []
+        test_preds = []
+        
+        oof = np.zeros(len(X))
+        for i, (train_index, valid_index) in enumerate(tscv.split(X)):
+
+                print(f'FOLD:{i+1}')
+
+                X_train, y_train = X.iloc[train_index, :], y.iloc[train_index]
+                X_valid, y_valid = X.iloc[valid_index, :], y.iloc[valid_index]
+
+                model = train_model(
+                        train=(X_train, y_train),
+                        model_params=model_params,
+                        model_type=model_type,
+                        cat_features=cat_features,
+                        valid=(X_valid, y_valid))
+                
+                score = scorer(y_valid, post_process(model.predict(X_valid)))
+                print(f'Score:{score:.5f}')
+
+                models.append(model)
+                scores.append(score)
+
+        print(f"Scores:{scores}")
+        print(f'Mean Score:{np.mean(scores):.5f} +- {np.std(scores):.3f}')
+
+        return models, scores
+        

src/models/train_model.py

@@ -0,0 +1,27 @@
+from catboost import CatBoostRegressor
+
+def train_model(
+        train,
+        model_params,
+        model_type,
+        cat_features,
+        valid=None,
+):
+    
+    X_train, y_train = train
+    
+    if model_type == 'CATBOOST':
+
+        model = CatBoostRegressor(**model_params,
+                                  cat_features=cat_features)
+        
+        if valid:
+            X_valid, y_valid = valid
+            eval_set=[(X_valid,y_valid)]
+        
+        model.fit(X_train,y_train,
+            eval_set=eval_set,
+            verbose=200
+        )
+
+    return model

src/utils/helper_functions.py

@@ -0,0 +1,68 @@
+import pickle
+import os
+import numpy as np
+from datetime import datetime
+import pandas as pd
+
+def save_models(models, model_type, directory):
+
+    print('Saving models...')
+
+    for i, model in enumerate(models):
+
+        with open(f'{directory}/{model_type}_FOLD_{i+1}.pkl', 'wb') as file:  
+            pickle.dump(model, file)
+
+
+def load_models(directory):
+        
+    print('Loading models...')
+
+    models = []
+    # List all files in the directory
+    files = os.listdir(directory)
+
+    pkl_files = [file for file in files if file.endswith('.pkl')]
+
+    for file in pkl_files:
+        with open(os.path.join(directory, file), 'rb') as file:  
+            model = pickle.load(file)
+        models.append(model)
+    
+    return models
+
+def get_predictions(models, X_test):
+
+    print('Forecasting test data...')
+
+    preds = []
+    for model in models:
+        preds.append(post_process(model.predict(X_test)))
+
+    return np.mean(preds, axis=0)
+
+def post_process(predictions):
+    
+    predictions = predictions.clip(0)
+    
+    return predictions
+
+def save_results(dataframe, file_name):
+
+    print('Saving results...')
+
+    today_date = datetime.now().strftime("%Y-%m-%d")
+
+    dataframe.to_excel(f'demand_predictions/{file_name}_{today_date}.xlsx', index=False)
+
+def save_parquet(dataframe, path):
+    dataframe.to_parquet(path, index=False)
+
+def load_parquet(path):
+    return pd.read_parquet(path)
+
+
+
+
+    
+