# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + id="gyejscI6OJyQ"
# !pip install scikit-learn==1.0
# !pip install xgboost==1.4.2
# !pip install catboost==0.26.1
# !pip install pandas==1.3.3
# !pip install radiant-mlhub==0.3.0
# !pip install rasterio==1.2.8
# !pip install numpy==1.21.2
# !pip install pathlib==1.0.1
# !pip install tqdm==4.62.3
# !pip install joblib==1.0.1
# !pip install matplotlib==3.4.3
# !pip install Pillow==8.3.2
# !pip install torch==1.9.1
# !pip install plotly==5.3.1


# + id="sPoySFJmN4Vv" outputId="4e9dcac1-fd55-4b55-c410-04b1f242fcb5"
import warnings
warnings.filterwarnings('ignore')
# warnings.filterwarnings('RuntimeWarning')

from radiant_mlhub import Collection
import tarfile
import os
from pathlib import Path
import json
from tqdm import tqdm

import datetime
import rasterio
import numpy as np
import pandas as pd

import gc

gc.collect()

# + id="H9NkhaqSN4V0" outputId="7176ac71-3434-42a0-e345-40efd42adbb8"
bands = ['B01','B02','B03','B04','B05','B06','B07','B08','B8A','B09','B11','B12']
aggregations  = ['mean','median','lq','uq']


for agg in aggregations:
    print(f'---------------Aggregation {agg}-----------------------')
    count = 0
    for band in tqdm(bands):
        print(f'preparing the aggregation merging on {agg} for band {band}')
        for i in range(1,7,1):
            batchid = i 
            if batchid==1:
                df = pd.read_csv(f'/root/Test_data_prep/Batchwise_data/{band}/{agg}_{band}_{batchid}.csv')
            else:
                df1 = pd.read_csv(f'/root/Test_data_prep/Batchwise_data/{band}/{agg}_{band}_{batchid}.csv')
                df  = df.append(df1)
            df = df[df['field_id']!=0]
            
        if count ==0:
            agg_df = df
            
        else:
            agg_df = agg_df.merge(df,on=['field_id'],how='left')
        count = count+1
    print(agg_df.shape)
    agg_df.to_csv(f'/root/Test_data_prep/merged_test_data/test_{agg}.csv',index=False)

# + id="T1ZMTCrbN4V1"


StarcoderdataJupyterScriptsDedupFiltered