# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/omarelsebaey002/test01/blob/main/automated_flow_all_included_new_threshold_algo.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="sxK973JUpvmp"
# **How to run**
#
#
# 1.   Upload helper.py to a folder on the drive
#       *   For example create a folder called "trading app" and then upload the helper.py to this folder
#
#
# 2. Set the path to the helper.py in the next cell in this line 
#       * sys.path.append('/content/drive/MyDrive/trading_app')
#
# 3. For single run 
#       * Set the variables as elaborated in the cell#2
#
# 4. For multi runs with a file containing the input files 
#       * Call the funciton generate_net_threshold_for_multi_runs and pass the file to it like Cell#3
#       * The columns of the file should have columns names like this : 
#         * input_file,output_file,ticker,start_date,end_date
#

# + colab={"base_uri": "https://localhost:8080/"} id="pv6iZ1O1NxgW" outputId="f13d499a-5d74-44d6-c05d-bc81c198ce0c"
import os
from google.colab import drive
import sys
sys.path.append('/content/drive/MyDrive/trading_app')
drive.mount('/content/drive', force_remount=True)

# + id="1vihho9zx_fe"
import pandas as pd
import numpy as np 
pd.options.mode.chained_assignment = None
from datetime import  date,time
import os
from datetime import datetime 
def generate_net_threshold_for_multi_runs(afile) : 
  df = pd.read_csv(afile)
  df.fillna("", inplace=True)
  for i in range(len(df)) : 
    outputfile         = df['output_file'][i]
    input_file         = df['input_file'][i]
    START_DATE         = df['start_date'][i]
    END_DATE           = df['end_date'][i]
    TICKER             = str(df['ticker'][i])
    print("[INFO] Generating for file number: ",i,)
    generate_net_threshold(input_file,START_DATE,END_DATE,TICKER,outputfile)
  print("[INFO] Finished generating net thresholds for all files")

def generate_1_min_and_pe_and_ce(file_name,ticker,START_DATE,END_DATE):
    main_df = pd.read_csv(file_name)
    main_df.fillna("", inplace=True)
    if ("Ticker" in main_df.columns and (ticker != None)):
        main_df = main_df[main_df['Ticker'] == ticker]
        main_df.reset_index(inplace=True,drop=True)
    swing = 300 
    if (ticker == "BANKNIFTY") : 
        swing = 300 
    elif (ticker == "NIFTY"):
        swing = 100 
    
    main_df['Date/Time'] = pd.to_datetime(main_df['Date/Time'], dayfirst=True )
    main_df['Date'] = main_df['Date/Time'].dt.date
    main_df['Time'] = main_df['Date/Time'].dt.time
    START_DATE = pd.to_datetime(START_DATE,dayfirst=True)
    END_DATE   = pd.to_datetime(END_DATE,dayfirst=True)

    print("[INFO] Selecting from START_DATE: ",START_DATE )
    print("[INFO]           to   START_DATE: ",END_DATE )
    
    main_df = main_df[(main_df['Date/Time'] >= START_DATE) & (main_df['Date/Time'] <= END_DATE) ]
    main_df.reset_index(inplace=True,drop=True)
    
    #print("Main selected : \n" , main_df) 
    if (main_df.shape[0] == 0 ) : 
      print("[INFO] Didn't find any entries matching the selected dates and tickers")
      print("[INFO] Exiting the generation process for this file")
      return pd.DataFrame(),pd.DataFrame(),pd.DataFrame()
    else:
      print("[INFO] Found {} entries matching the selection criteria ".format(main_df.shape[0]))
    print("[INFO] Processing Start...")
    main_df_1_min = main_df.copy()
    
    main_df["New_Close"] = 0.0
    

    for i in range(len(main_df)):
        advance = i+2 
        if (advance < len(main_df)):
            if (main_df['Date'][i] == main_df['Date'][advance]):
              advance_value = main_df['Close'][advance]
            else:
              #print("Did not select for {} the one from the next day point {}".format(main_df['Date/Time'][i],main_df['Date/Time'][advance]))
              advance_value = main_df['Close'][i]
        else : 
            advance_value = main_df['Close'][i]
        main_df['New_Close'][i] = advance_value
    
    to_be_selected = [] 
    three_min_storage = pd.DataFrame()
    for adate in set(main_df['Date']) : 
      selected_from_one_day_chunk = pd.DataFrame()
      one_day_chunk = pd.DataFrame()
      to_be_selected_from_one_day = []

      one_day_chunk = main_df[main_df['Date'] == adate ]
      one_day_chunk.sort_values(by=['Date/Time'], inplace=True)
      one_day_chunk.reset_index(inplace=True,drop=True)
      #print("This is one day chunck \n",one_day_chunk )
      for i in range(len(one_day_chunk)) : 
        if ((i%3) == 0 ):
          to_be_selected_from_one_day.append(i)
          #print("\n\n\n==Will select this element list : " , i , "From day : " , adate )
          #print("Value: ", one_day_chunk.iloc[[i]])
      #print("+++++Finally : Will select : " , to_be_selected_from_one_day , "For Date : " , adate )
      selected_from_one_day_chunk = one_day_chunk.iloc[to_be_selected_from_one_day]
      #print("\n\n\nFor date: {} Selected these values : \n{}".format(adate,selected_from_one_day_chunk))
      three_min_storage = three_min_storage.append(selected_from_one_day_chunk)
    #print("\n\n\n\nThree min storage is now : \n",three_min_storage)
    
    three_min_storage.reset_index(inplace=True,drop=True)

    main_df_3_min = three_min_storage.copy()
    main_df_3_min.drop('Close',axis='columns', inplace=True)
    main_df_3_min['Close'] = main_df_3_min['New_Close']
    main_df_3_min.drop('New_Close',axis='columns', inplace=True)
    main_df_3_min.reset_index(inplace=True,drop=True)
    main_df_3_min = main_df_3_min[['Date/Time','Close']]
    main_df_3_min['ROUNDUP'] = np.round(main_df_3_min["Close"]/100)*100
    main_df_3_min['ROUNDUP'] = main_df_3_min['ROUNDUP'].astype(int)
    main_df_3_min['Date/Time']  = main_df_3_min['Date/Time'] + np.timedelta64(3, 'm')

    main_df_3_min_pe = main_df_3_min.copy()
    main_df_3_min_ce = main_df_3_min.copy()

    main_df_3_min_pe['Strike'] = main_df_3_min_pe['ROUNDUP'] + swing
    main_df_3_min_ce['Strike'] = main_df_3_min_pe['ROUNDUP'] - swing

    main_df_3_min_pe['Type'] = "PE"
    main_df_3_min_ce['Type'] = "CE"

    main_df_3_min_pe['Contract'] = main_df_3_min_pe['Strike'].astype(str) + main_df_3_min_pe['Type']
    main_df_3_min_ce['Contract'] = main_df_3_min_ce['Strike'].astype(str) + main_df_3_min_ce['Type']
    
    main_df_3_min_pe['Date'] = main_df_3_min_pe['Date/Time'].dt.date
    main_df_3_min_pe['Time'] = main_df_3_min_pe['Date/Time'].dt.time
    
    main_df_3_min_ce['Date'] = main_df_3_min_ce['Date/Time'].dt.date
    main_df_3_min_ce['Time'] = main_df_3_min_ce['Date/Time'].dt.time
    
    return main_df_1_min,main_df_3_min_pe,main_df_3_min_ce
def generate_threshold_files(main_df,px_df):
    full_df = main_df
    df2 = px_df
    arr = np.array([])
    check_times = [3,6,10,15,20,24,30,40,50,60,75,96,120]
    lim_time = time(15,28)
    df_cur = df2.loc[df2['Date'] == df2['Date'][0]]
    terminal_time = lim_time
    df2 = df2.assign(End_Time=pd.to_datetime(df2['Date/Time']).dt.time)
    df2 = df2.assign(Return=pd.to_numeric([0]*len(df2)))
    for threshold_min in range(1,50):
        #prev_order_seen = None
        hold_order = 0
        cur_max_ret = 0
        ct = 0
        time_ind = -1
        df = full_df.loc[full_df['Date'] == full_df['Date'][0]]
        df.reset_index(drop=True, inplace=True)

        df_cur = df2.loc[df2['Date'] == full_df['Date'][0]]
        df_cur.reset_index(drop=True, inplace=True)

        print("Threshold_level:",threshold_min)

        for j in range(len(df_cur)):
            #id_val = (df.index[df['Time'] == df_cur['Time'][j]])[0]
            id_val_temp = (df.index[df['Time'] == df_cur['Time'][j]])
            id_val = 0
            try:
                id_val = id_val_temp[0]
            except:
                continue
            i = id_val
            opt_type_temp = df_cur['Contract'][j][-2:]

            print("Time:",df_cur['Time'][j],"date:",df_cur['Date'][j],"threshold:",threshold_min)
            cur_max_ret = 0

            if(opt_type_temp=='CE'):

                temp_val = df['Close'][i] - df['Open'][i]
                if(temp_val>cur_max_ret):
                    cur_max_ret = temp_val
                    prev_order = j
                    time_ind = i
                    hold_order = 1
                    opt_type = opt_type_temp
                else:

                    if(cur_max_ret-temp_val>threshold_min):

                        df_cur['End_Time'][j] = df_cur['Time'][j]
                        df_cur['Return'][j] = temp_val
                        prev_order = -1
                        hold_order = 0
                        continue

                    else:
                        prev_order = j
                        time_ind = i
                        hold_order = 1
                        opt_type = opt_type_temp



            else:
                temp_val = df['Open'][i] - df['Close'][i]
                if(temp_val>cur_max_ret):
                    cur_max_ret = temp_val
                    prev_order = j
                    time_ind = i
                    hold_order = 1
                    opt_type = opt_type_temp
                else:

                    if(cur_max_ret-temp_val>threshold_min):

                        df_cur['End_Time'][j] = df_cur['Time'][j]
                        df_cur['Return'][j] = temp_val
                        prev_order = -1
                        hold_order = 0
                        continue

                    else:
                        prev_order = j
                        time_ind = i
                        hold_order = 1
                        opt_type = opt_type_temp


            for i in range(id_val+1,len(df)):

                    #print(df_cur['Time'][j],df['Time'][i])
                    if(df['Time'][i]==terminal_time or i==len(df)-1):

                            check_val = df_cur.index[df_cur['Time'] == df['Time'][i]]
                            # if(len(check_val) > 0):
                            #     prev_order_seen = df_cur.iloc[check_val].reset_index()
                            #print('Executed',prev_order)
                            ct +=1
                            df_cur['End_Time'][prev_order] = df['Time'][i]
                            if(opt_type=='CE'):
                                df_cur['Return'][prev_order] = df['Close'][i]-df['Open'][time_ind]
                            else:
                                df_cur['Return'][prev_order] = df['Open'][time_ind]-df['Close'][i]
                            prev_order = -1
                            hold_order = 0
                            break
                    if(opt_type=='CE'):
                        temp_val = df['Close'][i] - df['Open'][time_ind]
                        if(temp_val>cur_max_ret):
                            cur_max_ret = temp_val
                        else:
                            if(cur_max_ret-temp_val>threshold_min):

                                #print('Executed',prev_order)
                                ct +=1
                                #print(df_cur[prev_order:prev_order+1])
                                df_cur['End_Time'][prev_order] = df['Time'][i]
                                df_cur['Return'][prev_order] = temp_val
                                prev_order = -1
                                hold_order = 0
                                break

                    else:
                        temp_val = df['Open'][time_ind] - df['Close'][i]
                        if(temp_val>cur_max_ret):
                            cur_max_ret = temp_val
                        else:
                            if(cur_max_ret-temp_val>threshold_min):

                                #print('Executed',prev_order)
                                ct +=1
                                df_cur['End_Time'][prev_order] = df['Time'][i]
                                df_cur['Return'][prev_order] = temp_val
                                prev_order = -1
                                hold_order = 0
                                break    
        res = df_cur
        for d in full_df['Date'].unique()[1:]:

            hold_order = 0
            cur_max_ret = 0
            ct = 0
            time_ind = -1
            #prev_order_seen = None

            df = full_df.loc[full_df['Date'] == d]
            df.reset_index(drop=True, inplace=True)

            df_cur = df2.loc[df2['Date'] == d]
            df_cur.reset_index(drop=True, inplace=True)

            for j in range(len(df_cur)):

                #id_val = (df.index[df['Time'] == df_cur['Time'][j]])[0]
                id_val_temp = (df.index[df['Time'] == df_cur['Time'][j]])
                id_val = 0
                try:
                    id_val = id_val_temp[0]
                except:
                    continue
                i = id_val
                opt_type_temp = df_cur['Contract'][j][-2:]

                print("Time:",df_cur['Time'][j],"date:",d,"threshold:",threshold_min)
                cur_max_ret = 0

                if(opt_type_temp=='CE'):

                    temp_val = df['Close'][i] - df['Open'][i]
                    if(temp_val>cur_max_ret):
                        cur_max_ret = temp_val
                        prev_order = j
                        time_ind = i
                        hold_order = 1
                        opt_type = opt_type_temp
                    else:

                        if(cur_max_ret-temp_val>threshold_min):

                            df_cur['End_Time'][j] = df['Time'][i]
                            df_cur['Return'][j] = temp_val
                            prev_order = -1
                            hold_order = 0
                            continue

                        else:
                            prev_order = j
                            time_ind = i
                            hold_order = 1
                            opt_type = opt_type_temp



                else:
                    temp_val = df['Open'][i] - df['Close'][i]
                    if(temp_val>cur_max_ret):
                        cur_max_ret = temp_val
                        prev_order = j
                        time_ind = i
                        hold_order = 1
                        opt_type = opt_type_temp
                    else:

                        if(cur_max_ret-temp_val>threshold_min):

                            df_cur['End_Time'][j] = df['Time'][i]
                            df_cur['Return'][j] = temp_val
                            prev_order = -1
                            hold_order = 0
                            continue

                        else:
                            prev_order = j
                            time_ind = i
                            hold_order = 1
                            opt_type = opt_type_temp


                for i in range(id_val+1,len(df)):

                        if(df['Time'][i]==terminal_time or i==len(df)-1):

                                check_val = df_cur.index[df_cur['Time'] == df['Time'][i]]
                                # if(len(check_val) > 0):
                                #     prev_order_seen = df_cur.iloc[check_val].reset_index()
                                #print('Executed',prev_order)
                                ct +=1
                                df_cur['End_Time'][prev_order] = df['Time'][i]
                                if(opt_type=='CE'):
                                    df_cur['Return'][prev_order] = df['Close'][i]-df['Open'][time_ind]
                                else:
                                    df_cur['Return'][prev_order] = df['Open'][time_ind]-df['Close'][i]
                                prev_order = -1
                                hold_order = 0
                                break
                        
                        if(opt_type=='CE'):
                            temp_val = df['Close'][i] - df['Open'][time_ind]
                            if(temp_val>cur_max_ret):
                                cur_max_ret = temp_val
                            else:
                                if(cur_max_ret-temp_val>threshold_min):

                                    #print('Executed',prev_order)
                                    ct +=1
                                    #print(df_cur[prev_order:prev_order+1])
                                    df_cur['End_Time'][prev_order] = df['Time'][i]
                                    df_cur['Return'][prev_order] = temp_val
                                    prev_order = -1
                                    hold_order = 0
                                    break

                        else:
                            temp_val = df['Open'][time_ind] - df['Close'][i]
                            if(temp_val>cur_max_ret):
                                cur_max_ret = temp_val
                            else:
                                if(cur_max_ret-temp_val>threshold_min):

                                    #print('Executed',prev_order)
                                    ct +=1
                                    df_cur['End_Time'][prev_order] = df['Time'][i]
                                    df_cur['Return'][prev_order] = temp_val
                                    prev_order = -1
                                    hold_order = 0
                                    break    

            res = res.append(df_cur,ignore_index=True)
        print("#"*20)
        print("#"*20)
        print("#"*20)
        print("Results",np.sum(res['Return'].values))
        arr = np.append(arr,np.sum(res['Return'].values))
        print("#"*20)
        print("#"*20)
        print("#"*20)
        res.to_csv('Total_Result_Threshhold_{}_limit.csv'.format(threshold_min),index=False)
    res.to_csv('Total_Result_Threshhold_all.csv',index=False)

def get_max_threshold_frame(full_df,arr) : 
    tp = []
    for i in range(1,50):
        tp.append(i)
    summary_df = pd.DataFrame(columns=tp,index=full_df['Date'].unique())
    for i in range(1,50):
        res = pd.read_csv('Total_Result_Threshhold_{}_limit.csv'.format(i))
        res['Date'] = (pd.to_datetime(res['Date']).dt.date)
        for d in full_df['Date'].unique(): 
            df_cur_temp = res.loc[res['Date'] == d]
            summary_df[i][d] = np.sum(df_cur_temp['Return'].values)
    max_threshold = np.where(arr == np.amax(arr))[0][0] + 1
    max_thr_df = pd.read_csv("Total_Result_Threshhold_{}_limit.csv".format(max_threshold))
    max_thr_df = max_thr_df[max_thr_df['Return'] > 19]
    max_thr_df.reset_index(inplace=True,drop=True)
    max_thr_df['Date/Time'] = pd.to_datetime(max_thr_df['Date/Time'], dayfirst=True )
    max_thr_df['Date'] = max_thr_df['Date/Time'].dt.date
    max_thr_df['Time'] = max_thr_df['Date/Time'].dt.time
    return max_thr_df

def drop_last_piece_of_tail(adf) : 
    dates = set(adf['Date'])
    conc  =  pd.DataFrame()
    for i in dates:
        temp = adf[ adf['Date'] == i ]
        temp.drop(temp.tail(2).index,inplace=True)
        conc = conc.append(temp)
    return conc

def generate_net_threshold(input_file,START_DATE,END_DATE,TICKER,output_file):
  ticker_dict = {"BANKNIFTY" : "Swing will be +/- 300","NIFTY" : "Swing will be +/- 100"} 
  for f_index in range(len([1])) :
      input_file  = input_file
      output_file = output_file
      print("[INFO] Processing input file: {}".format(input_file))
      ticker = TICKER
      if (ticker == "" ) : 
        ticker = None
      if(ticker != None) : 
        print("[INFO] Ticker is set to : {} and {}".format(ticker,ticker_dict[ticker]))


      main_df,pe_df,ce_df = generate_1_min_and_pe_and_ce(input_file,ticker,START_DATE,END_DATE)
      if (main_df.shape[0] == 0 ) : 
        return 
      pe_df         = drop_last_piece_of_tail(pe_df)
      ce_df         = drop_last_piece_of_tail(ce_df)

      pe_thr_arr    = generate_threshold_files(main_df,pe_df)
      max_thr_pe_df = get_max_threshold_frame(main_df,pe_thr_arr)

      ce_thr_arr    = generate_threshold_files(main_df,ce_df)
      max_thr_ce_df = get_max_threshold_frame(main_df,ce_thr_arr)

      pe_plus_ce    = max_thr_pe_df.append(max_thr_ce_df)

      pe_plus_ce["Date/Time"] = pd.to_datetime(pe_plus_ce['Date/Time'], dayfirst=True )
      pe_plus_ce["Time"]      = pd.to_datetime(pe_plus_ce['Date/Time'], dayfirst=True ).dt.time
      pe_plus_ce["Date"]      = pd.to_datetime(pe_plus_ce['Date/Time'], dayfirst=True ).dt.date
      pe_plus_ce.sort_values(by=['Date/Time'], inplace=True)
      pe_plus_ce.reset_index(inplace=True,drop=True)

      arr = generate_threshold_files(main_df,pe_plus_ce)
      net_max_thr = get_max_threshold_frame(main_df,arr)
      print("[INFO] Writing the output to: {} ".format(output_file))
      net_max_thr.to_csv(output_file,index=False)

# + [markdown] id="2fs_gPfzhKvJ"
# Single Run example

# + id="P3MFq4kYOwvV" colab={"base_uri": "https://localhost:8080/"} outputId="64d48481-5a34-420a-c5bc-4805af092ba2"
outputfile         = '/content/drive/MyDrive/trading_app/net_thr_bug.csv'
input_file         = '/content/drive/MyDrive/trading_app/bug.csv'
START_DATE         = '17-02-22 15:21'
END_DATE           = '18-02-22 09:30'
TICKER             = "BANKNIFTY"
generate_net_threshold(input_file,START_DATE,END_DATE,TICKER,outputfile)

# + [markdown] id="t6_U9CLEhPxc"
# Multiple Runs from a csv file 

# + id="zxRC9dExhWC3" colab={"base_uri": "https://localhost:8080/"} outputId="c7a6b110-834e-43fb-a693-8e275c20a799"
multi_files_input = '/content/drive/MyDrive/trading_app/input_file.csv'
generate_net_threshold_for_multi_runs(multi_files_input)

StarcoderdataJupyterScriptsDedupFiltered