# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="2fxU3pux7YgU"
# #Move first chunk
#

# + id="L4aLA17u7z09"
# %%capture
# !apt-get install lzop

# + id="aGa26N3X7cFm"
# !cp "drive/MyDrive/twitter/onechunk/part-00000.lzo" file.lzo

# + id="pKOlV_MPcffo"
# !lzop -x file.lzo
# !mv file Dataset.csv

# + id="UZ6364iF8Fz1"
import gc
import numpy as np
np.random.seed(1234)
gc.collect()
all_features = ["text_tokens", "hashtags", "tweet_id", "present_media", "present_links", "present_domains",\
                "tweet_type","language", "tweet_timestamp", "engaged_with_user_id", "engaged_with_user_follower_count",\
               "engaged_with_user_following_count", "engaged_with_user_is_verified", "engaged_with_user_account_creation",\
               "engaging_user_id", "engaging_user_follower_count", "engaging_user_following_count", "engaging_user_is_verified",\
               "engaging_user_account_creation", "engagee_follows_engager","reply_timestamp", "retweet_timestamp", "retweet_with_comment_timestamp", "like_timestamp"]
all_features_to_idx = dict(zip(all_features, range(len(all_features))))

# + [markdown] id="PTFIb0pY8aZl"
# #Load with pandas and train/test splitting

# + id="HX38qqxj8hcA"
import pandas as pd
df= pd.read_csv("Dataset.csv",sep='\x01',header=None)
df.columns=all_features

# + colab={"base_uri": "https://localhost:8080/"} id="_tBfYb0EDm9U" executionInfo={"status": "ok", "timestamp": 1619388226568, "user_tz": -120, "elapsed": 44480, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="09fc0f4c-972d-458a-84f5-ad5a4dfc717e"
gc.collect()

# + [markdown] id="frKyJDgy9R00"
# ##Target classes encode

# + id="CeqyvUY_9RFI"
df["like"]=df["like_timestamp"].apply(lambda row: 1 if row>0 else 0).astype("uint8")
df.drop("like_timestamp",axis=1,inplace=True)

df["retweet"]=df["retweet_timestamp"].apply(lambda row: 1 if row>0 else 0).astype("uint8")
df.drop("retweet_timestamp",axis=1,inplace=True)

df["retweetCom"]=df["retweet_with_comment_timestamp"].apply(lambda row: 1 if row>0 else 0).astype("uint8")
df.drop("retweet_with_comment_timestamp",axis=1,inplace=True)

df["reply"]=df["reply_timestamp"].apply(lambda row: 1 if row>0 else 0).astype("uint8")
df.drop("reply_timestamp",axis=1,inplace=True)
gc.collect()
df["interaction"]=df["reply"]+df["like"]+df["retweet"]+df["retweetCom"]
df["interaction"]=df["interaction"].astype("uint8")


# + [markdown] id="_rPEvh2n9t2M"
# ##Divide media types

# + colab={"base_uri": "https://localhost:8080/"} id="XMDPDOXr9l3t" executionInfo={"status": "ok", "timestamp": 1619388240619, "user_tz": -120, "elapsed": 54869, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="eda063f0-4dfc-4066-c4b9-3dd1d5008e20"
def countMedia(arr,mediaType):
    c=0
    for m in arr:
        if m==mediaType:
            c+=1
    return c
df.fillna(value={"present_media":""},inplace=True)
df["photos"]=df["present_media"].apply(lambda row: countMedia((row.split("\t")),"Photo")).astype("uint8")
df["gifs"]=df["present_media"].apply(lambda row: countMedia((row.split("\t")),"GIF")).astype("uint8")
df["videos"]=df["present_media"].apply(lambda row: countMedia((row.split("\t")),"Video")).astype("uint8")
gc.collect()


# + [markdown] id="nGDYjvK99bD2"
# ##Replace text tokens , link, media etc with their count

# + colab={"base_uri": "https://localhost:8080/"} id="ujJogGYn9aOe" executionInfo={"status": "ok", "timestamp": 1619388255694, "user_tz": -120, "elapsed": 68667, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="80400b1d-c859-4e0c-9f80-6195962bb242"
df["text_tokens"]=df["text_tokens"].apply(lambda row: len(row.split("\t")) if row!="" else 0).astype("uint16")
df.fillna(value={"hashtags":""},inplace=True)
df["hashtags"]=df["hashtags"].apply(lambda row: len(row.split("\t")) if row!="" else 0).astype("uint8")

df["present_media"]=df["present_media"].apply(lambda row: len(row.split("\t")) if row!="" else 0).astype("uint8")
df.fillna(value={"present_links":""},inplace=True)
df["present_links"]=df["present_links"].apply(lambda row: len(row.split("\t")) if row!="" else 0).astype("uint8")
df.fillna(value={"present_domains":""},inplace=True)
df["present_domains"]=df["present_domains"].apply(lambda row: len(row.split("\t")) if row!="" else 0).astype("uint8")
gc.collect()

# + [markdown] id="Ys0HPV-NAbOQ"
# ##Drop columns i don't use

# + colab={"base_uri": "https://localhost:8080/"} id="mYezX7OVAeul" executionInfo={"status": "ok", "timestamp": 1619214745981, "user_tz": -120, "elapsed": 52288, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="087a5a14-bf2a-4fc0-ecea-a3c1494a6ee3"
df.drop("tweet_id",axis=1,inplace=True)
df.drop("engaging_user_account_creation",axis=1,inplace=True)
df.drop("engaged_with_user_account_creation",axis=1,inplace=True)
df.drop("tweet_timestamp",axis=1,inplace=True)
gc.collect()

# + [markdown] id="LMBY1piGAfbg"
# ##Check types

# + colab={"base_uri": "https://localhost:8080/"} id="AqzHeBvz-h--" executionInfo={"status": "ok", "timestamp": 1619214745990, "user_tz": -120, "elapsed": 52287, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="359be715-0d06-4c18-a6e8-2fcfbee23a48"
df.dtypes

# + [markdown] id="nF_51saxEHar"
# ##Display head

# + colab={"base_uri": "https://localhost:8080/"} id="cJi0mqyyEGzC" executionInfo={"status": "ok", "timestamp": 1619214745992, "user_tz": -120, "elapsed": 52281, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="c492d347-934c-4349-f3fe-3b84d9402cdc"
df.head()

# + colab={"base_uri": "https://localhost:8080/"} id="rPqCQAT_Fnlb" executionInfo={"status": "ok", "timestamp": 1619214747044, "user_tz": -120, "elapsed": 53326, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="ae3630f2-cb24-40a8-9a2e-25ed8749cc2b"
df.describe()

# + [markdown] id="YhRrKY5G-XA2"
# ##Split dataset

# + id="MyRnL5cB-mhe"
 msk = np.random.rand(len(df)) < 0.8
 
 train=df[msk]

 test=df[~msk]

# + id="c-wX_eTSATeB"
train.to_csv("train.csv", header=train.columns)
test.to_csv("test.csv", header=test.columns)

# + id="bal4uvZcBRYl"
#import pickle
#with open('ColNames.pkl', 'wb') as f:
#    pickle.dump(train.columns, f)


# + id="pBaH4__ACVA_"
# !bzip2 -c --best train.csv > train.bz2
# !bzip2 -c --best test.csv > test.bz2

# + id="89r2DVg7AzeN"
# !cp train.bz2 "drive/MyDrive/twitter/one chunk/train.bz2"
# !cp test.bz2 "drive/MyDrive/twitter/one chunk/test.bz2"
# #!cp ColNames.pkl "drive/MyDrive/twitter/one chunk/ColNames.pkl"

# + [markdown] id="R4mspB6eIy_l"
# #Prepare for Cugraph

# + id="goC91SD4I2CE"
trainCu=pd.DataFrame(train.loc[df["interaction"]>0])


# + id="BaATnM7v4kPs"
for col in trainCu.columns:
    if col not in ["engaged_with_user_id","engaging_user_id","interaction"]:
        trainCu.drop(col,axis=1,inplace=True)

# + id="804WvrY2t8MT"
trainCu["interactions"]=trainCu.groupby(["engaged_with_user_id","engaging_user_id"])['interaction'].transform('sum')

# + id="IKCnzyBku7H2"
trainCu= trainCu.drop_duplicates(subset= ["engaged_with_user_id","engaging_user_id"]).drop("interaction",axis=1)

# + colab={"base_uri": "https://localhost:8080/", "height": 202} id="ApY1DjhWJbAd" executionInfo={"status": "ok", "timestamp": 1619215271263, "user_tz": -120, "elapsed": 2162, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="628698fd-a924-4f12-f80d-4bf71351d6eb"
trainCu[:5]

# + colab={"base_uri": "https://localhost:8080/"} id="ordME01SudK5" executionInfo={"status": "ok", "timestamp": 1619215276396, "user_tz": -120, "elapsed": 758, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="6e514177-215d-4d4b-992c-bc821cdc3a8c"
trainCu.columns

# + id="F8dBnfCOKmWV"
trainCu.to_csv("trainCu.csv", header=trainCu.columns)

# + id="OdydlBN5LnD7"
# !bzip2 -c --best trainCu.csv > trainCu.bz2

# + id="MrM7ATYFLq0Z"
# !cp trainCu.bz2 "drive/MyDrive/twitter/onechunk/trainCu.bz2"

# + [markdown] id="Up7hgRSQUMxJ"
# #After CuGraph

# + id="eIsukFBdUQDc"
# !cp  drive/MyDrive/twitter/onechunk/* ../content/

# + id="DT3ARdW9VUMw" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619439677507, "user_tz": -120, "elapsed": 30946, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="ada1e12b-d749-4426-8cdf-0f68e0cdb022"
import os
files=os.listdir()
files=[f for f in files if f.endswith(".csv")]
files

# + id="FP1Q3MH9VY2T" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619439677508, "user_tz": -120, "elapsed": 29942, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="64d6b209-1e4c-4ef4-fc55-12db0ae97f6c"
for fileT in files:
    with open(fileT) as f:
        print(f"{fileT}:{f.readline()}",end="")

# + id="KiKuuwOeVgur"
# !bunzip2 train.bz2
# !mv train train.csv

# + id="dCPuaM0PWfoT" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440195830, "user_tz": -120, "elapsed": 14314, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="8b866470-a07f-466a-bb8e-c8d05d9be40b"
import pandas as pd
train=pd.read_csv("train.csv", index_col=0)

# + id="0OPBljlXXGOI" colab={"base_uri": "https://localhost:8080/", "height": 222} executionInfo={"status": "ok", "timestamp": 1619440195835, "user_tz": -120, "elapsed": 13518, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="aa49fcb6-1d9a-46a2-c2c7-0f98226b2a04"
df=train
train[:5]

# + [markdown] id="vSwQm7xJbPDU"
# #ADD pagerank

# + id="o7rSYcEfWpkr" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440198623, "user_tz": -120, "elapsed": 9142, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="296d5c74-8f7a-4b5b-99b1-93cc59419e35"
pagerank_ed_to_ing=pd.read_csv("pagerank.csv",index_col=0)


# + id="9Z1obNo5YwOL"
pagerank_ed_to_ing.columns=["pagerank_ed_to_ing","engaged_with_user_id"]

# + id="N4qJG8IvW8zo" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440198629, "user_tz": -120, "elapsed": 9123, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="ffc01c72-1b61-45f7-f4de-fb523cb9e6b5"
pagerank_ed_to_ing[:5]

# + id="xuawkAG_XQpg"
df=df.merge(pagerank_ed_to_ing,on="engaged_with_user_id",how="left")

# + id="bjfzjGOgZGCC"
df["pagerank_ed_to_ing_ed"]=df["pagerank_ed_to_ing"].apply(lambda x: x if x>0 else 0)
df.drop("pagerank_ed_to_ing",axis=1,inplace=True)

# + id="8JcsaF2_Zxrg"
pagerank_ed_to_ing.columns=["pagerank_ed_to_ing","engaging_user_id"]

# + id="T-Tslj4bZ00L"
df=df.merge(pagerank_ed_to_ing,on="engaging_user_id",how="left")

# + id="4veB-aOjZ3PZ"
df["pagerank_ed_to_ing_ing"]=df["pagerank_ed_to_ing"].apply(lambda x: x if x>0 else 0)
df.drop("pagerank_ed_to_ing",axis=1,inplace=True)


# + id="ONQGozhHaG1C" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440213494, "user_tz": -120, "elapsed": 23932, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="e62e38d8-daa8-4c4a-a56e-7c25f50cde8f"
df[:5]

# + id="zY7W3kXGeCff"
del pagerank_ed_to_ing

# + [markdown] id="3h59Q4R0bsQW"
# #ADD HITS

# + id="qlPkeNqObsQn" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440216148, "user_tz": -120, "elapsed": 24015, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="bbbaf0f2-966d-4cf9-f240-e19ac3ef5157"
hits_ed_to_ing=pd.read_csv("hits.csv",index_col=0)


# + id="P1PyQhWUbsQo"
hits_ed_to_ing.columns=["hubs_ed","authorities_ed","engaged_with_user_id"]

# + id="Ik4XUf9xbsQp" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440216155, "user_tz": -120, "elapsed": 23996, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="921009e8-e48e-4077-8e0a-2fb14d7936a0"
hits_ed_to_ing[:5]

# + id="aLss5qzgbsQq"
df=df.merge(hits_ed_to_ing,on="engaged_with_user_id",how="left")

# + id="yXqy0UABbsQr"
df["hubs_ed"]=df["hubs_ed"].apply(lambda x: x if x>0 else 0)
df["authorities_ed"]=df["authorities_ed"].apply(lambda x: x if x>0 else 0)

# + id="qo47KpvbbsQs"
hits_ed_to_ing.columns=["hubs_ing","authorities_ing","engaging_user_id"]

# + id="6yV-f2sfbsQt"
df=df.merge(hits_ed_to_ing,on="engaging_user_id",how="left")

# + id="--21AIGQbsQu"
df["hubs_ing"]=df["hubs_ing"].apply(lambda x: x if x>0 else 0)
df["authorities_ing"]=df["authorities_ing"].apply(lambda x: x if x>0 else 0)

# + id="Ue6rK2HjbsQv" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440229864, "user_tz": -120, "elapsed": 37650, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="4cf78c9d-2e82-4b44-a840-b67b5c7c8d0d"
df[:5]

# + id="toak9lu1dL2v"
del hits_ed_to_ing

# + [markdown] id="3VifCZ3ciIT9"
# #ADD Katz

# + id="QcX1zSgEiIUI" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440232639, "user_tz": -120, "elapsed": 39329, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="62fdf821-13d8-4b1d-f792-4bab4faa38c8"
katz_ed_to_ing=pd.read_csv("katz_centrality.csv",index_col=0)


# + id="NFkOXjf_iIUL"
katz_ed_to_ing.columns=["katz_ed","engaged_with_user_id"]

# + id="3lfA0i7PiIUL" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440232647, "user_tz": -120, "elapsed": 39314, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="166144fc-57eb-4a8c-93ee-0d8720f0bcf0"
katz_ed_to_ing[:5]

# + id="3h51Zx2aiIUM"
df=df.merge(katz_ed_to_ing,on="engaged_with_user_id",how="left")

# + id="R_qbtSrhiIUN"
df["katz_ed"]=df["katz_ed"].apply(lambda x: x if x>0 else 0)

# + id="vc2poHnriIUO"
katz_ed_to_ing.columns=["katz_ing","engaging_user_id"]

# + id="cNLXRskDiIUO"
df=df.merge(katz_ed_to_ing,on="engaging_user_id",how="left")

# + id="WCANcqROiIUP"
df["katz_ing"]=df["katz_ing"].apply(lambda x: x if x>0 else 0)

# + id="JSr6MrEBiIUQ" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440243858, "user_tz": -120, "elapsed": 50463, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="c368aaf1-4f3b-4d98-f7ff-fee1bb00a5cd"
df[:5]

# + id="GZNn4bCXiIUR"
del katz_ed_to_ing

# + [markdown] id="s2asHVispvp9"
# #ADD Jaccard

# + [markdown] id="fn4PSn56pvqO"
# ##ADD Jaccard to the dataframe

# + id="d-_Fa2DZpvqP"
jaccard_ed_to_ing=pd.read_csv("jaccard-ed-to-ing.csv",index_col=0)


# + id="jYJJMgYSpvqR"
jaccard_ed_to_ing.columns=["jaccard_ed_to_ing_ed","engaged_with_user_id","engaging_user_id"]

# + id="_xDWSqM0pvqS" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440243863, "user_tz": -120, "elapsed": 49998, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="c9e9348c-8fc3-4541-938f-e26caa0df870"
jaccard_ed_to_ing[:5]

# + id="NWn00PSapvqT"
df=df.merge(jaccard_ed_to_ing,on=["engaged_with_user_id","engaging_user_id"],how="left")

# + id="T8twAEOWpvqT"
df["jaccard_ed_to_ing_ed"]=df["jaccard_ed_to_ing_ed"].apply(lambda x: x if x>0 else 0)


# + id="ApyPV2TkpvqU"
jaccard_ed_to_ing.columns=["jaccard_ed_to_ing_ing","engaging_user_id","engaged_with_user_id"]

# + id="kgs6fb05pvqU"
df=df.merge(jaccard_ed_to_ing,on=["engaging_user_id","engaged_with_user_id"],how="left")

# + id="2vmsrAiVpvqV"
df["jaccard_ed_to_ing_ing"]=df["jaccard_ed_to_ing_ing"].apply(lambda x: x if x>0 else 0)


# + id="Ws1bHkwepvqW" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440255847, "user_tz": -120, "elapsed": 61917, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="c858f2f6-26d3-4fb5-e422-9c8c04542c90"
df[:5]

# + id="xE9rGjRWpvqX"
del jaccard_ed_to_ing

# + [markdown] id="IM3rObgQsoZE"
# ##ADD Jaccard to the dataframe opposite

# + id="VXEiird-soZG"
jaccard_ing_to_ed=pd.read_csv("jaccard-ing-to-ed.csv",index_col=0)


# + id="yCSujcNOsoZI"
jaccard_ing_to_ed.columns=["jaccard_ing_to_ed_ed","engaged_with_user_id","engaging_user_id"]

# + id="3O4cZGM6soZJ" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440255860, "user_tz": -120, "elapsed": 61882, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="0256c9e9-aed6-490b-c986-07c2afb2a4bc"
jaccard_ing_to_ed[:5]

# + id="Ung3N7g4soZK"
df=df.merge(jaccard_ing_to_ed,on=["engaged_with_user_id","engaging_user_id"],how="left")

# + id="M0c4lVobsoZL"
df["jaccard_ing_to_ed_ed"]=df["jaccard_ing_to_ed_ed"].apply(lambda x: x if x>0 else 0)


# + id="fC6zL69SsoZL"
jaccard_ing_to_ed.columns=["jaccard_ing_to_ed_ing","engaging_user_id","engaged_with_user_id"]

# + id="Wzvy1mlTsoZM"
df=df.merge(jaccard_ing_to_ed,on=["engaging_user_id","engaged_with_user_id"],how="left")

# + id="eOFdGeFnsoZN"
df["jaccard_ing_to_ed_ing"]=df["jaccard_ing_to_ed_ing"].apply(lambda x: x if x>0 else 0)


# + id="mCEUVcaNsoZO" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440267048, "user_tz": -120, "elapsed": 73001, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="2f7adcee-2f40-458d-a615-1a5db20edf7f"
df[:5]

# + id="jrgvLI2FsoZP"
del jaccard_ing_to_ed

# + [markdown] id="yzSHpIA7lYm5"
# #ADD Overlap

# + id="gC-kUoFZlYm7"
overlap_ed_to_ing=pd.read_csv("over.csv",index_col=0)


# + id="fQ1Y0wEqlYm9"
overlap_ed_to_ing.columns=["overlap_ed","engaged_with_user_id","engaging_user_id"]

# + id="XHKeP4MxlYm-" colab={"base_uri": "https://localhost:8080/", "height": 202} executionInfo={"status": "ok", "timestamp": 1619440268248, "user_tz": -120, "elapsed": 73202, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="f766ad54-9a35-4c25-a517-a063d8b59154"
overlap_ed_to_ing[:5]

# + id="rIQlz5vIlYm_"
df=df.merge(overlap_ed_to_ing,on=["engaged_with_user_id","engaging_user_id"],how="left")

# + id="UOr3mbAFlYnA"
df["overlap_ed"]=df["overlap_ed"].apply(lambda x: x if x>0 else 0)


# + id="A6D6SEw5lYnB"
overlap_ed_to_ing.columns=["overlap_ing","engaging_user_id","engaged_with_user_id"]

# + id="FWYXsWYxlYnB"
df=df.merge(overlap_ed_to_ing,on=["engaging_user_id","engaged_with_user_id"],how="left")

# + id="ihFUXiW6lYnC"
df["overlap_ing"]=df["overlap_ing"].apply(lambda x: x if x>0 else 0)


# + id="tmgyL-oSoYPY"
def f(row):
    e=row["overlap_ed"]
    i=row["overlap_ing"]
    if e >0:
        if i>0:
            return (e+i)/2
        else:
            return e
    else:
        if i>0:
            return i
        else:
            return 0
df["overlap"]=df.apply(f, axis=1)

df.drop("overlap_ed",axis=1,inplace=True)
df.drop("overlap_ing",axis=1,inplace=True)

# + id="CfnQf2RulYnD" colab={"base_uri": "https://localhost:8080/", "height": 222} executionInfo={"status": "ok", "timestamp": 1619440318146, "user_tz": -120, "elapsed": 123036, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="15b174e5-f6ff-4d84-c816-686832fa1e3d"
df[:5]

# + id="bkKFdbHZlYnE"
del overlap_ed_to_ing

# + [markdown] id="iA713BDNBAqX"
# #Use datafrme

# + [markdown] id="AcocgzL2EXLK"
# ##Fix dataset

# + id="6On2a8LSFjs8"
istrain=True

# + [markdown] id="urf1ED66WI34"
# ##Check if talk that language

# + id="8fS3y0xWDswp" colab={"base_uri": "https://localhost:8080/", "height": 222} executionInfo={"status": "ok", "timestamp": 1619440318151, "user_tz": -120, "elapsed": 120378, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="41600230-793a-4506-80b4-41e859037664"
df[:5]

# + id="SfT8w82kX8Cx" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440318447, "user_tz": -120, "elapsed": 120663, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="ae32e611-c276-48f4-89cf-ddd22483761b"
import gc
gc.collect()

# + id="s0APditcADgc" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440318448, "user_tz": -120, "elapsed": 120654, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="d5dea91a-4901-4487-94d7-42038c568dfe"

t=df[["engaging_user_id","language","interaction"]].copy()
lang=t
langDir=lang[lang["interaction"]>0]
langDir.drop("interaction",inplace=True,axis=1)
langDir.columns=["id","language"]

# + id="UU4mVdwSAE0E" colab={"base_uri": "https://localhost:8080/", "height": 202} executionInfo={"status": "ok", "timestamp": 1619440318922, "user_tz": -120, "elapsed": 121115, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="eb21fb57-e8f4-493e-c659-49fdaaafe936"
langDir[:5]

# + id="9x-lx5u1AL4-" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440318926, "user_tz": -120, "elapsed": 121108, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="ac5c6041-dd70-4f9b-9b86-fb5370b7fa0c"
t=df[["engaged_with_user_id","language","interaction"]].copy()
lang=t
langInv=lang[lang["interaction"]>0]
langInv.drop("interaction",inplace=True,axis=1)
langInv.columns=["id","language"]

# + id="EgMGBi99ANE9" colab={"base_uri": "https://localhost:8080/", "height": 202} executionInfo={"status": "ok", "timestamp": 1619440318928, "user_tz": -120, "elapsed": 121097, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="3ce39165-eaa3-4e3b-95f9-731b9807ba5a"
langInv[:5]

# + id="LRs8NoFqAQcd"
lang=pd.concat([langDir,langInv])

# + id="DbsRMij5ARdj" colab={"base_uri": "https://localhost:8080/", "height": 202} executionInfo={"status": "ok", "timestamp": 1619440319239, "user_tz": -120, "elapsed": 121384, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="99c8b466-67a5-456d-d200-9f939f09c313"
lang[:5]

# + id="kSFr7aXtAUsm"
del langDir
del langInv

# + id="my6VMiBmAWdC"
lang.insert(2,"understands",1)
lang.columns=["engaging_user_id","language","understands"]
lang=lang.groupby(["engaging_user_id","language"]).sum()

# + id="zCDKTlEOAWXn" colab={"base_uri": "https://localhost:8080/", "height": 233} executionInfo={"status": "ok", "timestamp": 1619440327992, "user_tz": -120, "elapsed": 130104, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="cd4de895-6828-42cf-d612-9601bcf1d845"
lang[10:15]

# + id="EGlNLuKaA_0k"
lang["understands"]=lang["understands"].apply(lambda x: (x>1) if istrain else x>0)

# + id="-Thk9bsJBBQs" colab={"base_uri": "https://localhost:8080/", "height": 233} executionInfo={"status": "ok", "timestamp": 1619440328335, "user_tz": -120, "elapsed": 130420, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="116cf577-89ea-4c80-bdbe-519c684e288f"
lang[10:15]

# + id="Sv24w6p_BFqL"
temp=lang[lang["understands"]].copy()

# + id="XycISM_MBGyE" colab={"base_uri": "https://localhost:8080/", "height": 233} executionInfo={"status": "ok", "timestamp": 1619440328341, "user_tz": -120, "elapsed": 130396, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="25c4173b-9382-48f9-8564-ce378895332b"
temp[:5]

# + id="K3NRFqzgBO-P" colab={"base_uri": "https://localhost:8080/", "height": 222} executionInfo={"status": "ok", "timestamp": 1619440328343, "user_tz": -120, "elapsed": 130381, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="20f6f9e9-cee1-4d85-ee81-89b0e2314d69"
df[:5]

# + id="lQrYWq3vWIa8"

df=df.merge(temp,on=["engaging_user_id","language"],how="left")


# + id="9nwt5tMb-V7a" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440333473, "user_tz": -120, "elapsed": 135486, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="393f2e3c-c5b2-402c-f28b-3e963dc99c15"
df[:5]

# + id="dn4SYch5ExRX"
df["understands"]=df["understands"].fillna(False)

# + id="lA2YrraI_Lxt" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440333836, "user_tz": -120, "elapsed": 135824, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="591eddfe-95db-47ce-956c-6bca6278e657"
df[:5]

# + id="9mdaBCvdJtVO"
del lang
del temp

# + [markdown] id="qO0xpY_LFOCD"
# ##LastPreprocess

# + id="TM7QG71FFR2Z"
df.drop("language",axis=1,inplace=True)

# + id="vOB7YyI6CQjT"
df.drop("engaged_with_user_id",axis=1,inplace=True)
df.drop("engaging_user_id",axis=1,inplace=True)
df.drop("interaction",axis=1,inplace=True)

# + id="J06QOyjcA8o4" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440337962, "user_tz": -120, "elapsed": 137190, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="7d82c250-6710-421c-f674-fee1acbc4a35"
df.describe()

# + [markdown] id="z1bcYo6KBwBz"
# ##For Like
#

# + id="jyQsiNzqBf2j"
df.drop("retweet",axis=1,inplace=True)
df.drop("retweetCom",axis=1,inplace=True)
df.drop("reply",axis=1,inplace=True)
target="like"

# + [markdown] id="gcAv1VPtB_aD"
# ##For ret
#

# + id="ioze_V__CAxc"
df.drop("like",axis=1,inplace=True)
df.drop("retweetCom",axis=1,inplace=True)
df.drop("reply",axis=1,inplace=True)
target="retweet"

# + [markdown] id="kQpCe9DDCDfz"
# ##For quote

# + id="eHRp5XiJCFDh"
df.drop("retweet",axis=1,inplace=True)
df.drop("like",axis=1,inplace=True)
df.drop("reply",axis=1,inplace=True)
target="retweetCom"

# + [markdown] id="ZuArLTd4CITE"
# ##For Reply

# + id="AhrbSSnACJvU"
df.drop("retweet",axis=1,inplace=True)
df.drop("retweetCom",axis=1,inplace=True)
df.drop("like",axis=1,inplace=True)
target="reply"

# + [markdown] id="Mfo3ifUmC7jB"
# #Model

# + id="7FOz7eOTDGgk"
import numpy as np
from xgboost import XGBClassifier
from sklearn.datasets import load_iris
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score, KFold

# + id="1UX_q0VoDM4K"
targetCol= train[target]
df.drop(target,axis=1,inplace=True)

# + id="2mAnddg8DYU4"
import xgboost as xgb

# + id="CNZooFRHDe8h" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440341534, "user_tz": -120, "elapsed": 136721, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="319eb736-feb5-4da2-8c08-7b78bcadd317"
from sklearn.metrics import mean_squared_error
train=df.values
target=targetCol.values
del df
del targetCol
import gc
gc.collect()

# + id="UG3VaDIyDgsN"
xgbc = xgb.XGBClassifier()

# + id="7kEtlJBIDimD" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440902125, "user_tz": -120, "elapsed": 697293, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="43b87f66-f8cd-4252-eb53-3d940689d569"
xgbc.fit(train, target)

# + id="ocgE8QrbH_ql"
trainPred=xgbc.predict(train)


# + id="EQThLRnNIvxv" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619440914152, "user_tz": -120, "elapsed": 709302, "user": {"displayName": "pietrohideki", "photoUrl": "", "userId": "17128222802287801574"}} outputId="eb0ec2aa-2f25-4507-e3cc-7196a2e2d941"
cm = confusion_matrix(target,trainPred)
cm

# + id="0o-o8JUVF6xc"
scores = cross_val_score(xgbc, train, target, cv=5)
print("Mean cross-validation score: %.2f" % scores.mean())

# + id="92nFcaJMLihe"
print(xgbc.feature_importances_)

# + id="diODMZ8yLgxK"
xgb.plot_importance(xgbc)

# + id="cNE-8dnpLlO3"
xgbc.save_model('model_Like.json')

# + id="t_1tEwnPLluk"
# !cp model_Like.json drive/MyDrive/model_like.json

# + id="KfaK3Ze8LwF5"
train.columns

# + [markdown] id="jipYU1f9bpHR"
# #Close On complete

# + id="O4UsU1FibrCy"
# !kill -9 -1

StarcoderdataJupyterScriptsDedupFiltered