"""
# This notebook is to underestand easily for especially train data.
## (ver6 adding submission test and explanation)
## if it is helpful for you, please upvote!
"""
import numpy as np 
import pandas as pd 
import os
import cv2
import matplotlib.pyplot as plt
"""
# 0. Confirming each csv file
"""
traindf = pd.read_csv("../input/shopee-product-matching/train.csv")
traindf
testdf = pd.read_csv("../input/shopee-product-matching/test.csv")
testdf
sample = pd.read_csv("../input/shopee-product-matching/sample_submission.csv")
sample
TRAIN_PATH = "../input/shopee-product-matching/train_images"
TEST_PATH = "../input/shopee-product-matching/test_images"

traindf["path"] = [os.path.join(TRAIN_PATH,s) for s in traindf["image"]]
testdf["path"] = [os.path.join(TEST_PATH,s) for s in testdf["image"]]

traindf.to_csv("traindf.csv",index=False)
testdf.to_csv("testdf.csv",index=False)
"""
# 1. Train.csv
"""
traindf.head(3)
traindf.info()
"""
* no nan data
"""
for col in traindf.columns:
    print(col + ":" + str(len(traindf[col].unique())))
"""
* This result shows the how many uniques in each columns.
* Total rows are 34250. Each column has some duplicates.
* There are duplicates in images, hashes, and titles, not just label groups.
"""
"""
## 1.1 easy visualizing image
"""
img = cv2.imread(traindf["path"].iloc[0])
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
plt.imshow(img)
tmpdf = traindf[traindf["label_group"]==traindf["label_group"].iloc[0]]
tmpdf
for a in tmpdf["path"]:
    img = cv2.imread(a)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    plt.figure()
    plt.imshow(img)
    plt.axis("off")
"""
* It can be confirmed that both images are victoria's seacret.
"""
"""
## 1.2 Understanding same label_groups
"""
"""
### 1.2.1 Counting the number of images in each label group
"""
labels = traindf.groupby("label_group")["image"].count().reset_index()
labels.columns=["label_group","image_num"]
labels
sortlabels = labels.sort_values("image_num")
sortlabels
"""
* minimum images are 2, max images are 51
"""
"""
### 1.2.2 Counting the label_groups in each image_num
"""
imagecount = labels.groupby("image_num").count().reset_index()
imagecount.columns=["image_num","counts"]
imagecount
"""
* Image num counts in each labels are almost 2.
"""
plt.bar(imagecount["image_num"],imagecount["counts"])
"""
### 1.2.3 visualizing images of label_group with max counts 
"""
tmpdf = traindf[traindf["label_group"]==sortlabels["label_group"].iloc[-1]]
tmpdf.head(5)
    
plt.figure(figsize=(20,20))

for num,a in enumerate(tmpdf["path"]):
    plt.subplot(11,5,num+1)
    img = cv2.imread(a)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    plt.axis("off")
    plt.imshow(img)
    
    
"""
## 1.3 Understanding same images
"""
imagegroup = traindf.groupby("image")["path"].count().reset_index()
imagegroup.columns=["image","counts"]
imagegroup
tmpdf = imagegroup[imagegroup["counts"] > 1]
tmpdf.sort_values("counts")
tmpdf
"""
* min and max same images are 2.
"""
tmpdf["image"].iloc[0]
traindf[traindf["image"]==tmpdf["image"].iloc[0]]
"""
* posting id is different, but others are same. it should be duplicated ?
"""
traindf[traindf["image"]==tmpdf["image"].iloc[1]]
"""
* Other example shows not only posting id, but also title is different. it seemes that it was edited ?
"""
traindf.groupby(["image","image_phash"])["path"].count().reset_index()
"""
* 32412 is the same numbers as images. at least, image and image_phash are same in same images.
"""
"""
## 1.4 Understanding same image_phash
"""
phashgroup = traindf.groupby("image_phash")["path"].count().reset_index()
phashgroup.columns=["image_phash","counts"]
phashgroup
sortphash = phashgroup.sort_values("counts")
sortphash
"""
* maximum counts are 26.
"""
tmpdf = traindf[traindf["image_phash"]==sortphash["image_phash"].iloc[-1]]
tmpdf
"""
* same phash, but images are different. Visualing the images with max counts.
"""
plt.figure(figsize=(20,20))

for num,a in enumerate(tmpdf["path"]):
    plt.subplot(6,5,num+1)
    img = cv2.imread(a)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    plt.axis("off")
    plt.title(tmpdf["posting_id"].iloc[num])
    plt.imshow(img)
    
"""
* images are different but we can see all same.
"""
"""
## 1.5 Understanding same titles
"""
traindf
titlegroup = traindf.groupby("title")["path"].count().reset_index()
titlegroup.columns=["title","counts"]
titlegroup
sorttitle = titlegroup.sort_values("counts")
sorttitle
"""
* max same title is 9. Visualing the images with max counts title.
"""
tmpdf = traindf[traindf["title"]==sorttitle["title"].iloc[-1]]
tmpdf
plt.figure(figsize=(20,20))

for num,a in enumerate(tmpdf["path"]):
    plt.subplot(3,5,num+1)
    img = cv2.imread(a)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    plt.axis("off")
    plt.imshow(img)
    
"""
* title is same but images are different.
"""
"""
# 2. test.csv
"""
testdf
plt.figure(figsize=(20,20))

for num,a in enumerate(testdf["path"]):
    
    plt.subplot(1,3,num+1)
    img = cv2.imread(a)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    plt.imshow(img)
    plt.axis("off")
    
"""
# 3.submission.csv
"""
sample
"""
## submission rules

sample_submission.csv - a sample submission file in the correct format.

posting_id - the ID code for the posting.

matches - Space delimited list of all posting IDs that match this posting. Posts always self-match. Group sizes were capped at 50, so there's no need to predict more than 50 matches.
"""
"""
# 4.Easy test to submission
### From above results, I thought it would match if the image_phash were the same.
### To achieve that, I used defaultdict, but firstly I practiced with train.csv because test.csv is too short and difficult to understand.
"""
practicedf = traindf.iloc[:1000,:]
practicedf
from collections import defaultdict
every_phash = defaultdict(list)
for num, row in enumerate(practicedf[['posting_id', 'image_phash']].values):
    every_phash[row[1]].append(row[0])
#every_phash
every_phash_list = []

for num, row in enumerate(practicedf[['posting_id','image_phash']].values):
    pred = ""
    for a in every_phash[row[1]]:
        pred = pred + a + " "

    
    pred=pred[:-1] # delete last space
    
    every_phash_list.append(pred)
every_phash_list[:20]
practicedf["matches"] = every_phash_list
practicedf
"""
## The case using test data to submit
"""
testdf
every_phash = defaultdict(list)
for num, row in enumerate(testdf[['posting_id', 'image_phash']].values):
    every_phash[row[1]].append(row[0])
every_phash
every_phash_list = []

for num, row in enumerate(testdf[['posting_id','image_phash']].values):
    pred = ""
    for a in every_phash[row[1]]:
        pred = pred + a + " "

    
    pred=pred[:-1] # delete last space
    
    every_phash_list.append(pred)
testdf["matches"] = every_phash_list
testdf
submission = testdf[["posting_id","matches"]]
submission
submission.to_csv("submission.csv",index=False)
"""
## Thank you for reading!! Attension : internet must be off.
"""