# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# +
# default_exp pictures

# + active=""
# # Pictures
#
# >  Create List with all pictures
#
# Will use a csv file to store the list data
# -

#hide
from nbdev.showdoc import *

# #export 
#
# import os
# import pandas as pd
# from os.path import isfile, join
#

# The first step is to get all directories. The reason for this is that the same photo might be stored in different directories. But there also might be photos with the same name but with different content. We will need to compare all photos with the same name in the different directories.  
# Therefore we start in building a pandas DataFrame with all essential data.  
#
# It will contain the directory, the file name, the size and the different time stamps. If this information is not sufficient we will also need to read the picture pixel by pixel and compare. 

# We now start with the directory list. The root is the mapped MyCloud fileserver with the "My Pictures" folder. We will create an iteratively created list of subdirectories and append them to 'my_Dirs'.

# +
#export

mypath = r"/media/mycloud/My Pictures"

def fast_scandir(dirname):
    subfolders= [f.path for f in os.scandir(dirname) if f.is_dir()]
    for dirname in list(subfolders):
        subfolders.extend(fast_scandir(dirname))
    return subfolders
my_Dirs = fast_scandir(mypath)
# -

#hide 
print(len(my_Dirs))
for i in my_Dirs:
    print(i)

# This cell gets all the files which are in the above collected directories.  
# The next step is to create lists which contain the 
# - directory name, the file name and absolute path to each file
# - statistical data, like the file size and creation time stamp.

# +
#export
stat_data = []
entry_data = []

for dirname in my_Dirs:
    with os.scandir(dirname) as dir_content:
        for entry in dir_content:
            if isfile(join(dirname, entry)):
                info = entry.stat()        
                stat_data.append(info)
                entry_data.append([dirname, entry.name, entry.path])
# -

# The next step is to create pandas DataFrames out of the lists, and To make this more readable we change the column names to be more recognizeable. Then we concatenate them to a bigger DataFrame.  
# .

# +
#hide
print(len(stat_data))
print(len(entry_data))

print(entry_data[50:60])
# -

#export
df1 = pd.DataFrame(entry_data)

#hide
df1.head()

# Renaming of DataFrame 1:
#
# > note: the parameter 'inplace=True' is necessary otherwise the DataFrame immediately forgets the renaming again. 

#export 
df1.rename(columns={0: "directory", 1: "file_name", 2: "absolute_file_name"}, inplace=True)
# df1.head()

# Renaming of DataFrame 2:
#
# > note: the rename function did not allow to insert a line break to make the code more readable. Therefore I decided to split the code into 2 lines.

#export
df2 = pd.DataFrame(stat_data)
df2.rename(columns={ 0 : "st_mode", 1: "st_ino", 2 : "st_dev", 3 : "st_nlink", 4 : "st_uid", 5: "st_gid",}, inplace=True) 
df2.rename(columns={ 6 : "st_size", 7 : "st_atime", 8 : "st_mtime", 9 : "st_ctime"}, inplace=True) 

#export 
df = pd.concat([df1, df2], axis=1)
df.head()

# Now we start processing the data. The first step is to sort the files according to the name. By this I hope to identify a lot of duplicates. 

#export
df.sort_values("file_name", inplace=True)
df.head()

#export  
df.duplicated(subset=['file_name'])

# +
#export
df_duplicates = df[df['file_name'].duplicated(keep=False)]
len(df_duplicates)

# df[df.duplicated(['ID'], keep=False)]
# data.sort_values(by=['Order ID'], inplace=True)
# df = data[data['Order ID'].duplicated(keep=False)]

# -

#export
df_duplicates[80:113]



StarcoderdataJupyterScriptsDedupFiltered