MarkCitrus / MarkStreamlit.py
JPLTedCas's picture
Upload 2 files
6d6e4bc
raw
history blame
21.6 kB
import streamlit as st
import pandas as pd
uploaded_file = st.file_uploader("Choose product file", type="csv")
if uploaded_file:
#df = pd.read_excel(uploaded_file)
df = pd.read_csv(uploaded_file, encoding='utf8')
#st.dataframe(df)
uploaded_file2 = st.file_uploader("Choose inventory file", type="csv")
if uploaded_file2:
#df2 = pd.read_excel(uploaded_file2)
df2 = pd.read_csv(uploaded_file2, encoding='utf8')
#st.dataframe(df2)
#st.table(df2)
def ConvertCitrus(df,df2):
# Load pandas
import re as re
import RemoveHTMLtags as RHT
#INPUT FILE
#df = pd.read_csv('C:/Users/15572890/Desktop/I+D/MarksCsvConversion/Validation2/products_export_1 21-10-22.csv', encoding='utf8')
#df2 = pd.read_csv('C:/Users/15572890/Desktop/I+D/MarksCsvConversion/Validation2/inventory_export_1 21-10-22.csv', encoding='utf8')
df.to_excel('C:/Users/15572890/Desktop/I+D/MarksCsvConversion/Validation2/products_export_1.xlsx',index=False)
df2.to_excel('C:/Users/15572890/Desktop/I+D/MarksCsvConversion/Validation2/inventory_export_1.xlsx',index=False)
tagsp=str('<style type=')+str('"')+str('"')+str('text/css')+str('"')+str('"')+str('><!--')
tags_list = ['<p class=','"p1"', 'data-mce-fragment="1">,','<b data-mce-fragment="1">','<i data-mce-fragment="1">','<p>' ,'</p>' , '<p*>',
'<ul>','</ul>',
'</i>','</b>','</p>','</br>',
'<li>','</li>',
'<br>',
'<strong>','</strong>',
'<span*>','</span>', '"utf-8"','UTF-8',
'<a href*>','</a>','<meta charset=utf-8>',';;',
'<em>','</em>','"','<meta charset=','utf-8>','<p>','<p','data-mce-fragment=1',';','<style type=','<style type=','><!--','text/css','<style type=\"\"text/css\"\"><!--','--></style>','td {border: 1px solid #ccc','}br {mso-data-placement:same-cell','}','>']
def remove_html_tags(text):
"""Remove html tags from a string"""
import re
clean = re.compile('<.*?>')
return re.sub(clean, '', text)
#for tag in tags_list:
## df['overview_copy'] = df['overview_copy'].str.replace(tag, '')
# df.replace(to_replace=tag, value='', regex=True, inplace=True)
for index, row in df.iterrows():
df.iloc[index,2]=RHT.remove_tags(str(df.iloc[index,2]))
print(df.iloc[:,2])
df.to_excel('C:/Users/15572890/Desktop/I+D/MarksCsvConversion/Validation2/products_export_1-nohtml.xlsx')
#df.fillna('', inplace=True)
df.iloc[:,2] = pd.Series(df.iloc[:,2],dtype="string")
print(df.iloc[:,2].dtype)
#s = pd.Series(['a', 'b', 'c'], dtype="string")
#s.dtype
#CONVERT FORMATS
#Column A(0) – Ignore
#Column B(1) “Title” > Column B(1) “Product Name”
#Column C(2) – Ignore
#Column D(3) “Vendor” > Column K(10) “Brand”
#Column F(5) “Custom Product Type” > Column AF(31) “Short Description”
#Column J(9) “Option1 Value” > Column I(8) “Size 1”
#Column L(11) “Option2 Value” > Column H(7) > Colour
#Column M(12) - Ignore
#Column N(13) “Option 3 Value” > Column A(0) “Style Number”
#1. Problems in Column N. Some codes do not stay as a number when the Citrus Lime csv is re-opened (8.05652E+12 instead of 8056516179091) The saved csv keeps turning this column back to “general’ format column when I re-open it, even after I save it as number column. The upload must keep this as a number formatted column.
#Column O(14) - Ignore
#Column P(15) “Variant Grams” > Column AE (30) “Weight (grams)”
#Column R(17) “Variant Inventory Qty” > Column R (17) “Stock Count”. THIS IS THE KEY TO THE DAILY UPLOAD
#Column U(20) “Variant Price” > Column F (5) “Unit MSRP”
#Column Y > C&D
#################################################################################################
temp_cols=df.columns.tolist()
new_cols=temp_cols.copy()
new_cols[1]=temp_cols[1]
new_cols[17]=temp_cols[17]
#################################################################################################
#THERE IS NO EXISTING COLUMN ON THE SHOPIFY EXPORT TO DIRECTLY PROVIDE DATA FOR COLUMN E ON THE CITRUS LIME CSV (which is the wholesale price ex VAT to the retailer). However – Column U “ Variant Price” can provide the information for Column E with the following formula:
#((Column U/1.2)/1.6)*0.96
#Column Y “Variant Barcode” > Column C “Vendor SKU” (2) (and D "UPC/EAN" (3)??)
#There are 2 problems with converting Column Y to Column C.
#2. Shopify exports the UPC data and adds an apostrophe. This fails the SIM process. We need to get data without the apostrophe.
#3. Vendor SKU. The CSV file keeps switching the data to a non-number eg 8056516178308 shows as 8.05652E+12. The saved csv keeps turning this column to “general’ format column when I re-open it, even after I save it as number column. The upload must keep this as a number formatted column.
#This is where it gets complicated…
#Shopify exports the image file as https:// links in an odd way. Instead of attributing image 1, image 2, and image 3 etc in dedicated and separate columns, it spreads them across the sizes for the related product in the same column (Column Z “Image Src”). Column AA in the Shopify export csv just shows the image position instead. We need to find a solution. We need to be able to provide https// image links in separate columns for each product and size. For example, if a product has 3 images, these need to be converted into Citrus Lime CSV columns Column Z “Image 1”, Column AA “Image 2”, Column AB “Image 3”, Column AC “Image 4” etc.
#new_cols[4]=((temp_cols[20]/1.2)/1.96)*0.96
#Column C “Body (HTML)” > Column AG “Long Description” (32)
df_copy=df[new_cols].copy(deep=True)
print("SKU")
print(df.iloc[:,24])
local_df = df.copy(deep=True)
df_copy.iloc[:,0]=local_df.iloc[:,13].copy(deep=True)
df_copy.iloc[:,5]=local_df.iloc[:,20].copy(deep=True)
df_copy.iloc[:,7]=local_df.iloc[:,11].copy(deep=True)
#24 is variant Bar code
df_copy.iloc[:,2]=local_df.iloc[:,24].copy(deep=True)
df_copy.iloc[:,8]=local_df.iloc[:,9].copy(deep=True)
df_copy.iloc[:,10]=local_df.iloc[:,3].copy(deep=True)
df_copy.rename(columns={df_copy.columns[10]: 'Brand'},inplace=True)
df_copy.columns.values[10] = 'Brand'
df_copy.iloc[:,30]=local_df.iloc[:,15].copy(deep=True)
df_copy.iloc[:,31]=local_df.iloc[:,5].copy(deep=True)
df_copy.iloc[:,32]=local_df.iloc[:,2].copy(deep=True)
df_copy.rename(columns={df_copy.columns[8]: 'Size 1'},inplace=True)
print(list(df_copy.columns.values))
#WE CONVERT COLUMN 20 to numeric (in case it's read as string)
df_copy.iloc[:,20] = df_copy.iloc[:,20].astype(float)
df_copy.iloc[:,4]=(((df_copy.iloc[:,20]/1.2)/1.96)*0.96)
from babel.numbers import format_currency
df_copy.iloc[:,4] = df_copy.iloc[:,4].apply(lambda x: format_currency(x, currency="GBP", locale="en_GB"))
df_copy.iloc[:,5] = df_copy.iloc[:,5].apply(lambda x: format_currency(x, currency="GBP", locale="en_GB"))
print(((df_copy.iloc[:,20]/1.2)/1.96)*0.96)
#df_copy.iloc[:,2]=df_copy.iloc[:,2].str.replace("'","")
df_copy.iloc[:,2] = df_copy.iloc[:,2].astype(str).str.replace("'","")
#df_copy.iloc[:,24]=df_copy.iloc[:,24].str.replace("'","")
df_copy.iloc[:,24] = df_copy.iloc[:,24].astype(str).str.replace("'","")
print("SKU")
print(df_copy.iloc[:,2])
#rename specific column names
#df_copy.rename(columns = {'Variant Inventory Qty':'Stock Count','Variant Grams' : 'Weight (grams)'}, inplace = True)
#df_copy.rename(columns = {'Option2 Value':'Colour','Option1 Value' : 'Size 1'}, inplace = True)
#df_copy.rename(columns = {'Vendor':'Brand','Title' : 'Product Name'}, inplace = True)
#df_copy.rename(columns = {'Body (HTML)':'Long Description'}, inplace = True)
#df_copy.rename(columns={df_copy.columns[4]: 'Unit Cost'},inplace=True)
print(list(df_copy.columns.values))
#df_copy.rename(columns={df_copy.columns[31]: 'Short Description'},inplace=True)
#df_copy.rename(columns={df_copy.columns[2]: 'Vendor SKU'},inplace=True)
df_copy.rename(columns={df_copy.columns[6]: 'Colour Code (Simple Colour)'},inplace=True)
##IN COLUMN H (6), WE HAVE SOME TAGS AND WE WANT TO GET THE TAG "MEN, WOMEN, LADY OR BOTH (UNISEX)"
#WE ARE GETTING THAT INFO BEFORE REMOVING DATA FROM 6
for index, row in df_copy.iterrows():
if index==0:
print(row['Colour Code (Simple Colour)'])
if " mens" in str(row['Colour Code (Simple Colour)']):
if " womens" in str(row['Colour Code (Simple Colour)']):
df_copy.iloc[index,12]="Unisex"
else:
df_copy.iloc[index,12]="Mens"
if " womens" in str(row['Colour Code (Simple Colour)']):
if " mens" in str(row['Colour Code (Simple Colour)']):
df_copy.iloc[index,12]="Unisex"
else:
df_copy.iloc[index,12]="Womens"
if " ladys" in str(row['Colour Code (Simple Colour)']):
df_copy.iloc[index,12]="Ladys"
if index==0:
print(row[12])
print(df_copy.iloc[:,12])
df_copy.iloc[:,6] = ""
#Style Number Product Name Vendor SKU UPC/EAN Unit Cost Unit MSRP Colour Code (Simple Colour) Colour
df_copy.rename(columns={df_copy.columns[0]: 'Style Number'},inplace=True)
df_copy.rename(columns={df_copy.columns[1]: 'Product Name'},inplace=True)
df_copy.rename(columns={df_copy.columns[2]: 'Vendor SKU'},inplace=True)
df_copy.rename(columns={df_copy.columns[3]: 'UPC/EAN'},inplace=True)
df_copy.rename(columns={df_copy.columns[4]: 'Unit Cost'},inplace=True)
df_copy.rename(columns={df_copy.columns[5]: 'Unit MSRP'},inplace=True)
df_copy.rename(columns={df_copy.columns[6]: 'Colour Code (Simple Colour)'},inplace=True)
print(df_copy.columns[6])
df_copy.rename(columns={df_copy.columns[7]: 'Colour'},inplace=True)
#Size 1 Size 2 Brand Year or Season Gender Manufacturer Part Code Other Barcode VAT Pack Qty
df_copy.rename(columns={df_copy.columns[8]: 'Size 1'},inplace=True)
df_copy.rename(columns={df_copy.columns[9]: 'Size 2'},inplace=True)
df_copy.rename(columns={df_copy.columns[10]: 'Brand'},inplace=True)
df_copy.rename(columns={df_copy.columns[11]: 'Year of Season'},inplace=True)
df_copy.rename(columns={df_copy.columns[12]: 'Gender'},inplace=True)
df_copy.rename(columns={df_copy.columns[13]: 'Manufacturer Part Code'},inplace=True)
df_copy.rename(columns={df_copy.columns[14]: 'Other Bar Code'},inplace=True)
df_copy.rename(columns={df_copy.columns[15]: 'VAT'},inplace=True)
df_copy.rename(columns={df_copy.columns[16]: 'Pack Qty'},inplace=True)
#Stock Count Price Band 1 Price Band 2 IE VAT Unit Cost in Euros MSRP in Euros
df_copy.rename(columns={df_copy.columns[17]: 'Stock Count'},inplace=True)
df_copy.rename(columns={df_copy.columns[18]: 'Price Band 1'},inplace=True)
df_copy.rename(columns={df_copy.columns[19]: 'Price Band 2'},inplace=True)
df_copy.rename(columns={df_copy.columns[20]: 'IE VAT'},inplace=True)
df_copy.rename(columns={df_copy.columns[21]: 'Unit Cost in Euros'},inplace=True)
df_copy.rename(columns={df_copy.columns[22]: 'MSRP in Euros'},inplace=True)
#Commodity Codes Country of Origin Image (multiple images can be added in separate columns if available)
df_copy.rename(columns={df_copy.columns[23]: 'Commodity Codes'},inplace=True)
df_copy.rename(columns={df_copy.columns[24]: 'Country of Origin'},inplace=True)
#Weight Short Description Long Description Video Link
df_copy.rename(columns={df_copy.columns[30]: 'Weight'},inplace=True)
df_copy.rename(columns={df_copy.columns[31]: 'Short Description'},inplace=True)
df_copy.rename(columns={df_copy.columns[32]: 'Long Description'},inplace=True)
df_copy.rename(columns={df_copy.columns[33]: 'Video Link'},inplace=True)
df_copy.iloc[:,9] = ""
df_copy.iloc[:,13] = ""
df_copy.iloc[:,14] = ""
df_copy.iloc[:,16] = ""
df_copy.iloc[:,18] = ""
df_copy.iloc[:,19] = ""
df_copy.iloc[:,20] = ""
df_copy.iloc[:,21] = ""
df_copy.iloc[:,22] = ""
#df_copy.rename(columns={df_copy.columns[26]: 'Weight (Grams)'},inplace=True)
#df_copy.iloc[:,26] = ""
df_copy.iloc[:,33] = ""
#df_copy.iloc[:,5] = " "
df_copy.iloc[:,15] = "20"
print(list(df_copy.columns.values))
#Column Y in the export and this code should go into both Columns C and D in the conversion with the titles “Vendor SKU” and “UPC/EAN” It is replicated for a complicated reason that I won’t explain here, but Column Y in the export should go into both Column C and D in the conversion
df_copy.iloc[:,3] = df_copy.iloc[:,2]
df_copy.columns.values[10] = 'Brand'
df_copy.iloc[:,11] = ""
df_copy.iloc[:,22] = ""
#df_copy.rename(columns={df_copy.columns[30]: 'Weight (Grams)'},inplace=True)
print("SKU")
print(df_copy.iloc[:,2])
#DATA COMING FROM THE OTHER CSV FILE
df_copy.iloc[:,23] = ""
df_copy.iloc[:,24] = ""
#WARNING: HEADER IS IN SECOND ROW. WE DONT HAVE INTO ACCOUNT FIRST ROW
#df2 = pd.read_excel('C:/Users/15572890/Desktop/I+D/MarksCsvConversion/inventory_export_12.xlsx',engine="openpyxl", header=1)
#WE HAVE TO REORDER COLUMNS COO and HS Code in df2 in order to match the index order of df
#list1=df_copy.set_index('Vendor SKU').T.to_dict('list')
#print(list1)
new_index=df['Variant SKU']
boolean = df['Variant SKU'].duplicated().any()
#print(boolean)
boolean = df2['SKU'].duplicated().any()
#print(boolean)
duplicateRows2 = df2[df2.duplicated(['SKU'],keep = False)]
#print(duplicateRows2['SKU'])
duplicateRows = df[df.duplicated(['Variant SKU'],keep = False)]
#print(duplicateRows)
#print(duplicateRows['Variant SKU'])
#print(new_index)
df2=df2.set_index('SKU')
#print(df2)
#i=df2.index
#for x in i:
# print(x)
df2.reindex(new_index)
#i=df2.index
#for x in i:
# print(x)
#print(df2)
#print(df2.index)
#df3 = pd.DataFrame(students, index=['a', 'b', 'c', 'd', 'e'])
#print("Original DataFrame: ")
#print(df)
print("TERMINE")
df_copy.iloc[:,24] = df2.loc[:,'COO']
df_copy.iloc[:,23] = df2.loc[:,'HS Code']
df_copy['Commodity Codes']=df2['HS Code'].values
df_copy['Country of Origin']=df2['COO'].values
#print(df2.loc[:,'COO'])
#print(df2.loc[:,'HS Code'])
#print(df_copy.iloc[:,24])
#print(df_copy.iloc[:,23])
print("SKU")
print(df_copy.iloc[:,2])
#WE COMPLETE THE DATAFRMAE WITH DUMMY COLUMNS TILL THE MAXIMUM DESIRED NUMBER
header_list=[]
for i in range(49,58):
#df.insert(i, "Dummy", [], True)
header_list.append(str(i))
df_copy[str(i)]=''
column_indices=[]
for i in range(0,24):
column_indices.append(34+i)
#Tech Specs Size Chart Geometry Chart Frame Rear Shock Fork
#Headset Stem Handlebar Bar Tape / Grip Brakes Levers Brake Calipers Tyres Wheels Front Derailleur
#Rear Derailleur Shift Levers Chain Cassette Chainset Bottom Bracket Pedals Saddle Seatpost
old_names = df_copy.columns[column_indices]
new_names = ['Tech Specs','Size Chart','Geometry Chart','Frame', 'Rear Shock', 'Fork', 'Headset', 'Stem', 'Handlebar', 'Bar Tape / Grip', 'Brakes Levers', 'Brake Calipers', 'Tyres', 'Wheels', 'Front Derailleur', 'Rear Derailleur', 'Shift Levers' ,'Chain' ,'Cassette' ,'Chainset' ,'Bottom Bracket', 'Pedals', 'Saddle', 'Seatpost']
old_names = df_copy.columns[column_indices]
df_copy.rename(columns=dict(zip(old_names, new_names)), inplace=True)
df_copy.iloc[:,34:58]=''
print("SKUf")
print(df_copy.iloc[:,2])
#print(df_copy.iloc[:,3])
## Rename all columns with list
#cols = ['Courses','Courses_Fee','Courses_Duration']
#df_copy.columns = cols
#print(df.columns)
###################
#PUT IMAGES IN A SIGNLE ROW: WE LOOK FOR IMAGES COMING FROM COMMON NAMES
#Shopify exports the image file as https:// links in an odd way. Instead of attributing image 1, image 2, and image 3 etc in dedicated
#and separate columns, it spreads them across the sizes for the related product in the same column (Column Z “Image Src”).
#Column AA in the Shopify export csv just shows the image position instead. We need to find a solution.
#We need to be able to provide https// image links in separate columns for each product and size. For example, if a product has 3 images,
#these need to be converted into Citrus Lime CSV columns Column Z “Image 1”, Column AA “Image 2”, Column AB “Image 3”, Column AC “Image 4”
#etc
####################
#region imagesRow2Column
#We get the list of rows with NAN data in Product Name column (same product name but different sizes (XS, XL...). Each of these rows has a image scr link
list_col=df_copy.loc[pd.isna(df_copy.loc[:,'Product Name']), :].index
images=df_copy.loc[list_col,'Image Src']
list_end=[]
for row in df_copy.index:
#NotNA gets rows where Product Name column has a name in it (first image and row where we should add the images)
if pd.notna(df_copy.loc[row,'Product Name']):
#print(df_copy.loc[row,'Product Name'])
rowNotNa=row
i=1
#j=1
list_img=[]
#WE INCLUDE IN THE LIST THE FIRST IMAGE
list_img.append(df_copy.loc[row,'Image Src'])
while pd.isna(df_copy.loc[row+i,'Product Name']) and row+i<len(df_copy.index)-1:
#WE ADD THE REST OF THE IMAGES (FOLLOWING ROWS)
if "http" in str(df_copy.loc[row+i,'Image Src']):
list_img.append(df_copy.loc[row+i,'Image Src'])
i=i+1
list_end.append(list_img)
#IN list_end WE HAVE ALL OF THE IMAGES FOR EACH PRODUCT NAME
index_nonnan=df_copy.loc[pd.notna(df_copy.loc[:,'Product Name']), :].index
max=0
for i in range(len(list_end)):
if max<len(list_end[i]):
max=len(list_end[i])
print("SKUf")
print(df_copy.iloc[:,2])
#WE CHANGE THE COLUMN NAME OF THE COLUMNS WHERE THERE ARE IMAGES: EACH COLUMN IS CALLED "Image x"
#We first delete old values in the Image columns
for j in range(max):
df_copy.iloc[:,25+j]=''
counter=0
for index in index_nonnan:
for j in range(len(list_end[counter])):
if list_end[counter][j]!='nan':
df_copy.iloc[index,25+j]=list_end[counter][j]
df_copy.rename(columns={df_copy.columns[25+j]: 'Image'+str(j+1)},inplace=True)
counter=counter+1
print("SKUf")
print(df_copy.iloc[:,2])
#WE HAVE TO FILL NAN ROWS (SAME PRODUCT BUT DIFFERENT SIZES) WITH THE SAME IMAGES THAT IN NON NAN ROWS (MAIN PRODUCT-SIZE)
listImages=[None] * max
list1=[None] * max
list2=[None] * max
list3=[None] * max
list4=[None] * max
list5=[None] * max
for index, row in df_copy.iterrows():
#NotNA gets rows where Product Name column has a name in it (first image and row where we should add the images)
#print(df_copy.iloc[index,1])
if pd.notna(df_copy.iloc[index,1]):
for j in range(0,max):
listImages[j]=str((df_copy.iloc[index,25+j]))
#list1[j]=str((df_copy.iloc[index,1+j]))
#list2[j]=str((df_copy.iloc[index,10+j]))
#list3[j]=str((df_copy.iloc[index,12+j]))
#list4[j]=str((df_copy.iloc[index,31+j]))
#list5[j]=str((df_copy.iloc[index,32+j]))
list1[j]=str((df_copy.iloc[index,1]))
list2[j]=str((df_copy.iloc[index,10]))
list3[j]=str((df_copy.iloc[index,12]))
list4[j]=str((df_copy.iloc[index,31]))
list5[j]=str((df_copy.iloc[index,32]))
else:
for j in range(0,max):
df_copy.iloc[index,25+j]=listImages[j]
#df_copy.iloc[index,1+j]=list1[j]
#df_copy.iloc[index,10+j]=list2[j]
#df_copy.iloc[index,12+j]=list3[j]
#df_copy.iloc[index,31+j]=list4[j]
#df_copy.iloc[index,32+j]=list5[j]
df_copy.iloc[index,1]=list1[j]
df_copy.iloc[index,10]=list2[j]
df_copy.iloc[index,12]=list3[j]
df_copy.iloc[index,31]=list4[j]
df_copy.iloc[index,32]=list5[j]
#endregion
print("SKUf")
print(df_copy.iloc[:,2])
#print(df_copy.iloc[:,3])
###################################################################################
df_copy.to_excel('C:/Users/15572890/Desktop/I+D/MarksCsvConversion/Validation2/OCCHIO-Cycle-Data-File_st.xlsx',index=False)
#df_copy.to_csv('C:/Users/15572890/Desktop/I+D/MarksCsvConversion/Validation2/OCCHIO-Cycle-Data-File.csv',index=False, encoding='utf-8')
df_copy.to_csv('C:/Users/15572890/Desktop/I+D/MarksCsvConversion/Validation2/OCCHIO-Cycle-Data-File_st.csv',index=False, encoding='utf_8_sig')
if uploaded_file and uploaded_file2:
ConvertCitrus(df,df2)