working scheduler

.env

@@ -1,3 +1,14 @@
+# Airflow
 AIRFLOW_UID=1000
+
+#CMC
 URL_CMC=https://pro-api.coinmarketcap.com
-API_KEY_CMC=8057498e-ad35-465c-8359-8f6cc9d1ae1b
+API_KEY_CMC=8057498e-ad35-465c-8359-8f6cc9d1ae1b
+
+# Etherscan API
+URL_ETHERSCAN=https://api.etherscan.io/api
+API_KEY_ETHERSCAN=VVJFE7IG5WSK8P2UJIJN5MST5UBQEZYVYW
+
+# Logging
+LOG_FOLDER=./logs/scrapping/
+LOG_STREAMLIT=./logs/streamlit/

app.py

@@ -1,39 +1,76 @@
-#-------------------------------------libraries ----------------------------------
-
+# ------------------------ Libraries --------------------------
 import os
 import pandas as pd
 import streamlit as st
 import plotly.graph_objs as go
-import numpy as np
-import plotly.express as px 
 import logging
-# Set up logging basic configuration
-logging.basicConfig(level=logging.INFO)
-# Example of logging
+import subprocess
+import threading
+from dotenv import load_dotenv
+from requests.exceptions import ConnectionError, Timeout, TooManyRedirects
+
+# ------------------------ Environment Variables --------------------------
+
+load_dotenv()
+log_folder = os.getenv("LOG_FOLDER")
+# Logging
+log_folder = os.getenv("LOG_STREAMLIT")
+os.makedirs(log_folder, exist_ok=True)
+log_file = os.path.join(log_folder, "front.log")
+log_format = "%(asctime)s [%(levelname)s] - %(message)s"
+logging.basicConfig(filename=log_file, level=logging.INFO, format=log_format)
 logging.info("Streamlit app has started")
+# Create output folder if it doesn't exist
+if not os.path.exists("output"):
+    os.makedirs("output")
+
+
+#-------------------------------------back----------------------------------
 
+def safe_read_csv(file_path, sep=','):
+    if os.path.exists(file_path) and os.path.getsize(file_path) > 0:
+        return pd.read_csv(file_path, sep=sep)
+    else:
+        logging.warning(f"File {file_path} is empty or does not exist.")
+        return pd.DataFrame()  # return an empty DataFrame
 
-#-------------------------------------back ----------------------------------
 
 # etherscan
 ## Load the data from the CSV files
-dataframes = []
+df_etherscan = pd.DataFrame()
 for filename in os.listdir('output'):
-    if filename.endswith('.csv'):
-        df_temp = pd.read_csv(os.path.join('output', filename), sep=';')
-        dataframes.append(df_temp)
-df_etherscan = pd.concat(dataframes)
-del df_temp
+    if filename.endswith('.csv') and 'transactions_' in filename:
+        df_temp = safe_read_csv(os.path.join('output', filename), sep=',')
+        df_etherscan = pd.concat([df_etherscan, df_temp], ignore_index=True)
 
 # CMC
 ## Load cmc data
-df_temp = pd.read_csv("output/top_100_update.csv", sep=',')
-df_cmc = df_temp[df_temp["last_updated"] == df_temp["last_updated"].max()]
-del df_temp
+df_cmc = safe_read_csv("output/top_100_update.csv", sep=',')
+df_cmc = df_cmc[df_cmc["last_updated"] == df_cmc["last_updated"].max()]
+
+# Function to execute the scraping functions
+def execute_etherscan_scraping():
+    subprocess.call(["python", "utils/scrap_etherscan.py"])
+    logging.info("Etherscan scraping completed")
+    threading.Timer(3600, execute_etherscan_scraping).start()
+    
+# Function to execute the scraping functions
+def execute_cmc_scraping():
+    subprocess.call(["python", "utils/scrap_cmc.py"])
+    logging.info("CMC scraping completed")
+    threading.Timer(2592000 / 9000, execute_cmc_scraping).start()
+
+if "initialized" not in st.session_state:
+    # Start the scraping threads
+    threading.Thread(target=execute_etherscan_scraping).start()
+    threading.Thread(target=execute_cmc_scraping).start()
+    st.session_state["initialized"] = True
+
 #-------------------------------------streamlit ----------------------------------
 
 # Set the title and other page configurations
 st.title('Crypto Analysis')
+
 # Create two columns for the two plots
 col1, col2 = st.columns(2)
 

logs/scrapping/cmc_scrapping.log

@@ -0,0 +1,178 @@
+2023-11-27 01:13:54,747 [INFO] - CMC data script execution completed.
+2023-11-27 01:13:54,748 [INFO] - Function fetch_and_process_cmc_data executed in 0.37 seconds
+2023-11-27 01:13:55,094 [INFO] - CMC data script execution completed.
+2023-11-27 01:13:55,095 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:13:55,162 [INFO] - CMC data script execution completed.
+2023-11-27 01:13:55,163 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-27 01:13:55,971 [INFO] - CMC data script execution completed.
+2023-11-27 01:13:55,972 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:14:01,229 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:01,230 [INFO] - Function fetch_and_process_cmc_data executed in 0.43 seconds
+2023-11-27 01:14:01,957 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:01,958 [INFO] - Function fetch_and_process_cmc_data executed in 0.15 seconds
+2023-11-27 01:14:08,224 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:08,225 [INFO] - Function fetch_and_process_cmc_data executed in 0.18 seconds
+2023-11-27 01:14:25,974 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:25,975 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:14:28,006 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:28,006 [INFO] - Function fetch_and_process_cmc_data executed in 0.15 seconds
+2023-11-27 01:14:28,131 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:28,132 [INFO] - Function fetch_and_process_cmc_data executed in 0.14 seconds
+2023-11-27 01:14:28,994 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:28,995 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:14:29,992 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:29,993 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:14:30,178 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:30,178 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:14:32,026 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:32,027 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:14:32,136 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:32,136 [INFO] - Function fetch_and_process_cmc_data executed in 0.15 seconds
+2023-11-27 01:14:33,268 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:33,269 [INFO] - Function fetch_and_process_cmc_data executed in 0.18 seconds
+2023-11-27 01:14:33,274 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:33,275 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-27 01:14:33,609 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:33,610 [INFO] - Function fetch_and_process_cmc_data executed in 0.15 seconds
+2023-11-27 01:14:33,702 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:33,703 [INFO] - Function fetch_and_process_cmc_data executed in 0.15 seconds
+2023-11-27 01:14:33,705 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:33,707 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:14:34,272 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:34,273 [INFO] - Function fetch_and_process_cmc_data executed in 0.15 seconds
+2023-11-27 01:14:34,402 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:34,403 [INFO] - Function fetch_and_process_cmc_data executed in 0.15 seconds
+2023-11-27 01:14:34,962 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:34,963 [INFO] - Function fetch_and_process_cmc_data executed in 0.14 seconds
+2023-11-27 01:14:36,072 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:36,073 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:14:36,137 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:36,137 [INFO] - Function fetch_and_process_cmc_data executed in 0.14 seconds
+2023-11-27 01:14:36,979 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:36,979 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:14:38,087 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:38,088 [INFO] - Function fetch_and_process_cmc_data executed in 0.18 seconds
+2023-11-27 01:14:38,274 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:38,275 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:14:38,291 [INFO] - CMC data script execution completed.
+2023-11-27 01:14:38,292 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:18:43,253 [INFO] - CMC data script execution completed.
+2023-11-27 01:18:43,254 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-27 01:18:43,564 [INFO] - CMC data script execution completed.
+2023-11-27 01:18:43,565 [INFO] - Function fetch_and_process_cmc_data executed in 0.18 seconds
+2023-11-27 01:18:43,626 [INFO] - CMC data script execution completed.
+2023-11-27 01:18:43,627 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:18:44,377 [INFO] - CMC data script execution completed.
+2023-11-27 01:18:44,377 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:18:49,644 [INFO] - CMC data script execution completed.
+2023-11-27 01:18:49,645 [INFO] - Function fetch_and_process_cmc_data executed in 0.18 seconds
+2023-11-27 01:18:50,367 [INFO] - CMC data script execution completed.
+2023-11-27 01:18:50,368 [INFO] - Function fetch_and_process_cmc_data executed in 0.18 seconds
+2023-11-27 01:18:56,635 [INFO] - CMC data script execution completed.
+2023-11-27 01:18:56,636 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:19:14,411 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:14,412 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-27 01:19:16,433 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:16,434 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:19:16,630 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:16,631 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-27 01:19:17,395 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:17,396 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:19:18,456 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:18,457 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:19:18,606 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:18,606 [INFO] - Function fetch_and_process_cmc_data executed in 0.18 seconds
+2023-11-27 01:19:20,439 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:20,440 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:19:20,549 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:20,550 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:19:21,867 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:21,869 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:19:21,897 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:21,899 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-27 01:19:22,052 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:22,053 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:19:22,231 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:22,231 [INFO] - Function fetch_and_process_cmc_data executed in 0.15 seconds
+2023-11-27 01:19:22,243 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:22,244 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:19:22,724 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:22,725 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:19:22,857 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:22,858 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:19:23,381 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:23,382 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:19:24,509 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:24,510 [INFO] - Function fetch_and_process_cmc_data executed in 0.18 seconds
+2023-11-27 01:19:24,553 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:24,554 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:19:25,377 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:25,378 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:19:26,517 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:26,518 [INFO] - Function fetch_and_process_cmc_data executed in 0.18 seconds
+2023-11-27 01:19:26,729 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:26,730 [INFO] - Function fetch_and_process_cmc_data executed in 0.15 seconds
+2023-11-27 01:19:27,019 [INFO] - CMC data script execution completed.
+2023-11-27 01:19:27,020 [INFO] - Function fetch_and_process_cmc_data executed in 0.46 seconds
+2023-11-27 01:23:31,981 [INFO] - CMC data script execution completed.
+2023-11-27 01:23:31,982 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:23:32,065 [INFO] - CMC data script execution completed.
+2023-11-27 01:23:32,066 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:23:32,087 [INFO] - CMC data script execution completed.
+2023-11-27 01:23:32,088 [INFO] - Function fetch_and_process_cmc_data executed in 0.58 seconds
+2023-11-27 01:23:32,782 [INFO] - CMC data script execution completed.
+2023-11-27 01:23:32,782 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:23:39,214 [INFO] - CMC data script execution completed.
+2023-11-27 01:23:39,215 [INFO] - Function fetch_and_process_cmc_data executed in 1.33 seconds
+2023-11-27 01:23:39,226 [INFO] - CMC data script execution completed.
+2023-11-27 01:23:39,227 [INFO] - Function fetch_and_process_cmc_data executed in 0.62 seconds
+2023-11-27 01:23:45,140 [INFO] - CMC data script execution completed.
+2023-11-27 01:23:45,141 [INFO] - Function fetch_and_process_cmc_data executed in 0.27 seconds
+2023-11-27 01:24:02,946 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:02,946 [INFO] - Function fetch_and_process_cmc_data executed in 0.30 seconds
+2023-11-27 01:24:05,151 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:05,152 [INFO] - Function fetch_and_process_cmc_data executed in 0.48 seconds
+2023-11-27 01:24:05,276 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:05,276 [INFO] - Function fetch_and_process_cmc_data executed in 0.41 seconds
+2023-11-27 01:24:06,876 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:06,877 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:24:07,313 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:07,314 [INFO] - Function fetch_and_process_cmc_data executed in 0.45 seconds
+2023-11-27 01:24:08,859 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:08,860 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:24:08,976 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:08,976 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:24:10,373 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:10,375 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:24:10,620 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:10,620 [INFO] - Function fetch_and_process_cmc_data executed in 0.40 seconds
+2023-11-27 01:24:10,689 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:10,689 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:24:10,790 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:10,792 [INFO] - Function fetch_and_process_cmc_data executed in 5.16 seconds
+2023-11-27 01:24:11,279 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:11,280 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:24:11,298 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:11,299 [INFO] - Function fetch_and_process_cmc_data executed in 0.30 seconds
+2023-11-27 01:24:11,781 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:11,782 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:24:12,955 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:12,956 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:24:14,058 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:14,059 [INFO] - Function fetch_and_process_cmc_data executed in 0.44 seconds
+2023-11-27 01:24:14,921 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:14,922 [INFO] - Function fetch_and_process_cmc_data executed in 0.16 seconds
+2023-11-27 01:24:15,135 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:15,136 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 01:24:15,500 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:15,501 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-27 01:24:15,701 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:15,701 [INFO] - Function fetch_and_process_cmc_data executed in 5.18 seconds
+2023-11-27 01:24:15,742 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:15,742 [INFO] - Function fetch_and_process_cmc_data executed in 5.44 seconds
+2023-11-27 01:24:17,986 [INFO] - CMC data script execution completed.
+2023-11-27 01:24:17,987 [INFO] - Function fetch_and_process_cmc_data executed in 5.17 seconds
+2023-11-27 23:21:54,744 [INFO] - CMC data script execution completed.
+2023-11-27 23:21:54,745 [INFO] - Function fetch_and_process_cmc_data executed in 0.45 seconds
+2023-11-27 23:26:43,314 [INFO] - CMC data script execution completed.
+2023-11-27 23:26:43,315 [INFO] - Function fetch_and_process_cmc_data executed in 0.28 seconds

logs/scrapping/etherscan_scrap.log

@@ -0,0 +1,85 @@
+2023-11-27 01:14:01,381 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-27 01:14:01,382 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-27 01:14:02,293 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:02,293 [ERROR] - Failed to retrieve latest block number for token: APE
+2023-11-27 01:14:02,305 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-27 01:14:02,305 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-27 01:14:02,364 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-27 01:14:02,364 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-27 01:14:02,397 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:16,638 [INFO] - Created files: 
+2023-11-27 01:14:16,638 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-27 01:14:16,638 [INFO] - Etherscan scraping script execution completed.
+2023-11-27 01:14:16,638 [INFO] - Function fetch_and_update_etherscan executed in 21.73 seconds
+2023-11-27 01:14:19,015 [INFO] - Created files: 
+2023-11-27 01:14:19,015 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-27 01:14:19,015 [INFO] - Etherscan scraping script execution completed.
+2023-11-27 01:14:19,015 [INFO] - Function fetch_and_update_etherscan executed in 17.22 seconds
+2023-11-27 01:14:28,488 [INFO] - Created files: 
+2023-11-27 01:14:28,488 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-27 01:14:28,488 [INFO] - Etherscan scraping script execution completed.
+2023-11-27 01:14:28,488 [INFO] - Function fetch_and_update_etherscan executed in 20.44 seconds
+2023-11-27 01:14:29,290 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-27 01:14:29,290 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-27 01:14:32,935 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:33,267 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-27 01:14:33,268 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-27 01:14:33,347 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:33,349 [ERROR] - Failed to retrieve latest block number for token: AXIE
+2023-11-27 01:14:33,987 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:33,988 [ERROR] - Failed to retrieve latest block number for token: APE
+2023-11-27 01:14:34,029 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:34,043 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:34,044 [ERROR] - Failed to retrieve latest block number for token: APE
+2023-11-27 01:14:34,088 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:34,211 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:34,364 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:34,402 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:34,458 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:34,459 [ERROR] - Failed to retrieve latest block number for token: AXIE
+2023-11-27 01:14:34,517 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:34,518 [ERROR] - Failed to retrieve latest block number for token: AXIE
+2023-11-27 01:14:34,995 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:34,995 [ERROR] - Failed to retrieve latest block number for token: GALA
+2023-11-27 01:14:35,058 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:35,096 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:35,116 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:35,189 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:35,217 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:35,218 [ERROR] - Failed to retrieve latest block number for token: AXIE
+2023-11-27 01:14:35,298 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:35,299 [ERROR] - Failed to retrieve latest block number for token: APE
+2023-11-27 01:14:35,418 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:35,464 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:35,465 [ERROR] - Failed to retrieve latest block number for token: MANA
+2023-11-27 01:14:36,369 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:36,370 [ERROR] - Failed to retrieve latest block number for token: APE
+2023-11-27 01:14:36,378 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:36,436 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:36,437 [ERROR] - Failed to retrieve latest block number for token: APE
+2023-11-27 01:14:36,646 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-27 01:14:36,647 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-27 01:14:37,386 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:38,647 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:38,648 [ERROR] - Failed to retrieve latest block number for token: APE
+2023-11-27 01:14:39,378 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-27 01:14:39,379 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-27 01:14:39,533 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-27 01:14:39,533 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-27 01:14:39,576 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:39,835 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-27 01:14:39,836 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-27 01:14:39,847 [ERROR] - Invalid response format or missing data in response: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}
+2023-11-27 01:14:39,848 [ERROR] - Failed to retrieve latest block number for token: GALA
+2023-11-27 01:14:49,754 [INFO] - Created files: 
+2023-11-27 01:14:49,754 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-27 01:14:49,754 [INFO] - Etherscan scraping script execution completed.
+2023-11-27 01:14:49,754 [INFO] - Function fetch_and_update_etherscan executed in 13.88 seconds
+2023-11-27 01:14:56,933 [INFO] - Created files: 
+2023-11-27 01:14:56,934 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-27 01:14:56,934 [INFO] - Etherscan scraping script execution completed.
+2023-11-27 01:14:56,934 [INFO] - Function fetch_and_update_etherscan executed in 20.11 seconds
+2023-11-27 23:22:15,713 [INFO] - Created files: 
+2023-11-27 23:22:15,713 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-27 23:22:15,713 [INFO] - Etherscan scraping script execution completed.
+2023-11-27 23:22:15,713 [INFO] - Function fetch_and_update_etherscan executed in 21.43 seconds

logs/streamlit/front.log

@@ -0,0 +1,178 @@
+2023-11-27 01:13:54,065 [INFO] - Streamlit app has started
+2023-11-27 01:13:54,506 [INFO] - Streamlit app has started
+2023-11-27 01:13:54,509 [INFO] - Streamlit app has started
+2023-11-27 01:13:54,822 [INFO] - CMC scraping completed
+2023-11-27 01:13:55,142 [INFO] - CMC scraping completed
+2023-11-27 01:13:55,205 [INFO] - CMC scraping completed
+2023-11-27 01:13:55,499 [INFO] - Streamlit app has started
+2023-11-27 01:13:56,005 [INFO] - CMC scraping completed
+2023-11-27 01:14:00,500 [INFO] - Streamlit app has started
+2023-11-27 01:14:01,263 [INFO] - CMC scraping completed
+2023-11-27 01:14:01,435 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:01,500 [INFO] - Streamlit app has started
+2023-11-27 01:14:01,991 [INFO] - CMC scraping completed
+2023-11-27 01:14:02,347 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:02,409 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:02,444 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:07,733 [INFO] - Streamlit app has started
+2023-11-27 01:14:08,262 [INFO] - CMC scraping completed
+2023-11-27 01:14:16,675 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:19,051 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:25,491 [INFO] - Streamlit app has started
+2023-11-27 01:14:26,011 [INFO] - CMC scraping completed
+2023-11-27 01:14:27,490 [INFO] - Streamlit app has started
+2023-11-27 01:14:27,540 [INFO] - Streamlit app has started
+2023-11-27 01:14:28,047 [INFO] - CMC scraping completed
+2023-11-27 01:14:28,170 [INFO] - CMC scraping completed
+2023-11-27 01:14:28,488 [INFO] - Streamlit app has started
+2023-11-27 01:14:28,532 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:29,028 [INFO] - CMC scraping completed
+2023-11-27 01:14:29,332 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:29,489 [INFO] - Streamlit app has started
+2023-11-27 01:14:29,502 [INFO] - Streamlit app has started
+2023-11-27 01:14:30,050 [INFO] - CMC scraping completed
+2023-11-27 01:14:30,212 [INFO] - CMC scraping completed
+2023-11-27 01:14:31,489 [INFO] - Streamlit app has started
+2023-11-27 01:14:31,502 [INFO] - Streamlit app has started
+2023-11-27 01:14:32,060 [INFO] - CMC scraping completed
+2023-11-27 01:14:32,171 [INFO] - CMC scraping completed
+2023-11-27 01:14:32,490 [INFO] - Streamlit app has started
+2023-11-27 01:14:32,502 [INFO] - Streamlit app has started
+2023-11-27 01:14:32,513 [INFO] - Streamlit app has started
+2023-11-27 01:14:32,530 [INFO] - Streamlit app has started
+2023-11-27 01:14:32,550 [INFO] - Streamlit app has started
+2023-11-27 01:14:33,150 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:33,453 [INFO] - CMC scraping completed
+2023-11-27 01:14:33,453 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:33,455 [INFO] - CMC scraping completed
+2023-11-27 01:14:33,649 [INFO] - CMC scraping completed
+2023-11-27 01:14:33,688 [INFO] - Streamlit app has started
+2023-11-27 01:14:33,702 [INFO] - Streamlit app has started
+2023-11-27 01:14:33,815 [INFO] - CMC scraping completed
+2023-11-27 01:14:33,819 [INFO] - CMC scraping completed
+2023-11-27 01:14:34,143 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:34,176 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:34,295 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:34,338 [INFO] - CMC scraping completed
+2023-11-27 01:14:34,423 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:34,457 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:34,467 [INFO] - CMC scraping completed
+2023-11-27 01:14:34,489 [INFO] - Streamlit app has started
+2023-11-27 01:14:35,000 [INFO] - CMC scraping completed
+2023-11-27 01:14:35,105 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:35,148 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:35,169 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:35,231 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:35,459 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:35,488 [INFO] - Streamlit app has started
+2023-11-27 01:14:35,501 [INFO] - Streamlit app has started
+2023-11-27 01:14:36,109 [INFO] - CMC scraping completed
+2023-11-27 01:14:36,174 [INFO] - CMC scraping completed
+2023-11-27 01:14:36,417 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:36,488 [INFO] - Streamlit app has started
+2023-11-27 01:14:36,714 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:37,016 [INFO] - CMC scraping completed
+2023-11-27 01:14:37,429 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:37,491 [INFO] - Streamlit app has started
+2023-11-27 01:14:37,538 [INFO] - Streamlit app has started
+2023-11-27 01:14:37,547 [INFO] - Streamlit app has started
+2023-11-27 01:14:38,138 [INFO] - CMC scraping completed
+2023-11-27 01:14:38,314 [INFO] - CMC scraping completed
+2023-11-27 01:14:38,336 [INFO] - CMC scraping completed
+2023-11-27 01:14:39,425 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:39,575 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:39,616 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:39,886 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:49,790 [INFO] - Etherscan scraping completed
+2023-11-27 01:14:56,975 [INFO] - Etherscan scraping completed
+2023-11-27 01:18:43,301 [INFO] - CMC scraping completed
+2023-11-27 01:18:43,600 [INFO] - CMC scraping completed
+2023-11-27 01:18:43,662 [INFO] - CMC scraping completed
+2023-11-27 01:18:44,413 [INFO] - CMC scraping completed
+2023-11-27 01:18:49,681 [INFO] - CMC scraping completed
+2023-11-27 01:18:50,404 [INFO] - CMC scraping completed
+2023-11-27 01:18:56,671 [INFO] - CMC scraping completed
+2023-11-27 01:19:14,447 [INFO] - CMC scraping completed
+2023-11-27 01:19:16,469 [INFO] - CMC scraping completed
+2023-11-27 01:19:16,665 [INFO] - CMC scraping completed
+2023-11-27 01:19:17,431 [INFO] - CMC scraping completed
+2023-11-27 01:19:18,492 [INFO] - CMC scraping completed
+2023-11-27 01:19:18,642 [INFO] - CMC scraping completed
+2023-11-27 01:19:20,475 [INFO] - CMC scraping completed
+2023-11-27 01:19:20,583 [INFO] - CMC scraping completed
+2023-11-27 01:19:21,955 [INFO] - CMC scraping completed
+2023-11-27 01:19:21,961 [INFO] - CMC scraping completed
+2023-11-27 01:19:22,091 [INFO] - CMC scraping completed
+2023-11-27 01:19:22,277 [INFO] - CMC scraping completed
+2023-11-27 01:19:22,290 [INFO] - CMC scraping completed
+2023-11-27 01:19:22,761 [INFO] - CMC scraping completed
+2023-11-27 01:19:22,892 [INFO] - CMC scraping completed
+2023-11-27 01:19:23,416 [INFO] - CMC scraping completed
+2023-11-27 01:19:24,545 [INFO] - CMC scraping completed
+2023-11-27 01:19:24,589 [INFO] - CMC scraping completed
+2023-11-27 01:19:25,414 [INFO] - CMC scraping completed
+2023-11-27 01:19:26,562 [INFO] - CMC scraping completed
+2023-11-27 01:19:26,763 [INFO] - CMC scraping completed
+2023-11-27 01:19:27,054 [INFO] - CMC scraping completed
+2023-11-27 01:23:32,015 [INFO] - CMC scraping completed
+2023-11-27 01:23:32,104 [INFO] - CMC scraping completed
+2023-11-27 01:23:32,132 [INFO] - CMC scraping completed
+2023-11-27 01:23:32,822 [INFO] - CMC scraping completed
+2023-11-27 01:23:39,263 [INFO] - CMC scraping completed
+2023-11-27 01:23:39,268 [INFO] - CMC scraping completed
+2023-11-27 01:23:45,177 [INFO] - CMC scraping completed
+2023-11-27 01:24:02,981 [INFO] - CMC scraping completed
+2023-11-27 01:24:05,187 [INFO] - CMC scraping completed
+2023-11-27 01:24:05,310 [INFO] - CMC scraping completed
+2023-11-27 01:24:06,910 [INFO] - CMC scraping completed
+2023-11-27 01:24:07,347 [INFO] - CMC scraping completed
+2023-11-27 01:24:08,894 [INFO] - CMC scraping completed
+2023-11-27 01:24:09,011 [INFO] - CMC scraping completed
+2023-11-27 01:24:10,421 [INFO] - CMC scraping completed
+2023-11-27 01:24:10,653 [INFO] - CMC scraping completed
+2023-11-27 01:24:10,723 [INFO] - CMC scraping completed
+2023-11-27 01:24:10,837 [INFO] - CMC scraping completed
+2023-11-27 01:24:11,319 [INFO] - CMC scraping completed
+2023-11-27 01:24:11,339 [INFO] - CMC scraping completed
+2023-11-27 01:24:11,819 [INFO] - CMC scraping completed
+2023-11-27 01:24:12,989 [INFO] - CMC scraping completed
+2023-11-27 01:24:14,094 [INFO] - CMC scraping completed
+2023-11-27 01:24:14,959 [INFO] - CMC scraping completed
+2023-11-27 01:24:15,177 [INFO] - CMC scraping completed
+2023-11-27 01:24:15,537 [INFO] - CMC scraping completed
+2023-11-27 01:24:15,737 [INFO] - CMC scraping completed
+2023-11-27 01:24:15,777 [INFO] - CMC scraping completed
+2023-11-27 01:24:18,023 [INFO] - CMC scraping completed
+2023-11-27 23:13:14,736 [INFO] - Streamlit app has started
+2023-11-27 23:13:30,823 [INFO] - Streamlit app has started
+2023-11-27 23:13:49,122 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,305 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,320 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,328 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,351 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,355 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,366 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,399 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,414 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,443 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,472 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,502 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,558 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,611 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,647 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,674 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,688 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,724 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,733 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,782 [INFO] - Streamlit app has started
+2023-11-27 23:13:56,787 [INFO] - Streamlit app has started
+2023-11-27 23:13:57,217 [INFO] - Streamlit app has started
+2023-11-27 23:13:58,139 [INFO] - Streamlit app has started
+2023-11-27 23:13:58,148 [INFO] - Streamlit app has started
+2023-11-27 23:14:00,124 [INFO] - Streamlit app has started
+2023-11-27 23:14:01,129 [INFO] - Streamlit app has started
+2023-11-27 23:14:07,138 [INFO] - Streamlit app has started
+2023-11-27 23:21:53,901 [INFO] - Streamlit app has started
+2023-11-27 23:21:54,811 [INFO] - CMC scraping completed
+2023-11-27 23:22:15,775 [INFO] - Etherscan scraping completed
+2023-11-27 23:26:43,359 [INFO] - CMC scraping completed

output/{transactions_ILV.csv → cmc_data_v1_cryptocurrency_listings_latest_100.json}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f5ce31d9e8d9c7b39bf1f73c2e05ac655f652ebffca121a6a662a0a89eaa62c9
-size 5552703
+oid sha256:f6565d58fdcb4f22b4ba93cb879ca3c74d71096a5b1ed911e625c35ae56432a0
+size 119080

output/top_100_update.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ff89b933c5ee4694a0ec72fe7660677ed15012d95a0d6184767d05eb33fd397
-size 16258
+oid sha256:767a255420ffd1a3e5480673610f1ca21a10b9c2a9f402d8a421212f5e63270e
+size 129574

output/transactions_APE.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c6094c453a4ae217cd7e6334ad0b92880e042d698db8b19978af61f42ceda1f
-size 25981544
+oid sha256:1fe609b08835b469af3f98a03959a75ecaaa119d50d222bd95981117cce95a56
+size 3430047

output/transactions_AXIE.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21eddd1decbb2f70f6fe9102cf81f0a4309c10d838bc9a172255ff70a2461cb8
-size 7599371
+oid sha256:497efc710a52dfb17f9dacc50500293cde3450644a84069fbdd4c1cae8462b37
+size 1586211

output/transactions_GALA.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd264d44fff732f21b170dcd839968ecb0408fba89cc6a993fa0da8f20fa8e05
-size 32066355
+oid sha256:dbdcc912f4193543018955be769a5418e98ab8dd3ebfaec7e47b290ac7c74f5d
+size 4094956

output/transactions_MANA.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8162afa63a588a222422d06a1f93508f92d225d124915c6ea51d5d051d4db1e
-size 12039331
+oid sha256:d530a0dd8a8f62f11f8c912b0f90cccc849349fa743873d05fbc5c14254fa84b
+size 2223053

output/transactions_PET.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa42451d8ab6696d44d5b648754791e1462f630de79439c580b8e92be5f016df
+oid sha256:1ffde90e51dd3bedbe01ab04e38ea840fd7d9c01094c6f0b6d4ee3779ed3e09c
 size 885

output/transactions_WEAOPON.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:35e366dc8930a78bd4f37409447da3c6b7f53f3b6a699c89a4c9d9d5740622f5
-size 537
+oid sha256:beac438541c99c972ae4322faae5e06a27171e15bb73436849aeed530e80b4da
+size 6082

requirements.txt

@@ -9,4 +9,5 @@ python-dotenv
 streamlit
 requests
 plotly
-nbformat
+nbformat
+schedule

ressources/dict_influencers_addr.json

@@ -0,0 +1,4 @@
+{"alexBecker":"0x4d224452801aced8b2f0aebe155379bb5d594381",
+"elliotrades":"0xc02f318365582557a482eb5d1834dfd7af4a3f59",
+"jrny":"0x08c1ae7e46d4a13b766566033b5c47c735e19f6f"
+}

ressources/dict_tokens_addr.json

@@ -0,0 +1,7 @@
+{"APE":"0x4d224452801aced8b2f0aebe155379bb5d594381",
+"AXIE":"0xbb0e17ef65f82ab018d8edd776e8dd940327b28b",
+"GALA":"0xd1d2Eb1B1e90B638588728b4130137D262C87cae",
+"MANA":"0x0f5d2fb29fb7d3cfee444a200298f468908cc942",
+"PET":"0xb870679a7fa65b924026f496de7f27c1dd0e5c5f",
+"WEAOPON":"0x0b8057c3cc676c329c25c1d0cd27776efa73762d"
+}

scrap_data_CMC.py → scrap_data_cmc.py

@@ -14,7 +14,7 @@ load_dotenv()
 
 url = os.getenv("URL_CMC")
 endpoints = ["v1/cryptocurrency/listings/latest",
-             "/v1/cryptocurrency/trending/latest",
+             #"/v1/cryptocurrency/trending/latest",
              ]
 start = "1"
 stop = "100"
@@ -64,7 +64,7 @@ df["percent_tokens_circulation"] = np.round((df["circulating_supply"]/df["total_
 # merge dataframe
 df = df.join(quote_df)
 df["last_updated"] = pd.to_datetime(df["last_updated"])
-#df.to_csv(f"output/top_{stop}_update.csv")
+df.to_csv(f"output/top_{stop}_update.csv")
 
 #-------------------------------------save data----------------------------------
 
@@ -85,4 +85,6 @@ else:
     df.to_csv(output_file, index=False)
 logging.info("Script execution completed.")
 
-#-------------------------------------end----------------------------------
+
+
+#-------------------------------------end----------------------------------

scrap_data_etherscan.py

@@ -3,7 +3,7 @@ import time
 import pandas as pd
 import json
 import os
-from utils.functions import update_and_save_csv
+from utils.scrap import update_and_save_csv
 
 # Create output folder
 if not os.path.exists("output"):

utils/scrap.py

@@ -0,0 +1,198 @@
+# ---------------------- Library Imports ----------------------
+import time
+import os
+import json
+import pandas as pd
+import numpy as np
+import logging
+import requests
+from dotenv import load_dotenv
+from requests import Session
+
+# ---------------------- Environment Variables ----------------------
+
+load_dotenv()
+# Etherscan API
+url_etherscan = os.getenv("URL_ETHERSCAN")
+api_key_etherscan = os.getenv("API_KEY_ETHERSCAN")
+
+# CoinMarketCap API
+url_cmc = os.getenv("URL_CMC")
+api_key_cmc = os.getenv("API_KEY_CMC")
+
+# Logging
+log_folder = os.getenv("LOG_FOLDER")
+os.makedirs(log_folder, exist_ok=True)
+log_file = os.path.join(log_folder, "scrapping.log")
+log_format = "%(asctime)s [%(levelname)s] - %(message)s"
+logging.basicConfig(filename=log_file, level=logging.INFO, format=log_format)
+
+# Load the JSON file into a dictionary
+with open("ressources/dict_tokens_addr.json", "r") as file:
+    dict_addresses = json.load(file)
+
+L_created = []
+L_updated = []
+
+# Define the number of blocks to retrieve transactions from
+n_blocks = 20000
+n_loop = n_blocks // 10_000
+
+# ---------------------- Processing ----------------------
+
+# Helper function for logging execution time
+def log_execution_time(func):
+    def wrapper(*args, **kwargs):
+        start_time = time.time()
+        result = func(*args, **kwargs)
+        end_time = time.time()
+        logging.info(f"Function {func.__name__} executed in {end_time - start_time:.2f} seconds")
+        return result
+    return wrapper
+
+
+@log_execution_time
+# function 1: Fetch and Update Etherscan Data
+def fetch_and_update_etherscan():
+    for tokenSymbol, contractAddr in dict_addresses.items():
+        file = f"output/transactions_{tokenSymbol}.csv"
+        if not os.path.exists(file):
+            L_created.append(file)
+            df_transactions = get_coin_data(contractAddr, n_loop)
+            df_transactions_no_dup = df_transactions.drop(["confirmations", "input"], axis=1).drop_duplicates(subset="hash")
+            df_transactions_no_dup.to_csv(file, sep=",", index=False)
+        else:
+            L_updated.append(file)
+            df_temp = pd.read_csv(file, sep=",")
+            df_temp = df_temp.sort_values("blockNumber", ascending=False)
+            start_block = df_temp["blockNumber"].iloc[0]
+
+            # Retrieve latest block number and calculate the difference
+            latest_block_number, diff = latest_block(start_block)
+            if latest_block_number is None:
+                logging.error(f"Failed to retrieve latest block number for token: {tokenSymbol}")
+                continue  # Skip to the next token if the latest block number could not be retrieved
+
+            n_loop_to_concat = (diff // 10000) + 1
+            df_transactions = get_coin_data(contractAddr, n_loop_to_concat)
+            df_latest = pd.concat([df_transactions, df_temp]).drop(["confirmations", "input"], axis=1)
+            df_latest_no_dup = df_latest.drop_duplicates(subset="hash")
+            df_latest_no_dup.loc[:, "blockNumber"] = df_latest_no_dup["blockNumber"].astype(int)
+            df_latest_no_dup = df_latest_no_dup.sort_values(by="blockNumber")
+            df_latest_no_dup.to_csv(file, sep=",", index=False)
+
+    logging.info("Created files: " + ", ".join(L_created))
+    logging.info("Updated files: " + ", ".join(L_updated))
+    logging.info("Script execution completed.")
+
+# Helper function to get latest block number
+def latest_block(start_block=None):
+    params = {
+        "module": "proxy",
+        "action": "eth_blockNumber",
+        "apikey": api_key_etherscan
+    }
+    response = requests.get(url_etherscan, params=params)
+    if response.status_code == 200:
+        try:
+            latest_block_number = int(response.json()["result"], 16)
+            if start_block is not None:
+                return latest_block_number, latest_block_number - start_block
+            return latest_block_number
+        except (ValueError, KeyError):
+            logging.error(f"Invalid response format or missing data in response: {response.json()}")
+            return None, None
+    else:
+        logging.error(f"API call failed with status code {response.status_code}: {response.json()}")
+        return None, None
+
+def get_coin_data(contractAddr, n):
+    latest_block_number = latest_block()
+    if latest_block_number is None:
+        logging.error(f"Could not retrieve latest block number for contract address {contractAddr}")
+        return pd.DataFrame()  # Return an empty DataFrame
+
+    df_transactions = pd.DataFrame()
+    transactions_per_call = 10_000
+    for i in range(n):
+        start_block = latest_block_number - (n - i) * transactions_per_call
+        end_block = latest_block_number - (n - 1 - i) * transactions_per_call
+        params = {
+            "module": "account",
+            "action": "tokentx",
+            "contractaddress": contractAddr,
+            "startblock": start_block,
+            "endblock": end_block,
+            "sort": "asc",
+            "apikey": api_key_etherscan
+        }
+        response = requests.get(url_etherscan, params=params)
+        transactions = response.json().get("result", [])
+        
+        # Check if transactions is a list of dictionaries
+        if not isinstance(transactions, list) or not all(isinstance(item, dict) for item in transactions):
+            logging.error(f"Invalid data format for transactions: {transactions}")
+            continue  # Skip this iteration if transactions data is invalid
+
+        df_temp = pd.DataFrame(transactions)
+        if not df_temp.empty:
+            df_transactions = pd.concat([df_transactions, df_temp])
+        time.sleep(1)
+    
+    if 'timeStamp' in df_transactions:
+        df_transactions['timeStamp'] = pd.to_datetime(df_transactions['timeStamp'].astype(int), unit='s')
+    else:
+        logging.error("'timeStamp' key not found in the response data.")
+        return pd.DataFrame()  # Return an empty DataFrame if key is missing
+    df_transactions['value'] = df_transactions['value'].astype(float) / 1e18
+    return df_transactions
+
+# function 2: Fetch and Process CMC Data
+@log_execution_time
+def fetch_and_process_cmc_data():
+    session = Session()
+    session.headers.update({
+        'Accepts': 'application/json',
+        'X-CMC_PRO_API_KEY': api_key_cmc,
+    })
+    parameters = {
+        'start': '1',
+        'limit': '100',
+        'convert': 'USD'
+    }
+
+    for endpoint in ["v1/cryptocurrency/listings/latest"]:
+        target = f"{url_cmc}/{endpoint}"
+        try:
+            response = session.get(target, params=parameters)
+            data = json.loads(response.text)
+            with open(f'output/cmc_data_{endpoint.replace("/", "_")}_100.json', 'w') as f:
+                json.dump(data, f)
+            process_cmc_data(data, '100')
+        except (ConnectionError, Timeout, TooManyRedirects) as e:
+            logging.error(f"Error while fetching data from {target}: {e}")
+
+def process_cmc_data(data, stop):
+    df = pd.DataFrame(data["data"])[["name", "symbol", "circulating_supply", "total_supply", "quote"]]
+    quote_df = pd.json_normalize(df['quote'].apply(lambda x: x['USD']))[["price", "percent_change_24h", "percent_change_7d", "percent_change_90d", "market_cap", "fully_diluted_market_cap", "last_updated"]]
+    df = df.drop("quote", axis=1)
+    df["percent_tokens_circulation"] = np.round((df["circulating_supply"] / df["total_supply"]) * 100, 1)
+    df = df.join(quote_df)
+    df["last_updated"] = pd.to_datetime(df["last_updated"])
+    save_cmc_data(df, stop)
+
+def save_cmc_data(df, stop):
+    output_file = f"output/top_{stop}_update.csv"
+    if os.path.isfile(output_file):
+        existing_data = pd.read_csv(output_file)
+        updated_data = pd.concat([existing_data, df], axis=0, ignore_index=True)
+        updated_data.drop_duplicates(subset=["symbol", "last_updated"], inplace=True)
+        updated_data.to_csv(output_file, index=False)
+    else:
+        df.to_csv(output_file, index=False)
+    logging.info("CMC data script execution completed.")
+
+# ---------------------- Execution ----------------------
+if __name__ == "__main__":
+    fetch_and_update_etherscan()
+    fetch_and_process_cmc_data()

utils/scrap_cmc.py

@@ -0,0 +1,82 @@
+# ---------------------- Library Imports ----------------------
+import os
+import json
+import pandas as pd
+import numpy as np
+import logging
+import requests
+from dotenv import load_dotenv
+from requests import Session
+from requests.exceptions import ConnectionError, Timeout, TooManyRedirects
+import time 
+
+# ---------------------- Environment Variables ----------------------
+load_dotenv()
+url_cmc = os.getenv("URL_CMC")
+api_key_cmc = os.getenv("API_KEY_CMC")
+log_folder = os.getenv("LOG_FOLDER")
+os.makedirs(log_folder, exist_ok=True)
+log_file = os.path.join(log_folder, "cmc_scrapping.log")
+log_format = "%(asctime)s [%(levelname)s] - %(message)s"
+logging.basicConfig(filename=log_file, level=logging.INFO, format=log_format)
+
+# ---------------------- Helper Functions ----------------------
+
+def log_execution_time(func):
+    def wrapper(*args, **kwargs):
+        start_time = time.time()
+        result = func(*args, **kwargs)
+        end_time = time.time()
+        logging.info(f"Function {func.__name__} executed in {end_time - start_time:.2f} seconds")
+        return result
+    return wrapper
+
+def process_cmc_data(data, stop):
+    df = pd.DataFrame(data["data"])[["name", "symbol", "circulating_supply", "total_supply", "quote"]]
+    quote_df = pd.json_normalize(df['quote'].apply(lambda x: x['USD']))[["price", "percent_change_24h", "percent_change_7d", "percent_change_90d", "market_cap", "fully_diluted_market_cap", "last_updated"]]
+    df = df.drop("quote", axis=1)
+    df["percent_tokens_circulation"] = np.round((df["circulating_supply"] / df["total_supply"]) * 100, 1)
+    df = df.join(quote_df)
+    df["last_updated"] = pd.to_datetime(df["last_updated"])
+    save_cmc_data(df, stop)
+
+def save_cmc_data(df, stop):
+    output_file = f"output/top_{stop}_update.csv"
+    if os.path.isfile(output_file):
+        existing_data = pd.read_csv(output_file)
+        updated_data = pd.concat([existing_data, df], axis=0, ignore_index=True)
+        updated_data.drop_duplicates(subset=["symbol", "last_updated"], inplace=True)
+        updated_data.to_csv(output_file, index=False)
+    else:
+        df.to_csv(output_file, index=False)
+    logging.info("CMC data script execution completed.")
+
+# ---------------------- CMC Scraping Function ----------------------
+
+@log_execution_time
+def fetch_and_process_cmc_data():
+    session = Session()
+    session.headers.update({
+        'Accepts': 'application/json',
+        'X-CMC_PRO_API_KEY': api_key_cmc,
+    })
+    parameters = {
+        'start': '1',
+        'limit': '100',
+        'convert': 'USD'
+    }
+
+    for endpoint in ["v1/cryptocurrency/listings/latest"]:
+        target = f"{url_cmc}/{endpoint}"
+        try:
+            response = session.get(target, params=parameters)
+            data = json.loads(response.text)
+            with open(f'output/cmc_data_{endpoint.replace("/", "_")}_100.json', 'w') as f:
+                json.dump(data, f)
+            process_cmc_data(data, '100')
+        except (ConnectionError, Timeout, TooManyRedirects) as e:
+            logging.error(f"Error while fetching data from {target}: {e}")
+
+# ---------------------- Execution ----------------------
+if __name__ == "__main__":
+    fetch_and_process_cmc_data()

utils/scrap_etherscan.py

@@ -0,0 +1,134 @@
+# ---------------------- Library Imports ----------------------
+import time
+import os
+import json
+import pandas as pd
+import logging
+import requests
+from dotenv import load_dotenv
+
+# ---------------------- Environment Variables ----------------------
+load_dotenv()
+url_etherscan = os.getenv("URL_ETHERSCAN")
+api_key_etherscan = os.getenv("API_KEY_ETHERSCAN")
+log_folder = os.getenv("LOG_FOLDER")
+os.makedirs(log_folder, exist_ok=True)
+log_file = os.path.join(log_folder, "etherscan_scrap.log")
+log_format = "%(asctime)s [%(levelname)s] - %(message)s"
+logging.basicConfig(filename=log_file, level=logging.INFO, format=log_format)
+
+# Load the JSON file into a dictionary
+with open("ressources/dict_tokens_addr.json", "r") as file:
+    dict_addresses = json.load(file)
+
+L_created = []
+L_updated = []
+n_blocks = 20000
+n_loop = n_blocks // 10_000
+
+# ---------------------- Helper Functions ----------------------
+def log_execution_time(func):
+    def wrapper(*args, **kwargs):
+        start_time = time.time()
+        result = func(*args, **kwargs)
+        end_time = time.time()
+        logging.info(f"Function {func.__name__} executed in {end_time - start_time:.2f} seconds")
+        return result
+    return wrapper
+
+def latest_block(start_block=None):
+    params = {
+        "module": "proxy",
+        "action": "eth_blockNumber",
+        "apikey": api_key_etherscan
+    }
+    response = requests.get(url_etherscan, params=params)
+    if response.status_code == 200:
+        try:
+            latest_block_number = int(response.json()["result"], 16)
+            if start_block is not None:
+                return latest_block_number, latest_block_number - start_block
+            return latest_block_number
+        except (ValueError, KeyError):
+            logging.error(f"Invalid response format or missing data in response: {response.json()}")
+            return None, None
+    else:
+        logging.error(f"API call failed with status code {response.status_code}: {response.json()}")
+        return None, None
+
+def get_coin_data(contractAddr, n):
+    latest_block_number = latest_block()
+    if latest_block_number is None:
+        logging.error(f"Could not retrieve latest block number for contract address {contractAddr}")
+        return pd.DataFrame()  # Return an empty DataFrame
+
+    df_transactions = pd.DataFrame()
+    transactions_per_call = 10_000
+    for i in range(n):
+        start_block = latest_block_number - (n - i) * transactions_per_call
+        end_block = latest_block_number - (n - 1 - i) * transactions_per_call
+        params = {
+            "module": "account",
+            "action": "tokentx",
+            "contractaddress": contractAddr,
+            "startblock": start_block,
+            "endblock": end_block,
+            "sort": "asc",
+            "apikey": api_key_etherscan
+        }
+        response = requests.get(url_etherscan, params=params)
+        transactions = response.json().get("result", [])
+        
+        if not isinstance(transactions, list) or not all(isinstance(item, dict) for item in transactions):
+            logging.error(f"Invalid data format for transactions: {transactions}")
+            continue  # Skip this iteration if transactions data is invalid
+
+        df_temp = pd.DataFrame(transactions)
+        if not df_temp.empty:
+            df_transactions = pd.concat([df_transactions, df_temp])
+        time.sleep(1)
+    
+    if 'timeStamp' in df_transactions:
+        df_transactions['timeStamp'] = pd.to_datetime(df_transactions['timeStamp'].astype(int), unit='s')
+    else:
+        logging.error("'timeStamp' key not found in the response data.")
+        return pd.DataFrame()  # Return an empty DataFrame if key is missing
+    df_transactions['value'] = df_transactions['value'].astype(float) / 1e18
+    return df_transactions
+
+# ---------------------- Main Function ----------------------
+@log_execution_time
+def fetch_and_update_etherscan():
+    for tokenSymbol, contractAddr in dict_addresses.items():
+        file = f"output/transactions_{tokenSymbol}.csv"
+        if not os.path.exists(file):
+            L_created.append(file)
+            df_transactions = get_coin_data(contractAddr, n_loop)
+            df_transactions_no_dup = df_transactions.drop(["confirmations", "input"], axis=1).drop_duplicates(subset="hash")
+            df_transactions_no_dup.to_csv(file, sep=",", index=False)
+        else:
+            L_updated.append(file)
+            df_temp = pd.read_csv(file, sep=",")
+            df_temp = df_temp.sort_values("blockNumber", ascending=False)
+            start_block = df_temp["blockNumber"].iloc[0]
+
+            latest_block_number, diff = latest_block(start_block)
+            if latest_block_number is None:
+                logging.error(f"Failed to retrieve latest block number for token: {tokenSymbol}")
+                continue
+
+            n_loop_to_concat = (diff // 10000) + 1
+            df_transactions = get_coin_data(contractAddr, n_loop_to_concat)
+            df_latest = pd.concat([df_transactions, df_temp]).drop(["confirmations", "input"], axis=1)
+            df_latest_no_dup = df_latest.drop_duplicates(subset="hash")
+            df_latest_no_dup.loc[:, "blockNumber"] = df_latest_no_dup["blockNumber"].astype(int)
+            df_latest_no_dup = df_latest_no_dup.sort_values(by="blockNumber")
+            df_latest_no_dup.to_csv(file, sep=",", index=False)
+
+    logging.info("Created files: " + ", ".join(L_created))
+    logging.info("Updated files: " + ", ".join(L_updated))
+    logging.info("Etherscan scraping script execution completed.")
+
+# ---------------------- Script Execution ----------------------
+if __name__ == "__main__":
+    fetch_and_update_etherscan()