# ------------------------ Libraries -------------------------- import os import pandas as pd import streamlit as st import plotly.graph_objs as go import logging import subprocess import threading from dotenv import load_dotenv from requests.exceptions import ConnectionError, Timeout, TooManyRedirects import plotly.express as px import json # ------------------------ Environment Variables -------------------------- load_dotenv() log_folder = os.getenv("LOG_FOLDER") # Logging log_folder = os.getenv("LOG_STREAMLIT") os.makedirs(log_folder, exist_ok=True) log_file = os.path.join(log_folder, "front.log") log_format = "%(asctime)s [%(levelname)s] - %(message)s" logging.basicConfig(filename=log_file, level=logging.INFO, format=log_format) logging.info("Streamlit app has started") # Create output folder if it doesn't exist if not os.path.exists("output"): os.makedirs("output") #-------------------------------------back---------------------------------- def safe_read_csv(file_path, sep=','): if os.path.exists(file_path) and os.path.getsize(file_path) > 0: return pd.read_csv(file_path, sep=sep) else: logging.warning(f"File {file_path} is empty or does not exist.") return pd.DataFrame() # return an empty DataFrame # etherscan ## Load the data from the CSV files df_etherscan = pd.DataFrame() for filename in os.listdir('output'): if filename.endswith('.csv') and 'transactions_' in filename: df_temp = safe_read_csv(os.path.join('output', filename), sep=',') df_etherscan = pd.concat([df_etherscan, df_temp], ignore_index=True) # CMC ## Load cmc data df_cmc = safe_read_csv("output/top_100_update.csv", sep=',') df_cmc = df_cmc[df_cmc["last_updated"] == df_cmc["last_updated"].max()] # Global metrics about the market def load_global_metrics(): try: return pd.read_csv("output/global_metrics.csv") except FileNotFoundError: logging.warning("Global metrics file not found.") return pd.DataFrame() # Return an empty DataFrame if file is not found # Load influencers def load_influencers(): try: with open("ressources/dict_influencers_addr.json", "r") as file: return json.load(file) except Exception as e: st.error(f"Error loading influencers: {e}") return {} def create_dominance_pie_chart(df_global_metrics): # Extract BTC and ETH dominance btc_dominance = df_global_metrics['btc_dominance'].iloc[0] eth_dominance = df_global_metrics['eth_dominance'].iloc[0] # Calculate the dominance of other cryptocurrencies others_dominance = 100 - btc_dominance - eth_dominance #print(btc_dominance,eth_dominance,others_dominance) # Prepare data for pie chart dominance_data = { 'Cryptocurrency': ['BTC', 'ETH', 'Others'], 'Dominance': [btc_dominance, eth_dominance, others_dominance] } df_dominance = pd.DataFrame(dominance_data) # Create a pie chart fig = px.pie(df_dominance, values='Dominance', names='Cryptocurrency', title='Market Cap Dominance') return fig def display_greed_fear_index(): try: df = pd.read_csv('output/greed_fear_index.csv') # Prepare data for plotting time_periods = ['One Year Ago', 'One Month Ago', 'One Week Ago', 'Previous Close', 'Now'] values = [ df['fgi_oneYearAgo_value'].iloc[0], df['fgi_oneMonthAgo_value'].iloc[0], df['fgi_oneWeekAgo_value'].iloc[0], df['fgi_previousClose_value'].iloc[0], df['fgi_now_value'].iloc[0] ] labels = [ df['fgi_oneYearAgo_valueText'].iloc[0], df['fgi_oneMonthAgo_valueText'].iloc[0], df['fgi_oneWeekAgo_valueText'].iloc[0], df['fgi_previousClose_valueText'].iloc[0], df['fgi_now_valueText'].iloc[0] ] # Create a Plotly figure fig = go.Figure(data=[ go.Scatter(x=time_periods, y=values, mode='lines+markers+text', text=labels, textposition='top center') ]) # Update layout fig.update_layout( title='Fear and Greed Index Over Time', xaxis_title='Time Period', yaxis_title='Index Value', yaxis=dict(range=[0, 100]) # Fear and Greed index ranges from 0 to 100 ) # Display the figure st.plotly_chart(fig) except FileNotFoundError: st.error("Greed and Fear index data not available. Please wait for the next update cycle.") #-------------------------------------scheduler ---------------------------------- # Function to execute the scraping functions def execute_etherscan_scraping(): subprocess.call(["python", "utils/scrap_etherscan.py"]) logging.info("Etherscan scraping completed") threading.Timer(3600, execute_etherscan_scraping).start() # Balancer scrapping def execute_influencers_scraping(): subprocess.call(["python", "utils/scrap_influencers_balance.py"]) logging.info("Influencers balance scraping completed") threading.Timer(3600, execute_influencers_scraping).start() # Run every hour, for example # Function to execute the scraping functions def execute_cmc_scraping(): subprocess.call(["python", "utils/scrap_cmc.py"]) logging.info("CMC scraping completed") threading.Timer(2592000 / 9000, execute_cmc_scraping).start() # Function to execute the global metrics scraping def execute_global_metrics_scraping(): subprocess.call(["python", "utils/scrap_cmc_global_metrics.py"]) logging.info("Global metrics scraping completed") threading.Timer(2592000 / 9000, execute_influencers_scraping).start() # Run every hour, for example def execute_greed_fear_index_scraping(): subprocess.call(["python", "utils/scrap_greed_fear_index.py"]) logging.info("Greed and Fear index scraping completed") threading.Timer(3600, execute_greed_fear_index_scraping).start() # Adjust the interval as needed if "initialized" not in st.session_state: # Start the scraping threads threading.Thread(target=execute_etherscan_scraping).start() threading.Thread(target=execute_cmc_scraping).start() threading.Thread(target=execute_influencers_scraping).start() threading.Thread(target=execute_global_metrics_scraping).start() threading.Thread(target=execute_greed_fear_index_scraping).start() st.session_state["initialized"] = True #-------------------------------------streamlit ---------------------------------- # Set the title and other page configurations st.title('Crypto Analysis') st.header("Global Cryptocurrency Market Metrics") # Create two columns for the two plots col1, col2 = st.columns(2) global_metrics_df = load_global_metrics() display_greed_fear_index() st.write(global_metrics_df) with col1: # Create and display the pie chart dominance_fig = create_dominance_pie_chart(global_metrics_df) dominance_fig.update_layout( autosize=False, width=300, height=300,) st.plotly_chart(dominance_fig) with col2: # cmc selected_var = st.selectbox('Select Var', ["percent_change_24h","percent_change_7d","percent_change_90d"], index=0) # Sort the DataFrame by the 'percent_change_24h' column in ascending order df_sorted = df_cmc.sort_values(by=selected_var, ascending=False) # Select the top 10 and worst 10 rows top_10 = df_sorted.head(10) worst_10 = df_sorted.tail(10) # Combine the top and worst dataframes for plotting combined_df = pd.concat([top_10, worst_10], axis=0) max_abs_val = max(abs(combined_df[selected_var].min()), abs(combined_df[selected_var].max())) # Create a bar plot for the top 10 with a green color scale fig = go.Figure(data=[ go.Bar( x=top_10["symbol"], y=top_10[selected_var], marker_color='rgb(0,100,0)', # Green color for top 10 hovertext= "Name : "+top_10["name"].astype(str)+ '
' + selected_var + " : " + top_10["percent_tokens_circulation"].astype(str) + '
' + 'Market Cap: ' + top_10["market_cap"].astype(str) + '
' + 'Fully Diluted Market Cap: ' + top_10["fully_diluted_market_cap"].astype(str) + '
' + 'Last Updated: ' + top_10["last_updated"].astype(str), name="top_10" ) ]) # Add the worst 10 to the same plot with a red color scale fig.add_traces(go.Bar( x=worst_10["symbol"], y=worst_10[selected_var], marker_color='rgb(255,0,0)', # Red color for worst 10 hovertext="Name:"+worst_10["name"].astype(str)+ '
' + selected_var + " : " + worst_10["percent_tokens_circulation"].astype(str) + '
' + 'Market Cap: ' + worst_10["market_cap"].astype(str) + '
' + 'Fully Diluted Market Cap: ' + worst_10["fully_diluted_market_cap"].astype(str) + '
' + 'Last Updated: ' + worst_10["last_updated"].astype(str), name="worst_10" ) ) # Customize aspect fig.update_traces(marker_line_color='rgb(8,48,107)', marker_line_width=1.5, opacity=0.8) fig.update_layout(title_text=f'Top 10 and Worst 10 by {selected_var.split("_")[-1]} Percentage Change') fig.update_xaxes(categoryorder='total ascending') fig.update_layout( autosize=False, width=300, height=300, margin=dict( l=50, r=50, b=100, t=100, pad=4 ), #paper_bgcolor="LightSteelBlue", ) st.plotly_chart(fig) st.header("Deep Dive into Specific Coins") col1, col2 = st.columns(2) with col1: # etherscan selected_token = st.selectbox('Select Token', df_etherscan['tokenSymbol'].unique(), index=0) # Filter the data based on the selected token filtered_df = df_etherscan[df_etherscan['tokenSymbol'] == selected_token] # Plot the token value over time st.plotly_chart( go.Figure( data=[ go.Scatter( x=filtered_df['timeStamp'], y=filtered_df['value'], mode='lines', name='Value over time' ) ], layout=go.Layout( title='Token Value Over Time', yaxis=dict( title=f'Value ({selected_token})', ), showlegend=True, legend=go.layout.Legend(x=0, y=1.0), margin=go.layout.Margin(l=40, r=0, t=40, b=30), width=300, height=300, ) ) ) with col2: influencers = load_influencers() with st.container(): influencer_input = st.text_input("Follow this name:address", placeholder="e.g., alice:0x123...ABC") if st.button("Add Influencer"): if ":" in influencer_input: try: new_influencer_name, new_influencer_addr = influencer_input.split(":") influencers[new_influencer_name.strip()] = new_influencer_addr.strip() with open("ressources/dict_influencers_addr.json", "w") as file: json.dump(influencers, file, indent=4) st.success(f"Influencer {new_influencer_name} added") subprocess.call(["python", "utils/scrap_influencers_balance.py"]) st.success("Balance updated") except ValueError: st.error("Invalid format. Please enter as 'name:address'") else: st.error("Please enter the influencer details as 'name:address'") # Load Ether balances try: df_balances = pd.read_csv("output/influencers_balances.csv") logging.info(f"Balances uploaded, shape of dataframe is {df_balances.shape}") #st.write("DataFrame Loaded:", df_balances) # Debugging line except FileNotFoundError: st.error("Balance data not found. Please wait for the next update cycle.") df_balances = pd.DataFrame() # Inverting the influencers dictionary inverted_influencers = {v.lower(): k for k, v in influencers.items()} if not df_balances.empty: df_balances["balance"] = df_balances["balance"].astype(float) / 1e18 # Convert Wei to Ether df_balances = df_balances.rename(columns={"account": "address"}) # Ensure addresses are in the same format as in the inverted dictionary (e.g., lowercase) df_balances["address"] = df_balances["address"].str.lower() # Perform the mapping df_balances["influencer"] = df_balances["address"].map(inverted_influencers) #st.write("Mapped DataFrame:", df_balances) # Debugging line fig = px.bar(df_balances, y="influencer", x="balance",orientation="h") fig.update_layout( title='Ether Balances of Influencers', xaxis=dict( title='Balance in eth', titlefont_size=16, tickfont_size=14, )) fig.update_layout( autosize=False, width=300, height=400,) st.plotly_chart(fig) else: logging.info("DataFrame is empty") st.header("Deep Dive into Specific Coins") #-------------------------------------end ----------------------------------