Data transformation completed

.gitignore

@@ -2,3 +2,5 @@ ars/
 .env
 Artifacts/
 logs/
+__pycache__/
+model_trainer/

Dockerfile

@@ -0,0 +1,17 @@
+# Use the official Python image as a base
+FROM python:3.9-slim
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Copy the app files into the container
+COPY . .
+
+# Install required packages
+RUN pip install -r requirements.txt
+
+# Expose the port that Streamlit uses
+EXPOSE 8501
+
+# Run the Streamlit app
+CMD ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"]

anime_recommender/entity/artifact_entity.py

@@ -5,3 +5,6 @@ from typing import Optional
 class DataIngestionArtifact: 
     feature_store_anime_file_path:str
     feature_store_userrating_file_path:str
+@dataclass
+class DataTransformationArtifact:
+    merged_file_path:str

anime_recommender/entity/config_entity.py

@@ -29,3 +29,14 @@ class DataIngestionConfig:
         self.feature_store_userrating_file_path: str = os.path.join(self.data_ingestion_dir, DATA_INGESTION_FEATURE_STORE_DIR, RATING_FILE_NAME)
         self.anime_filepath: str = ANIME_FILE_PATH
         self.rating_filepath: str = RATING_FILE_PATH 
+
+class DataTransformationConfig:
+    """
+    Configuration for data transformation, including paths for transformed data and preprocessing objects.
+    """
+    def __init__(self,training_pipeline_config:TrainingPipelineConfig):
+        """
+        Initialize data transformation paths.
+        """
+        self.data_transformation_dir:str = os.path.join(training_pipeline_config.artifact_dir,DATA_TRANSFORMATION_DIR)
+        self.merged_file_path:str = os.path.join(self.data_transformation_dir,DATA_TRANSFORMATION_TRANSFORMED_DATA_DIR,MERGED_FILE_NAME)

anime_recommender/source/data_transformation.py

@@ -0,0 +1,115 @@
+import sys 
+import numpy as np
+import pandas as pd
+
+from anime_recommender.loggers.logging import logging
+from anime_recommender.exception.exception import AnimeRecommendorException
+from anime_recommender.utils.main_utils.utils import export_data_to_dataframe
+from anime_recommender.constant import *
+from anime_recommender.entity.config_entity import DataTransformationConfig
+from anime_recommender.entity.artifact_entity import DataIngestionArtifact,DataTransformationArtifact
+ 
+class DataTransformation:
+    """
+    Class for handling data transformation for energy generation models.
+    """
+    def __init__(self,data_ingestion_artifact:DataIngestionArtifact,data_transformation_config:DataTransformationConfig):
+        """
+        Initializes the DataTransformation class with the given data ingestion and configuration artifacts.
+        
+        Args:
+            data_ingestion_artifact (DataIngestionArtifact): The artifact containing ingested data paths.
+            data_transformation_config (DataTransformationConfig): Configuration object for data transformation.
+        """
+        try:
+            self.data_ingestion_artifact = data_ingestion_artifact
+            self.data_transformation_config = data_transformation_config
+        except Exception as e:
+            raise AnimeRecommendorException(e,sys)
+    
+    @staticmethod
+    def read_data(file_path)->pd.DataFrame:
+        """
+        Reads data from a CSV file.
+
+        Args:
+            file_path (str): Path to the CSV file.
+
+        Returns:
+            pd.DataFrame: The DataFrame containing the data from the CSV file. 
+        """
+        try:
+            return pd.read_csv(file_path)
+        except Exception as e:
+            raise AnimeRecommendorException(e,sys)
+    
+    @staticmethod
+    def merge_data(anime_df: pd.DataFrame, rating_df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Merges the anime and rating DataFrames on 'anime_id'.
+
+        Args:
+            anime_df (pd.DataFrame): DataFrame containing anime information.
+            rating_df (pd.DataFrame): DataFrame containing user rating information.
+
+        Returns:
+            pd.DataFrame: Merged DataFrame on 'anime_id'.
+        """
+        try:
+            merged_df = pd.merge(rating_df, anime_df, on="anime_id", how="inner")
+            logging.info(f"Shape of the Merged dataframe:{merged_df.shape}")
+            logging.info(f"Column names: {merged_df.columns}") 
+            return merged_df
+        except Exception as e:
+            raise AnimeRecommendorException(e, sys)
+
+    @staticmethod
+    def clean_filter_data(merged_df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Cleans the merged DataFrame by replacing 'UNKNOWN' with NaN, filling NaN values with median and also filters the data.
+
+        Args:
+            merged_df (pd.DataFrame): Merged DataFrame to clean and filter.
+
+        Returns:
+            pd.DataFrame: Cleaned and Filtered DataFrame with NaN values handled.
+        """
+        try:
+            merged_df['average_rating'].replace('UNKNOWN', np.nan)
+            merged_df['average_rating'] = pd.to_numeric(merged_df['average_rating'], errors='coerce')
+            merged_df['average_rating'].fillna(merged_df['average_rating'].median())
+            merged_df = merged_df[merged_df['average_rating'] > 6]
+            cols_to_drop = [  'username', 'overview', 'type', 'episodes', 'producers',
+                'licensors', 'studios', 'source',   'rank', 'popularity',
+                'favorites', 'scored by', 'members' ]
+            cleaned_df = merged_df.copy()
+            cleaned_df.drop(columns=cols_to_drop, inplace=True)
+            logging.info(f"Shape of the Merged dataframe:{cleaned_df.shape}")
+            logging.info(f"Column names: {cleaned_df.columns}")
+            logging.info(f"Preview of the merged DataFrame:\n{cleaned_df.head()}")
+            return cleaned_df
+        except Exception as e:
+            raise AnimeRecommendorException(e, sys)
+        
+    def initiate_data_transformation(self)->DataTransformationArtifact:
+        """
+        Initiates the data transformation process by reading, transforming, and saving the data.
+
+        Returns:
+            DataTransformationArtifact: The artifact containing paths to the transformed data. 
+        """
+        logging.info("Entering initiate_data_transformation method of DataTransformation class.")
+        try:  
+            anime_df = DataTransformation.read_data(self.data_ingestion_artifact.feature_store_anime_file_path)
+            rating_df = DataTransformation.read_data(self.data_ingestion_artifact.feature_store_userrating_file_path) 
+            merged_df = DataTransformation.merge_data(anime_df, rating_df)
+            transformed_df = DataTransformation.clean_filter_data(merged_df)
+
+            export_data_to_dataframe(transformed_df, self.data_transformation_config.merged_file_path)
+            data_transformation_artifact = DataTransformationArtifact( 
+                merged_file_path=self.data_transformation_config.merged_file_path
+                            )
+            
+            return data_transformation_artifact
+        except Exception as e:
+            raise AnimeRecommendorException(e,sys)

app.py

@@ -0,0 +1,232 @@
+import sys
+import pandas as pd
+import streamlit as st
+from anime_recommender.content_filtering_models import ContentBasedRecommender
+from anime_recommender.collaborative_filtering_models import CollaborativeAnimeRecommender
+from anime_recommender.popularity_based_filtering import PopularityBasedFiltering
+import joblib
+from huggingface_hub import hf_hub_download
+from datasets import load_dataset
+
+st.set_page_config(page_title="Anime Recommendation System", layout="wide")
+
+if "anime_data" not in st.session_state or "anime_user_ratings" not in st.session_state:
+    # Load datasets from Hugging Face (assuming no splits)
+    animedataset = load_dataset("krishnaveni76/Animes", split=None)
+    mergeddataset = load_dataset("krishnaveni76/Anime_UserRatings", split=None)
+
+    # Convert the dataset to Pandas DataFrame
+    st.session_state.anime_data = pd.DataFrame(animedataset["train"])
+    st.session_state.anime_user_ratings = pd.DataFrame(mergeddataset["train"]) 
+
+    
+# Access the data from session state
+anime_data = st.session_state.anime_data 
+anime_user_ratings = st.session_state.anime_user_ratings
+
+# Display dataset info
+st.write("Anime Data:")
+st.dataframe(anime_data)
+ 
+st.write("Anime User Ratings Data:")
+st.dataframe(anime_user_ratings)
+
+# Define your repository name
+repo_name = "krishnaveni76/anime-recommendation-models"
+
+# Load models
+cosine_similarity_model_path = hf_hub_download(repo_name, "cosine_similarity.pkl")
+item_based_knn_model_path = hf_hub_download(repo_name, "itembasedknn.pkl")
+user_based_knn_model_path = hf_hub_download(repo_name, "userbasedknn.pkl")
+svd_model_path = hf_hub_download(repo_name, "svd.pkl")
+
+# # Load the models into memory
+# with open(cosine_similarity_model_path, "rb") as f:
+#     cosine_similarity_model = joblib.load(f)
+
+with open(item_based_knn_model_path, "rb") as f:
+    item_based_knn_model = joblib.load(f)
+
+with open(user_based_knn_model_path, "rb") as f:
+    user_based_knn_model = joblib.load(f)
+
+with open(svd_model_path, "rb") as f:
+    svd_model = joblib.load(f)
+
+# Now you can use these models for recommendations
+print("Models loaded successfully!")
+    
+# Streamlit UI
+app_selector = st.sidebar.radio(
+    "Select App", ("Content-Based Recommender", "Collaborative Recommender", "Top Anime Recommender")
+)
+
+if app_selector == "Content-Based Recommender":
+    st.title("Content-Based Recommender System") 
+    try:
+        
+        anime_list = anime_data["name"].tolist()
+        anime_name = st.selectbox("Select an Anime", anime_list)
+
+        # Set number of recommendations
+        max_recommendations = min(len(anime_data), 100)
+        n_recommendations = st.slider("Number of Recommendations", 1, max_recommendations, 10)
+
+        # Inject custom CSS for anime name font size
+        st.markdown(
+            """
+            <style>
+            .anime-title {
+                font-size: 14px !important;
+                font-weight: bold;
+                text-align: center;
+                margin-top: 5px;
+            }
+            </style>
+            """,
+            unsafe_allow_html=True,
+        ) 
+        # Get Recommendations
+        if st.button("Get Recommendations"):
+            try:
+                recommender = ContentBasedRecommender(anime_data)
+                recommendations = recommender.get_rec_cosine(anime_name, n_recommendations=n_recommendations,model_path=cosine_similarity_model_path)
+
+                if isinstance(recommendations, str):
+                    st.warning(recommendations)
+                elif recommendations.empty:
+                    st.warning("No recommendations found.")
+                else:
+                    st.write(f"Here are the Content-based Recommendations for {anime_name}:") 
+                    cols = st.columns(5)
+                    for i, row in enumerate(recommendations.iterrows()):
+                        col = cols[i % 5]
+                        with col:
+                            st.image(row[1]['Image URL'], use_container_width=True)
+                            st.markdown(
+                                f"<div class='anime-title'>{row[1]['Anime name']}</div>",
+                                unsafe_allow_html=True,
+                            )
+                            st.caption(f"Genres: {row[1]['Genres']} | Rating: {row[1]['Rating']}") 
+            except Exception as e:
+                st.error(f"Unexpected error: {str(e)}")
+ 
+    except Exception as e:
+        st.error(f"Unexpected error: {str(e)}")
+
+elif app_selector == "Collaborative Recommender":
+    st.title("Collaborative Recommender System")
+    
+    try: 
+        
+        # Sidebar for choosing the collaborative filtering method
+        collaborative_method = st.sidebar.selectbox(
+            "Choose a collaborative filtering method:", 
+            ["SVD Collaborative Filtering", "User-Based Collaborative Filtering", "Anime-Based KNN Collaborative Filtering"]
+        )
+
+        # User input
+        if collaborative_method == "SVD Collaborative Filtering" or collaborative_method == "User-Based Collaborative Filtering": 
+            user_ids = anime_user_ratings['user_id'].unique()  # Get unique user IDs
+            user_id = st.selectbox("Select a user ID ", user_ids) 
+            n_recommendations = st.slider("Number of Recommendations:", min_value=1, max_value=50, value=10)
+        elif collaborative_method == "Anime-Based KNN Collaborative Filtering": 
+            anime_list = anime_user_ratings["name"].dropna().unique().tolist()  # Ensure no NaN values in anime names
+            anime_name = st.selectbox("Select an Anime", anime_list)
+            n_recommendations = st.slider("Number of Recommendations:", min_value=1, max_value=50, value=10)
+   
+        # Get recommendations
+        if st.button("Get Recommendations"):
+            # Load the recommender
+            recommender = CollaborativeAnimeRecommender(anime_user_ratings) 
+            if collaborative_method == "SVD Collaborative Filtering": 
+                recommendations = recommender.get_svd_recommendations(user_id, n=n_recommendations, svd_model=svd_model) 
+                # st.write(recommendations.head())
+            elif collaborative_method == "User-Based Collaborative Filtering": 
+                recommendations = recommender.get_user_based_recommendations(user_id, n_recommendations=n_recommendations, knn_user_model=user_based_knn_model)
+            elif collaborative_method == "Anime-Based KNN Collaborative Filtering":
+                if anime_name: 
+                    recommendations = recommender.get_item_based_recommendations(anime_name, n_recommendations=n_recommendations, knn_item_model=item_based_knn_model)
+                else:
+                    st.error("Invalid Anime Name. Please enter a valid anime title.")
+            
+            if isinstance(recommendations, pd.DataFrame) and not recommendations.empty:
+                if len(recommendations) < n_recommendations:
+                    st.warning(f"Only {len(recommendations)} recommendations available, fewer than the requested {n_recommendations}.")
+                st.write(f"Here are the Collaborative Recommendations:")
+                cols = st.columns(5)
+                for i, row in enumerate(recommendations.iterrows()):
+                    col = cols[i % 5]
+                    with col:
+                        st.image(row[1]['Image URL'], use_container_width=True)
+                        st.markdown(
+                            f"<div class='anime-title'>{row[1]['Anime Name']}</div>",
+                            unsafe_allow_html=True,
+                        ) 
+                        st.caption(f"Genres: {row[1]['Genres']} | Rating: {row[1]['Rating']}")
+            else:
+                st.error("No recommendations found.")
+    except Exception as e:
+        st.error(f"An error occurred: {e}")
+
+elif app_selector == "Top Anime Recommender":
+    st.title("Top Anime Recommender System")
+    
+    try:
+        # Sidebar for choosing the popularity-based filtering method
+        popularity_method = st.sidebar.selectbox(
+            "Choose a Popularity-Based Filtering method:",
+            [
+                "Popular Animes",
+                "Top Ranked Animes",
+                "Overall Top Rated Animes",
+                "Favorite Animes",
+                "Top Animes by Members",
+                "Popular Anime Among Members",
+                "Top Average Rated Animes",
+            ]
+        )
+        
+        n_recommendations = st.slider("Number of Recommendations:", min_value=1, max_value=50, value=10)
+        
+        if st.button("Get Top Anime"):
+            # Load the popularity-based recommender
+            recommender = PopularityBasedFiltering(anime_data)
+            
+            # Get recommendations based on selected method
+            if popularity_method == "Popular Animes":
+                recommendations = recommender.popular_animes(n=n_recommendations)
+            elif popularity_method == "Top Ranked Animes":
+                recommendations = recommender.top_ranked_animes(n=n_recommendations)
+            elif popularity_method == "Overall Top Rated Animes":
+                recommendations = recommender.overall_top_rated_animes(n=n_recommendations)
+            elif popularity_method == "Favorite Animes":
+                recommendations = recommender.favorite_animes(n=n_recommendations)
+            elif popularity_method == "Top Animes by Members":
+                recommendations = recommender.top_animes_members(n=n_recommendations)
+            elif popularity_method == "Popular Anime Among Members":
+                recommendations = recommender.popular_anime_among_members(n=n_recommendations)
+            elif popularity_method == "Top Average Rated Animes":
+                recommendations = recommender.top_avg_rated(n=n_recommendations)
+            else:
+                st.error("Invalid selection. Please choose a valid method.")
+                recommendations = None
+            
+            # Display recommendations
+            if isinstance(recommendations, pd.DataFrame) and not recommendations.empty:
+                st.write(f"Here are the {popularity_method}:")
+                cols = st.columns(5)
+                for i, row in recommendations.iterrows():
+                    col = cols[i % 5]
+                    with col:
+                        st.image(row['Image URL'], use_container_width=True)
+                        st.markdown(
+                            f"<div class='anime-title'>{row['Anime name']}</div>",
+                            unsafe_allow_html=True,
+                        )
+                        st.caption(f"Genres: {row['Genres']} | Rating: {row['Rating']}")
+            else:
+                st.error("No recommendations found.")
+    except Exception as e:
+        st.error(f"An error occurred: {e}")
+

run_pipeline.py

@@ -2,9 +2,9 @@ import sys
 from anime_recommender.loggers.logging import logging
 from anime_recommender.exception.exception import AnimeRecommendorException
 from anime_recommender.source.data_ingestion import DataIngestion
-from anime_recommender.entity.config_entity import TrainingPipelineConfig,DataIngestionConfig
+from anime_recommender.entity.config_entity import TrainingPipelineConfig,DataIngestionConfig,DataTransformationConfig
 # ,DataTransformationConfig,CollaborativeModelConfig,ContentBasedModelConfig
-# from anime_recommender.source.data_transformation import DataTransformation
+from anime_recommender.source.data_transformation import DataTransformation
 # from anime_recommender.source.collaborative_recommenders import CollaborativeModelTrainer
 # from anime_recommender.source.content_based_recommenders import ContentBasedModelTrainer 
 # from anime_recommender.source.popularity_based_recommenders import PopularityBasedRecommendor
@@ -19,13 +19,13 @@ if __name__ == "__main__":
         logging.info(f"Data ingestion completed.")
         print(data_ingestion_artifact)
 
-        # # Data Transformation
-        # data_transformation_config = DataTransformationConfig(training_pipeline_config)
-        # data_transformation = DataTransformation(data_ingestion_artifact,data_transformation_config)
-        # logging.info("Initiating Data Transformation.")
-        # data_transformation_artifact = data_transformation.initiate_data_transformation()
-        # logging.info("Data Transformation Completed.")
-        # print(data_transformation_artifact)
+        # Data Transformation
+        data_transformation_config = DataTransformationConfig(training_pipeline_config)
+        data_transformation = DataTransformation(data_ingestion_artifact,data_transformation_config)
+        logging.info("Initiating Data Transformation.")
+        data_transformation_artifact = data_transformation.initiate_data_transformation()
+        logging.info("Data Transformation Completed.")
+        print(data_transformation_artifact)
 
         # # Collaborative Model Training
         # collaborative_model_trainer_config = CollaborativeModelConfig(training_pipeline_config)