Training pipeline created and completed

anime_recommender/constant/__init__.py

@@ -1,18 +1,16 @@
-import os 
 """
 Defining common constant variables for training pipeline
 """
-PIPELINE_NAME: str = "AnimeRecommendor"
+PIPELINE_NAME: str = "AnimeRecommender"
 ARTIFACT_DIR: str = "Artifacts"
 ANIME_FILE_NAME: str = "Animes.csv"
 RATING_FILE_NAME:str = "UserRatings.csv"
-MERGED_FILE_NAME:str = "Anime_UserRatings.csv"
-ZIP_FILE_PATH:str = 'datasets/archive.zip'
-DATASETS_FILE_PATH:str = "datasets"
+MERGED_FILE_NAME:str = "Anime_UserRatings.csv" 
 
 ANIME_FILE_PATH:str = "krishnaveni76/Animes"
 RATING_FILE_PATH:str = "krishnaveni76/UserRatings"
 ANIMEUSERRATINGS_FILE_PATH:str = "krishnaveni76/Anime_UserRatings"
+MODELS_FILEPATH = "krishnaveni76/anime-recommendation-models"
 
 """
 Data Ingestion related constant start with DATA_INGESTION VAR NAME
@@ -39,5 +37,4 @@ MODEL_TRAINER_USER_KNN_TRAINED_MODEL_NAME: str = "userbasedknn.pkl"
 
 MODEL_TRAINER_CON_TRAINED_MODEL_DIR:str = "content_based_recommenders"
 MODEL_TRAINER_COSINESIMILARITY_MODEL_NAME:str = "cosine_similarity.pkl" 
-
-MODEL_TRAINER_POP_TRAINED_MODEL_DIR:str = "popularity_based_recommenders" 
+ 

anime_recommender/pipelines/training_pipeline.py

@@ -0,0 +1,152 @@
+import sys
+from anime_recommender.loggers.logging import logging
+from anime_recommender.exception.exception import AnimeRecommendorException
+
+from anime_recommender.source.data_ingestion import DataIngestion
+from anime_recommender.source.data_transformation import DataTransformation
+from anime_recommender.source.collaborative_recommender import CollaborativeModelTrainer
+from anime_recommender.source.content_based_recommender import ContentBasedModelTrainer
+from anime_recommender.source.top_anime_recommenders import PopularityBasedRecommendor
+from anime_recommender.entity.config_entity import (
+    TrainingPipelineConfig,
+    DataIngestionConfig,
+    DataTransformationConfig,
+    CollaborativeModelConfig,
+    ContentBasedModelConfig,
+)
+from anime_recommender.entity.artifact_entity import (
+    DataIngestionArtifact,
+    DataTransformationArtifact,
+    CollaborativeModelArtifact,
+    ContentBasedModelArtifact,
+)
+
+class TrainingPipeline:
+    """
+    Orchestrates the entire anime recommender training pipeline, including
+    data ingestion, transformation, model training, and popularity-based recommendations.
+    """
+    def __init__(self):
+        """
+        Initialize the TrainingPipeline with required configurations.
+        """
+        self.training_pipeline_config = TrainingPipelineConfig()
+
+    def start_data_ingestion(self) -> DataIngestionArtifact:
+        """
+        Starts the data ingestion process.
+        Returns:
+            DataIngestionArtifact: Contains information about ingested data.
+        """
+        try:
+            logging.info("Initiating Data Ingestion...")
+            data_ingestion_config = DataIngestionConfig(self.training_pipeline_config)
+            data_ingestion = DataIngestion(data_ingestion_config=data_ingestion_config)
+            data_ingestion_artifact = data_ingestion.ingest_data()
+            logging.info(f"Data Ingestion completed: {data_ingestion_artifact}")
+            return data_ingestion_artifact
+        except Exception as e:
+            raise AnimeRecommendorException(e, sys)
+
+    def start_data_transformation(self, data_ingestion_artifact: DataIngestionArtifact) -> DataTransformationArtifact:
+        """
+        Starts the data transformation process.
+        Returns:
+            DataTransformationArtifact: Contains transformed data.
+        """
+        try:
+            logging.info("Initiating Data Transformation...")
+            data_transformation_config = DataTransformationConfig(self.training_pipeline_config)
+            data_transformation = DataTransformation(
+                data_ingestion_artifact=data_ingestion_artifact,
+                data_transformation_config=data_transformation_config
+            )
+            data_transformation_artifact = data_transformation.initiate_data_transformation()
+            logging.info(f"Data Transformation completed: {data_transformation_artifact}")
+            return data_transformation_artifact
+        except Exception as e:
+            raise AnimeRecommendorException(e, sys)
+
+    def start_collaborative_model_training(self, data_transformation_artifact: DataTransformationArtifact) -> CollaborativeModelArtifact:
+        """
+        Starts collaborative filtering model training.
+        Returns:
+            CollaborativeModelTrainerArtifact: Trained collaborative model artifact.
+        """
+        try:
+            logging.info("Initiating Collaborative Model Training...")
+            collaborative_model_config = CollaborativeModelConfig(self.training_pipeline_config)
+            collaborative_model_trainer = CollaborativeModelTrainer(
+                collaborative_model_trainer_config=collaborative_model_config,
+                data_transformation_artifact=data_transformation_artifact
+            )
+            collaborative_model_trainer_artifact = collaborative_model_trainer.initiate_model_trainer(model_type='user_knn')
+            logging.info(f"Collaborative Model Training completed: {collaborative_model_trainer_artifact}")
+            return collaborative_model_trainer_artifact
+        except Exception as e:
+            raise AnimeRecommendorException(e, sys)
+
+    def start_content_based_model_training(self, data_ingestion_artifact: DataIngestionArtifact) -> ContentBasedModelArtifact:
+        """
+        Starts content-based filtering model training.
+        Returns:
+            ContentBasedModelTrainerArtifact: Trained content-based model artifact.
+        """
+        try:
+            logging.info("Initiating Content-Based Model Training...")
+            content_based_model_config = ContentBasedModelConfig(self.training_pipeline_config)
+            content_based_model_trainer = ContentBasedModelTrainer(
+                content_based_model_trainer_config=content_based_model_config,
+                data_ingestion_artifact=data_ingestion_artifact
+            )
+            content_based_model_trainer_artifact = content_based_model_trainer.initiate_model_trainer()
+            logging.info(f"Content-Based Model Training completed: {content_based_model_trainer_artifact}")
+            return content_based_model_trainer_artifact
+        except Exception as e:
+            raise AnimeRecommendorException(e, sys)
+
+    def start_popularity_based_filtering(self, data_ingestion_artifact: DataIngestionArtifact):
+        """
+        Generates popularity-based recommendations.
+        """
+        try:
+            logging.info("Initiating Popularity-Based Filtering...")
+            filtering = PopularityBasedRecommendor(data_ingestion_artifact=data_ingestion_artifact)
+            recommendations = filtering.initiate_model_trainer(filter_type='popular_animes')
+            logging.info("Popularity-Based Filtering completed.")
+            return recommendations
+        except Exception as e:
+            raise AnimeRecommendorException(e, sys)
+
+    def run_pipeline(self):
+        """
+        Executes the entire training pipeline.
+        """
+        try:
+            # Data Ingestion
+            data_ingestion_artifact = self.start_data_ingestion()
+
+            # Data Transformation
+            data_transformation_artifact = self.start_data_transformation(data_ingestion_artifact)
+
+            # Collaborative Model Training
+            collaborative_model_trainer_artifact = self.start_collaborative_model_training(data_transformation_artifact)
+
+            # Content-Based Model Training
+            content_based_model_trainer_artifact = self.start_content_based_model_training(data_ingestion_artifact)
+
+            # Popularity-Based Filtering
+            popularity_recommendations = self.start_popularity_based_filtering(data_ingestion_artifact)
+
+            logging.info("Training Pipeline executed successfully.")
+        except Exception as e:
+            raise AnimeRecommendorException(e, sys)
+
+ 
+if __name__ == "__main__":
+    try:
+        pipeline = TrainingPipeline()
+        pipeline.run_pipeline()
+    except Exception as e:
+        logging.error(f"Pipeline execution failed: {str(e)}")
+        raise AnimeRecommendorException(e, sys)

app.py

@@ -1,10 +1,10 @@
-import sys
 import pandas as pd
 import streamlit as st
-from anime_recommender.content_filtering_models import ContentBasedRecommender
-from anime_recommender.collaborative_filtering_models import CollaborativeAnimeRecommender
-from anime_recommender.popularity_based_filtering import PopularityBasedFiltering
+from anime_recommender.model_trainer.content_based_modelling import ContentBasedRecommender
+from anime_recommender.model_trainer.collaborative_modelling import CollaborativeAnimeRecommender
+from anime_recommender.model_trainer.top_anime_filtering import PopularityBasedFiltering
 import joblib
+from anime_recommender.constant import *
 from huggingface_hub import hf_hub_download
 from datasets import load_dataset
 
@@ -12,14 +12,38 @@ st.set_page_config(page_title="Anime Recommendation System", layout="wide")
 
 if "anime_data" not in st.session_state or "anime_user_ratings" not in st.session_state:
     # Load datasets from Hugging Face (assuming no splits)
-    animedataset = load_dataset("krishnaveni76/Animes", split=None)
-    mergeddataset = load_dataset("krishnaveni76/Anime_UserRatings", split=None)
+    animedataset = load_dataset(ANIME_FILE_PATH, split=None)
+    mergeddataset = load_dataset(ANIMEUSERRATINGS_FILE_PATH, split=None)
 
     # Convert the dataset to Pandas DataFrame
     st.session_state.anime_data = pd.DataFrame(animedataset["train"])
     st.session_state.anime_user_ratings = pd.DataFrame(mergeddataset["train"]) 
 
-    
+# Load models only once
+if "models_loaded" not in st.session_state:
+    st.session_state.models_loaded = {}
+
+    # Define your repository name
+    repo_name = MODELS_FILEPATH
+
+    # Load models
+    st.session_state.models_loaded["cosine_similarity_model"] = hf_hub_download(repo_name, MODEL_TRAINER_COSINESIMILARITY_MODEL_NAME)
+    st.session_state.models_loaded["item_based_knn_model_path"] = hf_hub_download(repo_name, MODEL_TRAINER_ITEM_KNN_TRAINED_MODEL_NAME)
+    st.session_state.models_loaded["user_based_knn_model_path"] = hf_hub_download(repo_name, MODEL_TRAINER_USER_KNN_TRAINED_MODEL_NAME)
+    st.session_state.models_loaded["svd_model_path"] = hf_hub_download(repo_name, MODEL_TRAINER_SVD_TRAINED_MODEL_NAME)
+
+    # Load the models using joblib
+    with open(st.session_state.models_loaded["item_based_knn_model_path"], "rb") as f:
+        st.session_state.models_loaded["item_based_knn_model"] = joblib.load(f)
+
+    with open(st.session_state.models_loaded["user_based_knn_model_path"], "rb") as f:
+        st.session_state.models_loaded["user_based_knn_model"] = joblib.load(f)
+
+    with open(st.session_state.models_loaded["svd_model_path"], "rb") as f:
+        st.session_state.models_loaded["svd_model"] = joblib.load(f)
+
+    print("Models loaded successfully!")
+
 # Access the data from session state
 anime_data = st.session_state.anime_data 
 anime_user_ratings = st.session_state.anime_user_ratings
@@ -31,25 +55,30 @@ st.dataframe(anime_data)
 st.write("Anime User Ratings Data:")
 st.dataframe(anime_user_ratings)
 
-# Define your repository name
-repo_name = "krishnaveni76/anime-recommendation-models"
+# # Define your repository name
+# repo_name = "krishnaveni76/anime-recommendation-models"
+
+# # Load models
 
-# Load models
-cosine_similarity_model_path = hf_hub_download(repo_name, "cosine_similarity.pkl")
-item_based_knn_model_path = hf_hub_download(repo_name, "itembasedknn.pkl")
-user_based_knn_model_path = hf_hub_download(repo_name, "userbasedknn.pkl")
-svd_model_path = hf_hub_download(repo_name, "svd.pkl")
+# item_based_knn_model_path = hf_hub_download(repo_name, MODEL_TRAINER_ITEM_KNN_TRAINED_MODEL_NAME)
+# user_based_knn_model_path = hf_hub_download(repo_name, MODEL_TRAINER_USER_KNN_TRAINED_MODEL_NAME)
+# svd_model_path = hf_hub_download(repo_name,MODEL_TRAINER_SVD_TRAINED_MODEL_NAME)
 
-with open(item_based_knn_model_path, "rb") as f:
-    item_based_knn_model = joblib.load(f)
+# with open(item_based_knn_model_path, "rb") as f:
+#     item_based_knn_model = joblib.load(f)
 
-with open(user_based_knn_model_path, "rb") as f:
-    user_based_knn_model = joblib.load(f)
+# with open(user_based_knn_model_path, "rb") as f:
+#     user_based_knn_model = joblib.load(f)
 
-with open(svd_model_path, "rb") as f:
-    svd_model = joblib.load(f)
+# with open(svd_model_path, "rb") as f:
+#     svd_model = joblib.load(f)
 
-# Now you can use these models for recommendations
+
+# Access the models from session state
+cosine_similarity_model_path = hf_hub_download(repo_name, MODEL_TRAINER_COSINESIMILARITY_MODEL_NAME)
+item_based_knn_model = st.session_state.models_loaded["item_based_knn_model"]
+user_based_knn_model = st.session_state.models_loaded["user_based_knn_model"]
+svd_model = st.session_state.models_loaded["svd_model"] 
 print("Models loaded successfully!")
     
 # Streamlit UI
@@ -113,8 +142,7 @@ if app_selector == "Content-Based Recommender":
 elif app_selector == "Collaborative Recommender":
     st.title("Collaborative Recommender System")
     
-    try: 
-        
+    try:  
         # Sidebar for choosing the collaborative filtering method
         collaborative_method = st.sidebar.selectbox(
             "Choose a collaborative filtering method:", 

requirements.txt

@@ -6,4 +6,4 @@ transformers
 huggingface_hub
 datasets
 scikit-surprise
--e .
+# -e .

run_pipeline.py

@@ -8,6 +8,7 @@ from anime_recommender.source.collaborative_recommender import CollaborativeMode
 from anime_recommender.source.content_based_recommender import ContentBasedModelTrainer 
 from anime_recommender.source.top_anime_recommenders import PopularityBasedRecommendor
 
+
 if __name__ == "__main__":
     try:
         training_pipeline_config = TrainingPipelineConfig()