readme update

README.md

@@ -9,12 +9,26 @@ app_file: app.py
 pinned: false
 ---
 
-# Code to Doc Streamlit App
+# Python Code Summarizer
 
-A Streamlit-based application that converts code to documentation.
+This Streamlit app utilizes the CodeT5 model to generate summaries of Python code files, leveraging the Hugging Face Transformers library.
 
 ## Features
-- Code analysis
-- Documentation generation
-- Interactive interface
-- Real-time updates
+- Upload Python files or paste code directly
+- Generate file-level summaries
+- Generate function-level summaries
+- Generate class-level summaries
+
+## Usage
+1. Upload a Python file or paste your code
+2. Select the types of summaries you want to generate
+3. Click "Summarize Code"
+4. View the generated summaries
+
+## Model Information
+This app employs CodeT5, a pretrained model available on Hugging Face, developed by Salesforce Research for code understanding and generation tasks. It is trained on a vast corpus of code and documentation.
+
+## Limitations
+- Summaries may not always be accurate
+- Long files may be truncated due to model context limits
+- Complex code structures might not be properly understood

app.py

@@ -1,136 +1,360 @@
 import streamlit as st
-import os
-import tempfile
-import shutil
-from code_analyzer import CodeAnalyzer
-import plotly.express as px
-import pandas as pd
+import torch
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+import re
+import time
 
-st.set_page_config(
-    page_title="Code Analyzer",
-    page_icon="🔍",
-    layout="wide"
-)
+# Model constants
+CODET5_MODEL = "Salesforce/codet5-base-multi-sum"
 
-st.title("🔍 Code Project Analyzer")
-st.write("Upload your code files and analyze them with AI-powered insights")
-
-def create_metrics_chart(metrics):
-    """Create a bar chart for code metrics"""
-    df = pd.DataFrame({
-        'Metric': list(metrics.keys()),
-        'Value': list(metrics.values())
-    })
-    fig = px.bar(df, x='Metric', y='Value', title='Code Metrics')
-    return fig
-
-def display_tech_stack(tech_stack):
-    """Display technology stack in an organized way"""
-    st.subheader("🛠️ Technology Stack")
-    cols = st.columns(3)
+class CodeT5Summarizer:
+    def __init__(self, device=None):
+        """Initialize CodeT5 summarization model."""
+        self.device = device if device else ('cuda' if torch.cuda.is_available() else 'cpu')
+        
+        # Initialize model and tokenizer
+        with st.spinner("Loading CodeT5 model... this may take a minute..."):
+            self.tokenizer = AutoTokenizer.from_pretrained(CODET5_MODEL)
+            self.model = AutoModelForSeq2SeqLM.from_pretrained(CODET5_MODEL).to(self.device)
     
-    with cols[0]:
-        st.write("**Languages**")
-        if tech_stack["languages"]:
-            for lang in tech_stack["languages"]:
-                st.write(f"- {lang}")
-        else:
-            st.write("No languages detected")
+    def preprocess_code(self, code):
+        """Clean and preprocess the Python code."""
+        # Remove empty lines
+        code = re.sub(r'\n\s*\n', '\n', code)
+        
+        # Remove excessive comments (keeping docstrings)
+        code_lines = []
+        in_docstring = False
+        docstring_delimiter = None
+        
+        for line in code.split('\n'):
+            # Check for docstring delimiters
+            if '"""' in line or "'''" in line:
+                delimiter = '"""' if '"""' in line else "'''"
+                if not in_docstring:
+                    in_docstring = True
+                    docstring_delimiter = delimiter
+                elif docstring_delimiter == delimiter:
+                    in_docstring = False
+                    docstring_delimiter = None
             
-    with cols[1]:
-        st.write("**Frameworks**")
-        if tech_stack["frameworks"]:
-            for framework in tech_stack["frameworks"]:
-                st.write(f"- {framework}")
-        else:
-            st.write("No frameworks detected")
+            # Keep docstrings and non-comment lines
+            if in_docstring or not line.strip().startswith('#'):
+                code_lines.append(line)
+        
+        processed_code = '\n'.join(code_lines)
+        
+        # Normalize whitespace
+        processed_code = re.sub(r' +', ' ', processed_code)
+        
+        return processed_code
+    
+    def extract_functions(self, code):
+        """Extract individual functions for summarization"""
+        # Simple regex to find function definitions
+        function_pattern = r'def\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(.*?\).*?:'
+        function_matches = re.finditer(function_pattern, code, re.DOTALL)
+        
+        functions = []
+        for match in function_matches:
+            start_pos = match.start()
+            # Find the function body
+            function_name = match.group(1)
+            lines = code[start_pos:].split('\n')
             
-    with cols[2]:
-        st.write("**Dependencies**")
-        if tech_stack["dependencies"]:
-            for dep in tech_stack["dependencies"]:
-                st.write(f"- {dep}")
-        else:
-            st.write("No dependencies detected")
-
-def save_uploaded_files(uploaded_files):
-    """Save uploaded files to a temporary directory"""
-    temp_dir = tempfile.mkdtemp()
-    for uploaded_file in uploaded_files:
-        file_path = os.path.join(temp_dir, uploaded_file.name)
-        os.makedirs(os.path.dirname(file_path), exist_ok=True)
-        with open(file_path, "wb") as f:
-            f.write(uploaded_file.getbuffer())
-    return temp_dir
-
-# File upload section
-uploaded_files = st.file_uploader(
-    "Upload your code files",
-    accept_multiple_files=True,
-    type=['py', 'java', 'js', 'jsx', 'ts', 'tsx']
-)
-
-# Questions input
-st.subheader("📝 Analysis Questions")
-default_questions = """What is the project's abstract?
-What is the system architecture?
-What are the software requirements?
-What are the hardware requirements?"""
-
-questions = st.text_area(
-    "Enter your questions (one per line)",
-    value=default_questions,
-    height=150
-)
-
-analyze_button = st.button("🔍 Analyze Code")
-
-if analyze_button and uploaded_files:
-    with st.spinner("Analyzing your code..."):
-        # Save uploaded files
-        temp_dir = save_uploaded_files(uploaded_files)
+            # Skip the function definition line
+            body_start = 1
+            while body_start < len(lines) and not lines[body_start].strip():
+                body_start += 1
+                
+            if body_start < len(lines):
+                # Get the indentation of the function body
+                body_indent = len(lines[body_start]) - len(lines[body_start].lstrip())
+                
+                # Gather all lines with at least this indentation
+                function_body = [lines[0]]  # The function definition
+                i = 1
+                while i < len(lines):
+                    line = lines[i]
+                    if line.strip() and (len(line) - len(line.lstrip())) < body_indent and not line.strip().startswith('#'):
+                        break
+                    function_body.append(line)
+                    i += 1
+                
+                function_code = '\n'.join(function_body)
+                functions.append((function_name, function_code))
         
-        # Save questions to a temporary file
-        questions_file = os.path.join(temp_dir, "questions.txt")
-        with open(questions_file, "w") as f:
-            f.write(questions)
+        # Simple regex to find class methods
+        class_pattern = r'class\s+([a-zA-Z_][a-zA-Z0-9_]*)'
+        class_matches = re.finditer(class_pattern, code, re.DOTALL)
         
-        try:
-            # Run analysis
-            analyzer = CodeAnalyzer()
-            results = analyzer.analyze_project(temp_dir, questions_file)
+        for match in class_matches:
+            class_name = match.group(1)
+            start_pos = match.start()
             
-            # Display results in tabs
-            tab1, tab2, tab3 = st.tabs(["📊 Overview", "💻 Code Metrics", "❓ Q&A"])
+            # Find class methods using the function pattern
+            class_code = code[start_pos:]
+            method_matches = re.finditer(function_pattern, class_code, re.DOTALL)
             
-            with tab1:
-                st.subheader("🎯 Project Objective")
-                st.write(results["objective"])
+            for method_match in method_matches:
+                method_name = method_match.group(1)
+                # Skip if this is not a method (i.e., it's a function outside the class)
+                if method_match.start() > 200:  # Simple heuristic to check if method is within class scope
+                    break
                 
-                display_tech_stack(results["tech_stack"])
-            
-            with tab2:
-                st.subheader("📊 Code Metrics")
-                metrics_chart = create_metrics_chart(results["metrics"])
-                st.plotly_chart(metrics_chart, use_container_width=True)
+                # Get the full method code
+                method_start = method_match.start()
+                method_lines = class_code[method_start:].split('\n')
                 
-                # Complexity assessment
-                complexity = "Low" if results["metrics"]["complexity_score"] < 10 else \
-                           "Medium" if results["metrics"]["complexity_score"] < 30 else "High"
-                st.info(f"Project Complexity: {complexity}")
+                # Skip the method definition line
+                body_start = 1
+                while body_start < len(method_lines) and not method_lines[body_start].strip():
+                    body_start += 1
+                
+                if body_start < len(method_lines):
+                    # Get the indentation of the method body
+                    body_indent = len(method_lines[body_start]) - len(method_lines[body_start].lstrip())
+                    
+                    # Gather all lines with at least this indentation
+                    method_body = [method_lines[0]]  # The method definition
+                    i = 1
+                    while i < len(method_lines):
+                        line = method_lines[i]
+                        if line.strip() and (len(line) - len(line.lstrip())) < body_indent and not line.strip().startswith('#'):
+                            break
+                        method_body.append(line)
+                        i += 1
+                    
+                    method_code = '\n'.join(method_body)
+                    functions.append((f"{class_name}.{method_name}", method_code))
+        
+        return functions
+    
+    def extract_classes(self, code):
+        """Extract class definitions for summarization"""
+        class_pattern = r'class\s+([a-zA-Z_][a-zA-Z0-9_]*)'
+        class_matches = re.finditer(class_pattern, code, re.DOTALL)
+        
+        classes = []
+        for match in class_matches:
+            class_name = match.group(1)
+            start_pos = match.start()
             
-            with tab3:
-                st.subheader("❓ Analysis Results")
-                for question, answer in results["answers"].items():
-                    with st.expander(question):
-                        st.write(answer)
+            # Extract class body
+            class_lines = code[start_pos:].split('\n')
+            
+            # Skip the class definition line
+            body_start = 1
+            while body_start < len(class_lines) and not class_lines[body_start].strip():
+                body_start += 1
+            
+            if body_start < len(class_lines):
+                # Get the indentation of the class body
+                body_indent = len(class_lines[body_start]) - len(class_lines[body_start].lstrip())
+                
+                # Gather all lines with at least this indentation
+                class_body = [class_lines[0]]  # The class definition
+                i = 1
+                while i < len(class_lines):
+                    line = class_lines[i]
+                    if line.strip() and (len(line) - len(line.lstrip())) < body_indent:
+                        break
+                    class_body.append(line)
+                    i += 1
+                
+                class_code = '\n'.join(class_body)
+                classes.append((class_name, class_code))
+        
+        return classes
+    
+    def summarize(self, code, max_length=50):
+        """Generate summary using CodeT5."""
+        # Truncate input if needed
+        max_input_length = 512  # CodeT5 typically accepts up to 512 tokens
+        tokenized_code = self.tokenizer(code, truncation=True, max_length=max_input_length, return_tensors="pt").to(self.device)
+        
+        with torch.no_grad():
+            generated_ids = self.model.generate(
+                tokenized_code["input_ids"],
+                max_length=max_length,
+                num_beams=4,
+                early_stopping=True
+            )
+        
+        summary = self.tokenizer.decode(generated_ids[0], skip_special_tokens=True)
+        return summary
+    
+    def summarize_code(self, code, summarize_functions=True, summarize_classes=True):
+        """
+        Generate full file summary and optionally function/class level summaries.
+        Returns a dictionary with summaries.
+        """
+        preprocessed_code = self.preprocess_code(code)
+        
+        results = {
+            "file_summary": None,
+            "function_summaries": {},
+            "class_summaries": {}
+        }
         
+        # Generate file-level summary
+        try:
+            file_summary = self.summarize(preprocessed_code)
+            results["file_summary"] = file_summary
         except Exception as e:
-            st.error(f"An error occurred during analysis: {str(e)}")
-        
-        finally:
-            # Cleanup
-            shutil.rmtree(temp_dir)
-else:
-    if analyze_button:
-        st.warning("Please upload some code files first!")
+            results["file_summary"] = f"Error generating file summary: {str(e)}"
+        
+        # Generate function-level summaries if requested
+        if summarize_functions:
+            functions = self.extract_functions(preprocessed_code)
+            
+            for function_name, function_code in functions:
+                try:
+                    summary = self.summarize(function_code)
+                    results["function_summaries"][function_name] = summary
+                except Exception as e:
+                    results["function_summaries"][function_name] = f"Error: {str(e)}"
+        
+        # Generate class-level summaries if requested
+        if summarize_classes:
+            classes = self.extract_classes(preprocessed_code)
+            
+            for class_name, class_code in classes:
+                try:
+                    summary = self.summarize(class_code)
+                    results["class_summaries"][class_name] = summary
+                except Exception as e:
+                    results["class_summaries"][class_name] = f"Error: {str(e)}"
+        
+        return results
+
+def main():
+    st.set_page_config(
+        page_title="Python Code Summarizer",
+        page_icon="📝",
+        layout="wide"
+    )
+    
+    st.title("📝 Python Code Summarizer using CodeT5")
+    st.markdown("""
+    Upload a Python file or paste code directly to generate summaries.
+    This app uses CodeT5, a pretrained model for code understanding and generation.
+    """)
+    
+    # Initialize session state
+    if 'summarizer' not in st.session_state:
+        st.session_state.summarizer = None
+    
+    # Load model if not already loaded
+    if st.session_state.summarizer is None:
+        st.session_state.summarizer = CodeT5Summarizer()
+    
+    # Create tabs for different input methods
+    tab1, tab2 = st.tabs(["Upload Python File", "Paste Code"])
+    
+    with tab1:
+        uploaded_file = st.file_uploader("Choose a Python file", type=['py'])
+        if uploaded_file is not None:
+            code = uploaded_file.getvalue().decode('utf-8')
+            with st.expander("View Uploaded Code", expanded=False):
+                st.code(code, language='python')
+            
+            # Add summarization options
+            st.subheader("Summarization Options")
+            col1, col2 = st.columns(2)
+            with col1:
+                summarize_functions = st.checkbox("Generate function summaries", value=True)
+            with col2:
+                summarize_classes = st.checkbox("Generate class summaries", value=True)
+            
+            if st.button("Summarize Code", key="summarize_file"):
+                with st.spinner("Generating summaries..."):
+                    start_time = time.time()
+                    summaries = st.session_state.summarizer.summarize_code(
+                        code, 
+                        summarize_functions=summarize_functions,
+                        summarize_classes=summarize_classes
+                    )
+                    end_time = time.time()
+                    
+                    # Display summaries
+                    st.success(f"Summarization completed in {end_time - start_time:.2f} seconds!")
+                    
+                    # File summary
+                    st.subheader("File Summary")
+                    st.write(summaries["file_summary"])
+                    
+                    # Function summaries
+                    if summarize_functions and summaries["function_summaries"]:
+                        st.subheader("Function Summaries")
+                        for func_name, summary in summaries["function_summaries"].items():
+                            with st.expander(f"Function: {func_name}"):
+                                st.write(summary)
+                    
+                    # Class summaries
+                    if summarize_classes and summaries["class_summaries"]:
+                        st.subheader("Class Summaries")
+                        for class_name, summary in summaries["class_summaries"].items():
+                            with st.expander(f"Class: {class_name}"):
+                                st.write(summary)
+    
+    with tab2:
+        code = st.text_area("Paste Python code here", height=300)
+        if code:
+            # Add summarization options
+            st.subheader("Summarization Options")
+            col1, col2 = st.columns(2)
+            with col1:
+                summarize_functions = st.checkbox("Generate function summaries", value=True, key="func_paste")
+            with col2:
+                summarize_classes = st.checkbox("Generate class summaries", value=True, key="class_paste")
+            
+            if st.button("Summarize Code", key="summarize_paste"):
+                with st.spinner("Generating summaries..."):
+                    start_time = time.time()
+                    summaries = st.session_state.summarizer.summarize_code(
+                        code, 
+                        summarize_functions=summarize_functions,
+                        summarize_classes=summarize_classes
+                    )
+                    end_time = time.time()
+                    
+                    # Display summaries
+                    st.success(f"Summarization completed in {end_time - start_time:.2f} seconds!")
+                    
+                    # File summary
+                    st.subheader("File Summary")
+                    st.write(summaries["file_summary"])
+                    
+                    # Function summaries
+                    if summarize_functions and summaries["function_summaries"]:
+                        st.subheader("Function Summaries")
+                        for func_name, summary in summaries["function_summaries"].items():
+                            with st.expander(f"Function: {func_name}"):
+                                st.write(summary)
+                    
+                    # Class summaries
+                    if summarize_classes and summaries["class_summaries"]:
+                        st.subheader("Class Summaries")
+                        for class_name, summary in summaries["class_summaries"].items():
+                            with st.expander(f"Class: {class_name}"):
+                                st.write(summary)
+    
+    st.markdown("---")
+    st.markdown("### About")
+    st.markdown("""
+    This app uses the CodeT5 model to generate summaries of Python code. The model is trained on a large corpus of code and documentation.
+    
+    **Features:**
+    - File-level summaries
+    - Function-level summaries
+    - Class-level summaries
+    
+    **Limitations:**
+    - Summaries may not always be accurate
+    - Long files may be truncated
+    - Complex code structures might not be properly understood
+    """)
+
+if __name__ == "__main__":
+    main()

code_analyzer.py → code_analyzer2.py

@@ -7,11 +7,12 @@ from typing import List, Dict, Set, Any
 import pkg_resources
 import importlib.util
 from collections import defaultdict
+import huggingface_hub
 
-class CodeAnalyzer:
+class CodeAnalyzer2:
     def __init__(self):
         # Using different models for different types of analysis
-        self.summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
+        self.summarizer = pipeline("summarization", model="Graverman/t5-code-summary")
         
     def detect_technologies(self, code_files: Dict[str, str]) -> Dict[str, Any]:
         """Detect technologies used in the project"""
@@ -210,22 +211,22 @@ class CodeAnalyzer:
             "answers": answers
         }
 
-if __name__ == "__main__":
-    analyzer = CodeAnalyzer()
-    # Example usage
-    results = analyzer.analyze_project(
-        "./example_project",
-        "./questions.txt"
-    )
-    print("\nProject Objective:", results["objective"])
-    print("\nTechnology Stack:")
-    for category, items in results["tech_stack"].items():
-        print(f"- {category.title()}: {', '.join(items)}")
+# if __name__ == "__main__":
+#     analyzer = CodeAnalyzer()
+#     # Example usage
+#     results = analyzer.analyze_project(
+#         "./example_project",
+#         "./questions.txt"
+#     )
+#     print("\nProject Objective:", results["objective"])
+#     print("\nTechnology Stack:")
+#     for category, items in results["tech_stack"].items():
+#         print(f"- {category.title()}: {', '.join(items)}")
     
-    print("\nCode Metrics:")
-    for metric, value in results["metrics"].items():
-        print(f"- {metric.replace('_', ' ').title()}: {value}")
+#     print("\nCode Metrics:")
+#     for metric, value in results["metrics"].items():
+#         print(f"- {metric.replace('_', ' ').title()}: {value}")
     
-    print("\nAnswers to Questions:")
-    for q, a in results["answers"].items():
-        print(f"\n{q}:\n{a}")
+#     print("\nAnswers to Questions:")
+#     for q, a in results["answers"].items():
+#         print(f"\n{q}:\n{a}")

requirements.txt

@@ -1,8 +1,3 @@
-transformers[torch]==4.35.0
---extra-index-url https://download.pytorch.org/whl/cpu
-torch>=2.0.0
-numpy>=1.24.0
-pandas>=2.0.0
-streamlit>=1.30.0
-plotly>=5.18.0
-altair>=5.2.0
+streamlit>=1.22.0
+torch>=1.13.0
+transformers>=4.28.0