adding links works!

prompts/find_links.txt

@@ -0,0 +1,27 @@
+You are an expert at identifying key terms, concepts, and proper nouns in text that would benefit from having reference links. Your task is to analyze the provided transcript text and identify terms that should be linked to provide additional context and information for readers.
+
+Guidelines for identifying terms:
+1. Focus on technical terms, scientific concepts, proper nouns, and important ideas that readers might want to learn more about
+2. Avoid common words or phrases that don't need additional context
+3. For each term you identify, explain briefly why it would be valuable to link it
+4. If a term appears multiple times, only identify it once
+5. Limit your selection to the most important 3-5 terms per chunk of text
+
+Respond in this format for each term:
+TERM: <the exact term as it appears in text>
+REASON: <1-2 sentences explaining why this term should be linked>
+
+Example input:
+"The quantum computer used superconducting qubits to achieve quantum supremacy, a milestone that Google's Sycamore processor reached in 2019."
+
+Example output:
+TERM: superconducting qubits
+REASON: A fundamental building block of certain quantum computers that readers should understand to grasp quantum computing concepts.
+
+TERM: quantum supremacy
+REASON: A significant milestone in quantum computing that represents when quantum computers outperform classical computers.
+
+TERM: Google's Sycamore
+REASON: A specific quantum processor that made history, providing context about real-world quantum computing achievements.
+
+Analyze the following transcript text and identify key terms that should be linked: 

scripts/add_links.py

@@ -0,0 +1,209 @@
+import argparse
+from pathlib import Path
+import os
+import re
+from typing import List, Dict, Tuple
+from dataclasses import dataclass
+import anthropic
+from exa_py import Exa
+
+@dataclass
+class Term:
+    """A term identified for linking with its explanation"""
+    term: str
+    reason: str
+
+@dataclass
+class Link:
+    """A link found for a term"""
+    term: str
+    url: str
+    title: str
+
+def chunk_text(text: str, max_chunk_size: int = 2000) -> List[str]:
+    """Split text into chunks of roughly equal size at paragraph boundaries"""
+    paragraphs = text.split("\n\n")
+    chunks = []
+    current_chunk = []
+    current_size = 0
+    
+    for para in paragraphs:
+        para_size = len(para)
+        if current_size + para_size > max_chunk_size and current_chunk:
+            chunks.append("\n\n".join(current_chunk))
+            current_chunk = [para]
+            current_size = para_size
+        else:
+            current_chunk.append(para)
+            current_size += para_size
+    
+    if current_chunk:
+        chunks.append("\n\n".join(current_chunk))
+    
+    return chunks
+
+def parse_claude_response(response: str) -> List[Term]:
+    """Parse Claude's response to extract terms and reasons"""
+    terms = []
+    current_term = None
+    current_reason = None
+    
+    for line in response.split("\n"):
+        line = line.strip()
+        if not line:
+            continue
+            
+        if line.startswith("TERM: "):
+            # Save previous term if exists
+            if current_term and current_reason:
+                terms.append(Term(current_term, current_reason))
+            current_term = line[6:].strip()
+            current_reason = None
+        elif line.startswith("REASON: "):
+            current_reason = line[8:].strip()
+            
+    # Add final term
+    if current_term and current_reason:
+        terms.append(Term(current_term, current_reason))
+        
+    return terms
+
+def find_links_for_terms(exa: Exa, terms: List[Term]) -> Dict[str, Link]:
+    """Find best link for each term using Exa search"""
+    links = {}
+    
+    for term in terms:
+        # Construct a search query that looks for authoritative sources
+        # query = f"The best explanation or overview of {term.term} is (site: wikipedia.org OR site: .edu OR site: .gov):"
+        
+        try:
+            # Search with Exa
+            results = exa.search(term.term, num_results=1, type="auto")
+            if results.results:
+                result = results.results[0]
+                links[term.term] = Link(
+                    term=term.term,
+                    url=result.url,
+                    title=result.title
+                )
+        except Exception as e:
+            print(f"Error finding link for {term.term}: {e}")
+            continue
+            
+    return links
+
+def add_links_to_text(text: str, links: Dict[str, Link]) -> str:
+    """Add markdown links to text for all terms we have links for"""
+    # Sort terms by length (descending) to handle overlapping terms correctly
+    terms = sorted(links.keys(), key=len, reverse=True)
+    
+    # Create regex pattern that matches whole words only
+    patterns = [re.compile(fr'\b{re.escape(term)}\b') for term in terms]
+    
+    # Track which terms we've linked to avoid duplicate links
+    linked_terms = set()
+    
+    # Process each term
+    result = text
+    for term, pattern in zip(terms, patterns):
+        if term in linked_terms:
+            continue
+            
+        # Only replace first occurrence
+        link = links[term]
+        replacement = f"[{term}]({link.url})"
+        result = pattern.sub(replacement, result, count=1)
+        linked_terms.add(term)
+        
+    return result
+
+def process_transcript(
+    transcript_path: Path,
+    claude_client: anthropic.Client,
+    exa_client: Exa,
+    prompt_template: str
+) -> str:
+    """Process a transcript file to add reference links"""
+    # Read transcript
+    text = transcript_path.read_text()
+    
+    # Split into chunks
+    chunks = chunk_text(text)
+    
+    # Process each chunk
+    all_terms = []
+    for chunk in chunks:
+        # Get Claude's suggestions
+        prompt = prompt_template + "\n\n" + chunk
+        response = claude_client.messages.create(
+            model="claude-3-5-sonnet-20241022",
+            max_tokens=1024,
+            system="You are a helpful AI assistant.",
+            messages=[{"role": "user", "content": prompt}]
+        )
+        
+        # Parse response
+        terms = parse_claude_response(response.content[0].text)
+        all_terms.extend(terms)
+        
+    # Find links for all terms
+    links = find_links_for_terms(exa_client, all_terms)
+    
+    # Add links to text
+    linked_text = add_links_to_text(text, links)
+    
+    return linked_text
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "transcript", 
+        nargs="?",  # Make the argument optional
+        default="output/transcripts/transcript.md",
+        help="Path to transcript file (default: output/transcripts/transcript.md)"
+    )
+    parser.add_argument("--output", help="Output file path (default: input path with -linked suffix)")
+    args = parser.parse_args()
+    
+    transcript_path = Path(args.transcript)
+    if not transcript_path.exists():
+        raise FileNotFoundError(f"Transcript file not found: {transcript_path}")
+        
+    # Set up output path
+    if args.output:
+        output_path = Path(args.output)
+    else:
+        stem = transcript_path.stem
+        output_path = transcript_path.parent / f"{stem}-linked{transcript_path.suffix}"
+        
+    # Read prompt template
+    prompt_path = Path("prompts/find_links.txt")
+    if not prompt_path.exists():
+        raise FileNotFoundError(f"Prompt file not found: {prompt_path}")
+    prompt_template = prompt_path.read_text()
+    
+    # Initialize clients
+    claude_client = anthropic.Client(api_key=os.getenv("ANTHROPIC_API_KEY"))
+    exa_client = Exa(api_key=os.getenv("EXA_API_KEY"))
+    
+    try:
+        # Process transcript
+        linked_text = process_transcript(
+            transcript_path,
+            claude_client,
+            exa_client,
+            prompt_template
+        )
+        
+        # Save output
+        output_path.write_text(linked_text)
+        print(f"Processed transcript saved to: {output_path}")
+        
+    except Exception as e:
+        print(f"Error processing transcript: {e}")
+        return 1
+        
+    return 0
+
+if __name__ == "__main__":
+    main()