Spaces:
Sleeping
Sleeping
File size: 2,834 Bytes
8712c90 5ee5132 0de14de 5ee5132 0de14de 5ee5132 0de14de 5ee5132 0de14de 5ee5132 e46988f 5ee5132 6814601 5ee5132 0de14de 5ee5132 6814601 5ee5132 0de14de 5ee5132 0de14de 5ee5132 8712c90 5ee5132 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
import requests
import json
from typing import Dict, List
import numpy as np
def get_activation_values(text: str, feature_id: int) -> Dict:
"""Get activation values for a specific feature"""
url = "https://www.neuronpedia.org/api/activation/new"
data = {
"feature": {
"modelId": "gemma-2-2b",
"layer": "0-gemmascope-mlp-16k",
"index": str(feature_id)
},
"customText": text
}
response = requests.post(
url,
headers={"Content-Type": "application/json"},
json=data
)
return response.json()
def calculate_density(values: List[float], threshold: float = 0.5) -> float:
"""Calculate activation density (% of tokens with activation > threshold)"""
return sum(1 for v in values if v > threshold) / len(values)
def find_top_features_per_token(text: str, num_features: int = 5,
max_density: float = 0.01, batch_size: int = 100) -> Dict:
"""Find top features for each token with density filtering"""
# First get initial feature activations to get tokens
sample_activation = get_activation_values(text, 0)
tokens = sample_activation['tokens']
token_features = {token: [] for token in tokens}
# Process features in batches
for start_idx in range(0, 16384, batch_size):
for feature_id in range(start_idx, min(start_idx + batch_size, 16384)):
result = get_activation_values(text, feature_id)
values = result.get('values', [])
# Calculate density and skip if too high
density = calculate_density(values)
if density > max_density:
continue
# Add feature to each token's list if activated
for token_idx, (token, value) in enumerate(zip(tokens, values)):
if value > 0.5: # Activation threshold
token_features[token].append({
'feature_id': feature_id,
'activation': value,
'density': density
})
# Sort features for each token and keep top N
for token in token_features:
token_features[token].sort(key=lambda x: x['activation'], reverse=True)
token_features[token] = token_features[token][:num_features]
return token_features
# Test the function
text = "Nike - Just Do It"
token_features = find_top_features_per_token(text)
# Print results
print(f"Text: {text}\n")
for token, features in token_features.items():
if features: # Only show tokens with active features
print(f"\nToken: {token}")
for feat in features:
print(f" Feature {feat['feature_id']}: activation={feat['activation']:.3f}, density={feat['density']:.3%}") |