Spaces:
Sleeping
Sleeping
File size: 6,384 Bytes
e547b24 036b96f a1c8ee1 df8ede4 a1c8ee1 036b96f 5392ab0 036b96f a1c8ee1 036b96f a1c8ee1 036b96f a1c8ee1 036b96f df8ede4 a1c8ee1 036b96f a1c8ee1 d0338ea adb8560 036b96f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 |
import gradio as gr
from huggingface_hub import InferenceClient
from langchain.vectorstores import FAISS
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.document_loaders import TextLoader
# Initialize the Hugging Face Inference client with an open-source LLM
client = InferenceClient("HuggingFaceH4/zephyr-7b-beta") # You can use any supported model
# Sample knowledge base for Crustdata APIs
docs = """
# Crustdata Dataset API
## Description
The Crustdata Dataset API provides access to a wide variety of datasets across different domains. It allows users to search, filter, and retrieve datasets based on categories, tags, and other metadata.
## Key Endpoints
### 1. **GET /datasets**
- **Description**: Retrieves a list of available datasets.
- **Parameters**:
- `category` (optional): Filter datasets by a specific category.
- `tags` (optional): Filter datasets by tags (comma-separated).
- `limit` (optional): Maximum number of datasets to return (default: 10).
- **Example Request**:
```bash
curl -X GET "https://api.crustdata.com/datasets?category=finance&tags=economy,stocks&limit=5"
```
- **Example Response**:
```json
{
"datasets": [
{
"id": "12345",
"name": "Global Finance Dataset",
"category": "finance",
"tags": ["economy", "stocks"]
},
...
]
}
```
### 2. **GET /datasets/{id}**
- **Description**: Retrieves detailed information about a specific dataset.
- **Parameters**:
- `id` (required): The unique identifier of the dataset.
- **Example Request**:
```bash
curl -X GET "https://api.crustdata.com/datasets/12345"
```
- **Example Response**:
```json
{
"id": "12345",
"name": "Global Finance Dataset",
"description": "A comprehensive dataset on global financial markets.",
"category": "finance",
"tags": ["economy", "stocks"],
"source": "World Bank"
}
```
---
# Crustdata Discovery and Enrichment API
## Description
The Crustdata Discovery and Enrichment API allows users to enrich their datasets by adding metadata, geolocation information, and other relevant attributes.
## Key Endpoints
### 1. **POST /enrich**
- **Description**: Enriches input data with additional metadata based on the specified enrichment type.
- **Parameters**:
- `input_data` (required): A list of data entries to be enriched.
- `enrichment_type` (required): The type of enrichment to apply. Supported types:
- `geolocation`
- `demographics`
- **Example Request**:
```bash
curl -X POST "https://api.crustdata.com/enrich" \
-H "Content-Type: application/json" \
-d '{
"input_data": [{"address": "123 Main St, Springfield"}],
"enrichment_type": "geolocation"
}'
```
- **Example Response**:
```json
{
"enriched_data": [
{
"address": "123 Main St, Springfield",
"latitude": 37.12345,
"longitude": -93.12345
}
]
}
```
### 2. **POST /search**
- **Description**: Searches for relevant metadata or datasets based on user-provided criteria.
- **Parameters**:
- `query` (required): The search term or query string.
- `filters` (optional): Additional filters to narrow down the search results.
- **Example Request**:
```bash
curl -X POST "https://api.crustdata.com/search" \
-H "Content-Type: application/json" \
-d '{
"query": "energy consumption",
"filters": {"category": "energy"}
}'
```
- **Example Response**:
```json
{
"results": [
{
"id": "67890",
"name": "Energy Consumption Dataset",
"category": "energy",
"tags": ["consumption", "renewables"]
}
]
}
```
---
# General Notes
- All endpoints require authentication using an API key.
- API requests must include the `Authorization` header:
```plaintext
Authorization: Bearer YOUR_API_KEY
```
- Response format: JSON
- Base URL: `https://api.crustdata.com`
"""
# Split the documentation into smaller chunks
text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
doc_chunks = text_splitter.create_documents([docs])
# Create embeddings and initialize FAISS vector store
embedding_model = "sentence-transformers/all-MiniLM-L6-v2"
embeddings = HuggingFaceEmbeddings(model_name=embedding_model)
docsearch = FAISS.from_documents(doc_chunks, embeddings)
def retrieve_context(query):
"""Retrieve the most relevant context from the knowledge base."""
results = docsearch.similarity_search(query, k=2) # Retrieve top 2 most similar chunks
context = "\n".join([res.page_content for res in results])
return context
def respond(
message,
history: list[tuple[str, str]],
system_message,
max_tokens,
temperature,
top_p,
):
"""Generate a response using the Hugging Face Inference API."""
# Retrieve relevant context from the knowledge base
context = retrieve_context(message)
prompt = f"{system_message}\n\nContext:\n{context}\n\nUser: {message}\nAssistant:"
messages = [{"role": "system", "content": system_message}]
for val in history:
if val[0]:
messages.append({"role": "user", "content": val[0]})
if val[1]:
messages.append({"role": "assistant", "content": val[1]})
messages.append({"role": "user", "content": prompt})
response = ""
for message in client.chat_completion(
messages,
max_tokens=max_tokens,
stream=True,
temperature=temperature,
top_p=top_p,
):
token = message.choices[0].delta.content
response += token
yield response
# Gradio interface
demo = gr.ChatInterface(
respond,
additional_inputs=[
gr.Textbox(value="You are a technical assistant for Crustdata APIs.", label="System message"),
gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)"),
],
title="Crustdata API Chatbot",
description="Ask any technical questions about Crustdata’s Dataset and Discovery APIs.",
)
if __name__ == "__main__":
demo.launch(share=True)
|