test-llama3, not sure how it'll go on langserve

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.11.9
 
 WORKDIR /code
 
@@ -8,4 +8,4 @@ RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
 
 COPY . .
 
-CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["python", "main.py"]

main.py

@@ -0,0 +1,61 @@
+from fastapi import FastAPI
+
+from langchain_community.llms import LlamaCpp
+from langchain_core.callbacks import CallbackManager, StreamingStdOutCallbackHandler
+from langchain_core.prompts import PromptTemplate
+
+from langserve import add_routes
+from fastapi.middleware.cors import CORSMiddleware
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="A simple api server using Langchain's Runnable interfaces",
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=['*'],
+    allow_methods=['*'],
+    allow_headers=['*'],
+    allow_credentials=True
+)
+
+
+template = """Give a very concise one word answer to question.
+Question: {question}
+Answer:
+"""
+
+prompt = PromptTemplate.from_template(template)
+
+# Callbacks support token-wise streaming
+callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
+
+n_gpu_layers = -1  # The number of layers to put on the GPU. The rest will be on the CPU. If you don't know how many layers there are, you can use -1 to move all to GPU.
+n_batch = 512  # Should be between 1 and n_ctx, consider the amount of VRAM in your GPU.
+
+# Make sure the model path is correct for your system!
+llm = LlamaCpp(
+    model_path="Meta-Llama-3-8B-Instruct-v2.Q4_K_S.gguf",
+    n_gpu_layers=n_gpu_layers,
+    n_batch=n_batch,
+    callback_manager=callback_manager,
+    verbose=True,  # Verbose is required to pass to the callback manager
+)
+
+add_routes(
+    app,
+    prompt | llm,
+    path='/test'
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app)
+
+# llm_chain = prompt | llm
+
+# question = "Hi"
+# x = llm_chain.invoke({"question": question})

requirements.txt

@@ -0,0 +1,196 @@
+accelerate==0.30.1
+aiofiles==23.2.1
+aiohttp==3.9.5
+aiosignal==1.3.1
+altair==5.3.0
+annotated-types==0.6.0
+anyio==4.3.0
+asgiref==3.8.1
+asttokens==2.4.1
+attrs==23.2.0
+backoff==2.2.1
+bcrypt==4.1.2
+beautifulsoup4==4.12.3
+build==1.2.1
+cachetools==5.3.3
+certifi==2024.2.2
+charset-normalizer==3.3.2
+chroma-hnswlib==0.7.3
+chromadb==0.5.0
+click==8.1.7
+coloredlogs==15.0.1
+comm==0.2.2
+contourpy==1.2.1
+cycler==0.12.1
+dataclasses-json==0.6.5
+datasets==2.19.1
+debugpy==1.8.1
+decorator==5.1.1
+Deprecated==1.2.14
+dill==0.3.8
+diskcache==5.6.3
+distro==1.9.0
+dnspython==2.6.1
+email_validator==2.1.1
+evaluate==0.4.2
+executing==2.0.1
+faiss-cpu==1.8.0
+fastapi==0.111.0
+fastapi-cli==0.0.2
+ffmpy==0.3.2
+filelock==3.14.0
+flatbuffers==24.3.25
+fonttools==4.51.0
+frozenlist==1.4.1
+fsspec==2024.3.1
+google-auth==2.29.0
+googleapis-common-protos==1.63.0
+gradio==4.31.0
+gradio_client==0.16.2
+greenlet==3.0.3
+grpcio==1.63.0
+h11==0.14.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.23.0
+humanfriendly==10.0
+idna==3.7
+importlib-metadata==7.0.0
+importlib_resources==6.4.0
+ipykernel==6.29.4
+ipython==8.24.0
+jedi==0.19.1
+Jinja2==3.1.3
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.22.0
+jsonschema-specifications==2023.12.1
+jupyter_client==8.6.1
+jupyter_core==5.7.2
+kiwisolver==1.4.5
+kubernetes==29.0.0
+langchain==0.1.17
+langchain-community==0.0.36
+langchain-core==0.1.50
+langchain-openai==0.1.6
+langchain-text-splitters==0.0.1
+langsmith==0.1.53
+llama_cpp_python==0.2.75
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+marshmallow==3.21.2
+matplotlib==3.8.4
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+mmh3==4.1.0
+monotonic==1.6
+mpmath==1.3.0
+multidict==6.0.5
+multiprocess==0.70.16
+mypy-extensions==1.0.0
+nest-asyncio==1.6.0
+networkx==3.3
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+onnxruntime==1.17.3
+openai==1.25.1
+opentelemetry-api==1.24.0
+opentelemetry-exporter-otlp-proto-common==1.24.0
+opentelemetry-exporter-otlp-proto-grpc==1.24.0
+opentelemetry-instrumentation==0.45b0
+opentelemetry-instrumentation-asgi==0.45b0
+opentelemetry-instrumentation-fastapi==0.45b0
+opentelemetry-proto==1.24.0
+opentelemetry-sdk==1.24.0
+opentelemetry-semantic-conventions==0.45b0
+opentelemetry-util-http==0.45b0
+orjson==3.10.3
+overrides==7.7.0
+packaging==23.2
+pandas==2.2.2
+parso==0.8.4
+pexpect==4.9.0
+pillow==10.3.0
+platformdirs==4.2.1
+posthog==3.5.0
+prompt-toolkit==3.0.43
+protobuf==4.25.3
+psutil==5.9.8
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==16.1.0
+pyarrow-hotfix==0.6
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pydantic==2.7.1
+pydantic_core==2.18.2
+pydub==0.25.1
+Pygments==2.17.2
+pyparsing==3.1.2
+PyPika==0.48.9
+pyproject_hooks==1.1.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.1
+pyzmq==26.0.3
+referencing==0.35.1
+regex==2024.4.28
+requests==2.31.0
+requests-oauthlib==2.0.0
+rich==13.7.1
+rpds-py==0.18.1
+rsa==4.9
+ruff==0.4.4
+safetensors==0.4.3
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+soupsieve==2.5
+SQLAlchemy==2.0.29
+stack-data==0.6.3
+starlette==0.37.2
+sympy==1.12
+tenacity==8.2.3
+tiktoken==0.6.0
+tokenizers==0.19.1
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.3.0
+tornado==6.4
+tqdm==4.66.4
+traitlets==5.14.3
+transformers==4.41.0
+triton==2.3.0
+typer==0.12.3
+typing-inspect==0.9.0
+typing_extensions==4.11.0
+tzdata==2024.1
+ujson==5.9.0
+urllib3==2.2.1
+uvicorn==0.29.0
+uvloop==0.19.0
+vtk==9.3.0
+watchfiles==0.21.0
+wcwidth==0.2.13
+websocket-client==1.8.0
+websockets==11.0.3
+wrapt==1.16.0
+xxhash==3.4.1
+yarl==1.9.4
+zipp==3.18.1