ed-neo4j/text-to-cypher-unsloth-Llama-3.3-70B-Instruct-bnb-4bit

Uploaded model

Experimental Text2Cypher finetune of "unsloth/Llama-3.3-70B-Instruct-bnb-4bit". Trained with 5K modified examples from the original dataset released by Neo4j

• Developed by: ed-neo4j
• License: apache-2.0
• Finetuned from model : unsloth/Llama-3.3-70B-Instruct-bnb-4bit

This llama model was trained 2x faster with Unsloth and Huggingface's TRL library.

Inference with Unsloth

You will need an a machine with:

• NVIDIA processor
• 40GB+ of GPU Memory
• Python 3.10 (tested)

Next:

• Create a Python Environment
• pip install unsloth

import os
from unsloth import FastLanguageModel
import torch
from transformers import TextStreamer
from dotenv import load_dotenv

#load model 
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name = "ed-neo4j/text-to-cypher-unsloth-Llama-3.3-70B-Instruct-bnb-4bit",
    max_seq_length = 50000,
    dtype = None,
    load_in_4bit = True,
)
# It is interesting to see model architecture
print (model)

#load model for Unsloth inference (2x faster inference)
FastLanguageModel.for_inference(model)

#Prepare data for the model
SYSTEM_PROMPT = """
Task: Generate Cypher statement to query a graph database.
Instructions: Use only the provided relationship types and properties in the schema.
Do not include any explanations or apologies.
Do not respond to anything other than constructing a Cypher statement.
Do not include any text except the generated Cypher statement.
"""

SCHEMA="""
Node properties:\n- **Country**\n  - `location`: POINT \n  - `code`: STRING Example: "AFG"\n  - `name`: STRING Example: "Afghanistan"\n  - `tld`: STRING Example: "AF"\n- **Filing**\n  - `begin`: DATE_TIME Min: 2000-02-08T00:00:00Z, Max: 2017-09-05T00:00:00Z\n  - `end`: DATE_TIME Min: 2000-02-08T00:00:00Z, Max: 2017-11-03T00:00:00Z\n  - `originator_bank_id`: STRING Example: "cimb-bank-berhad"\n  - `sar_id`: STRING Example: "3297"\n  - `beneficiary_bank`: STRING Example: "Barclays Bank Plc"\n  - `filer_org_name_id`: STRING Example: "the-bank-of-new-york-mellon-corp"\n  - `originator_bank_country`: STRING Example: "Singapore"\n  - `beneficiary_bank_country`: STRING Example: "United Kingdom"\n  - `filer_org_name`: STRING Example: "The Bank of New York Mellon Corp."\n  - `originator_iso`: STRING Example: "SGP"\n  - `beneficiary_bank_id`: STRING Example: "barclays-bank-plc-london-england-gbr"\n  - `origin_lat`: STRING Example: "1.3667"\n  - `origin_lng`: STRING Example: "103.8"\n  - `end_date_format`: STRING Example: "2015-09-25T00:00:00Z"\n  - `begin_date_format`: STRING Example: "2015-03-25T00:00:00Z"\n  - `originator_bank`: STRING Example: "CIMB Bank Berhad"\n  - `beneficiary_lat`: STRING Example: "54"\n  - `beneficiary_iso`: STRING Example: "GBR"\n  - `beneficiary_lng`: STRING Example: "-2"\n  - `begin_date`: STRING Example: "Mar 25, 2015"\n  - `id`: STRING Example: "223254"\n  - `end_date`: STRING Example: "Sep 25, 2015"\n  - `amount`: INTEGER Min: 1.18, Max: 2721000000\n  - `number`: INTEGER Min: 1, Max: 174\n- **Entity**\n  - `id`: STRING Example: "the-bank-of-new-york-mellon-corp"\n  - `location`: POINT \n  - `name`: STRING Example: "The Bank of New York Mellon Corp."\n  - `country`: STRING Example: "CHN"\nRelationship properties:\n\nThe relationships:\n(:Filing)-[:BENEFITS]->(:Entity)\n(:Filing)-[:CONCERNS]->(:Entity)\n(:Filing)-[:ORIGINATOR]->(:Entity)\n(:Entity)-[:FILED]->(:Filing)\n(:Entity)-[:COUNTRY]->(:Country)
"""

QUESTION="""
Which 3 countries have the most entities linked as beneficiaries in filings?
"""
USER_INSTRUCTION = """
Schema:
{schema}
Question:
{question}
"""
# Create Messages in Llama format
messages = [
    {'role': 'system', 'content': SYSTEM_PROMPT}, #This is optional
    {"role": "user", "content": USER_INSTRUCTION.format(schema=SCHEMA, question=QUESTION)}, #this is mandatory (schema and user question in natural language)
]

#Apply the tokenizer chat template to the input messages)
prompt_text = tokenizer.apply_chat_template(messages, add_generation_prompt=True,tokenize=False)

#Turn the prompt text into a set of tokens and load them to GPU
inputs = tokenizer([prompt_text], return_tensors = "pt").to("cuda")

#Generate cypher (streaming mode on)
print ("***************************")
text_streamer = TextStreamer(tokenizer)
_ = model.generate(**inputs, streamer = text_streamer, max_new_tokens = 128)

Inference with bitsandbytes

#pip install bitsandbytes
#pip install python-dotenv
#pip install transformers
#pip install accelerate>=0.26.0
import os
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from dotenv import load_dotenv

#load environment variables
load_dotenv()
FINETUNED_MODEL_NAME= "ed-neo4j/text-to-cypher-unsloth-Llama-3.3-70B-Instruct-bnb-4bit"
LOAD_IN_4_BIT =True


SYSTEM_PROMPT = """
Task: Generate Cypher statement to query a graph database.
Instructions: Use only the provided relationship types and properties in the schema.
Do not include any explanations or apologies.
Do not respond to anything other than constructing a Cypher statement.
Do not include any text except the generated Cypher statement.
"""
SCHEMA="""
Node properties:\n- **Country**\n  - `location`: POINT \n  - `code`: STRING Example: "AFG"\n  - `name`: STRING Example: "Afghanistan"\n  - `tld`: STRING Example: "AF"\n- **Filing**\n  - `begin`: DATE_TIME Min: 2000-02-08T00:00:00Z, Max: 2017-09-05T00:00:00Z\n  - `end`: DATE_TIME Min: 2000-02-08T00:00:00Z, Max: 2017-11-03T00:00:00Z\n  - `originator_bank_id`: STRING Example: "cimb-bank-berhad"\n  - `sar_id`: STRING Example: "3297"\n  - `beneficiary_bank`: STRING Example: "Barclays Bank Plc"\n  - `filer_org_name_id`: STRING Example: "the-bank-of-new-york-mellon-corp"\n  - `originator_bank_country`: STRING Example: "Singapore"\n  - `beneficiary_bank_country`: STRING Example: "United Kingdom"\n  - `filer_org_name`: STRING Example: "The Bank of New York Mellon Corp."\n  - `originator_iso`: STRING Example: "SGP"\n  - `beneficiary_bank_id`: STRING Example: "barclays-bank-plc-london-england-gbr"\n  - `origin_lat`: STRING Example: "1.3667"\n  - `origin_lng`: STRING Example: "103.8"\n  - `end_date_format`: STRING Example: "2015-09-25T00:00:00Z"\n  - `begin_date_format`: STRING Example: "2015-03-25T00:00:00Z"\n  - `originator_bank`: STRING Example: "CIMB Bank Berhad"\n  - `beneficiary_lat`: STRING Example: "54"\n  - `beneficiary_iso`: STRING Example: "GBR"\n  - `beneficiary_lng`: STRING Example: "-2"\n  - `begin_date`: STRING Example: "Mar 25, 2015"\n  - `id`: STRING Example: "223254"\n  - `end_date`: STRING Example: "Sep 25, 2015"\n  - `amount`: INTEGER Min: 1.18, Max: 2721000000\n  - `number`: INTEGER Min: 1, Max: 174\n- **Entity**\n  - `id`: STRING Example: "the-bank-of-new-york-mellon-corp"\n  - `location`: POINT \n  - `name`: STRING Example: "The Bank of New York Mellon Corp."\n  - `country`: STRING Example: "CHN"\nRelationship properties:\n\nThe relationships:\n(:Filing)-[:BENEFITS]->(:Entity)\n(:Filing)-[:CONCERNS]->(:Entity)\n(:Filing)-[:ORIGINATOR]->(:Entity)\n(:Entity)-[:FILED]->(:Filing)\n(:Entity)-[:COUNTRY]->(:Country)
"""
QUESTION="""
Which 3 countries have the most entities linked as beneficiaries in filings?
"""
USER_INSTRUCTION = """
Schema:
{schema}
Question:
{question}

"""
# Messages in expected Llama Messages format
messages = [
    {'role': 'system', 'content': SYSTEM_PROMPT}, #This is optional
    {"role": "user", "content": USER_INSTRUCTION.format(schema=SCHEMA, question=QUESTION)}, #this is mandatory (schema and user question in natural language)
    #{'role': 'model', 'content': ""}, #the model will generate cypher for this question
]

#quantization config
quantization_config = BitsAndBytesConfig(load_in_4bit=LOAD_IN_4_BIT)
#load model and tokenizer
quantized_model = AutoModelForCausalLM.from_pretrained(
    FINETUNED_MODEL_NAME, device_map="auto", torch_dtype=torch.bfloat16, quantization_config=quantization_config)
tokenizer = AutoTokenizer.from_pretrained(FINETUNED_MODEL_NAME)
# It is interesting to see model architecture
print (quantized_model)


#Apply the tokenizer chat template to the input messages to the model - do not tokenize yet)
prompt_text = tokenizer.apply_chat_template(messages, add_generation_prompt=True,tokenize=False)

#Turn the prompt into a set of tokens and load tokens to GPU
input_ids = tokenizer([prompt_text], return_tensors = "pt").to("cuda")

#Ask the model to generates cypher 
cypher_output = quantized_model.generate(**input_ids, max_new_tokens=200)
print(tokenizer.decode(cypher_output[0], skip_special_tokens=True))