dangvantuan

/

vietnamese-document-embedding

like 13

@Laskytran
You can try this code:

"""
# script: convert-hf-to-gguf.py
A basic script to convert a Hugging Face model's state dict to a simplified GGUF-like binary format.
This script loads a model (using AutoModel from Hugging Face) and writes a binary file containing:
  - A magic header ("GGUF")
  - A JSON-encoded metadata block
  - A list of tensors with their names, shapes, dtype information, and raw binary data

Note: This is a demonstration implementation.
The official GGUF specification might require additional details or a different layout.
"""

import argparse
import json
import struct
import os

import torch
from transformers import AutoModel


def torch_dtype_to_str(dtype: torch.dtype) -> str:
    """Convert a torch dtype to a string representation."""
    if dtype == torch.float32:
        return "float32"
    elif dtype == torch.float16:
        return "float16"
    elif dtype == torch.int8:
        return "int8"
    elif dtype == torch.int16:
        return "int16"
    elif dtype == torch.int32:
        return "int32"
    elif dtype == torch.int64:
        return "int64"
    else:
        return str(dtype)


def convert_hf_to_gguf(model_path: str, output_file: str):
    # Load the model using Hugging Face's AutoModel. This will download the model if necessary.
    print(f"Loading model from {model_path} ...")
    model = AutoModel.from_pretrained(model_path, trust_remote_code=True)
    state_dict = model.state_dict()
    print("Model loaded. Found {} tensors.".format(len(state_dict)))

    # Create metadata (this can be extended as needed)
    metadata = {
        "model_path": model_path,
        "num_tensors": len(state_dict)
    }
    metadata_json = json.dumps(metadata).encode("utf-8")

    # Open the output file in binary write mode
    with open(output_file, "wb") as f:
        # Write a simple header: magic bytes to indicate file type.
        f.write(b"GGUF")

        # Write the length of the metadata and the metadata itself.
        f.write(struct.pack("I", len(metadata_json)))
        f.write(metadata_json)

        # Write the number of tensors (as unsigned int)
        f.write(struct.pack("I", len(state_dict)))

        # Iterate through tensors and write their info and binary data.
        for name, tensor in state_dict.items():
            # Convert tensor to CPU and numpy (if not already)
            tensor = tensor.cpu()
            np_array = tensor.numpy()

            # Write tensor name length and name
            name_bytes = name.encode("utf-8")
            f.write(struct.pack("I", len(name_bytes)))
            f.write(name_bytes)

            # Write the number of dimensions and each dimension size.
            shape = np_array.shape
            f.write(struct.pack("I", len(shape)))
            for dim in shape:
                f.write(struct.pack("I", dim))

            # Write the data type string.
            dtype_str = torch_dtype_to_str(tensor.dtype)
            dtype_bytes = dtype_str.encode("utf-8")
            f.write(struct.pack("I", len(dtype_bytes)))
            f.write(dtype_bytes)

            # Write the number of elements (for verification, if needed)
            num_elements = np_array.size
            f.write(struct.pack("I", num_elements))

            # Write the raw binary data.
            f.write(np_array.tobytes())

    print(f"Conversion complete! GGUF file written to: {output_file}")


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Convert a Hugging Face model to a GGUF-like binary format."
    )
    parser.add_argument("model_path", type=str, help="Path or identifier of the Hugging Face model")
    parser.add_argument("output_file", type=str, help="Output GGUF file path")
    args = parser.parse_args()

    # Check if the output directory exists.
    out_dir = os.path.dirname(args.output_file)
    if out_dir and not os.path.exists(out_dir):
        os.makedirs(out_dir)

    convert_hf_to_gguf(args.model_path, args.output_file)

then:

 python convert-hf-to-gguf.py dangvantuan/vietnamese-document-embedding output.gguf

thank you so much Mr. Tuan