src/routes/api/scrap-models/+server.ts

/** @type {import('./$types').RequestHandler} */

import { json } from '@sveltejs/kit';
// import moment from 'moment';
import prisma from '$lib/prisma';

export async function POST({ request }) {
  const headers = Object.fromEntries(request.headers.entries());

  if (headers["x-hf-token"] !== process.env.SECRET_HF_TOKEN) {
    return Response.json({
      message: "Wrong castle fam :^)"
    }, { status: 401 });
  }

  const response = await fetch(`https://huggingface.co/api/models?limit=5000&filter=lora%2Cdiffusers&sort=createdAt&full=true&config=true`)
  const responseData = await response.json();

  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  let index = 0;
  for (const model of responseData) {
    if (index % 50 === 0) {
      await new Promise(resolve => setTimeout(resolve, 4000));
    }

    const hasReadme = model?.siblings?.find((sibling: Record<string, string>) => sibling?.rfilename === "README.md")
    if (hasReadme) {
      const readmeRes = await fetch(`https://huggingface.co/${model.id}/raw/main/README.md`)
      const readme = await readmeRes.text().catch(() => null)
      if (!readme) {
        continue
      }
      const imageRegex = /!\[.*\]\((.*)\)/
      let image = readme.match(imageRegex)?.[1]

      if (!image) {
        continue
      }
      image = image.replace(/&#x2F;/g, "/")
      if (image.startsWith("http")) model.image = image
      else model.image = `https://huggingface.co/${model.id}/resolve/main/${image.replace("./", "")}`
    }

    await prisma.model.create({
      data: {
        id: model.id,
        image: model.image,
        likes: model.likes,
        downloads: model.downloads,
        isPublic: false,
      }
    }).catch(() => {})

    index++
  }

  const total_items = await prisma.model.count()

  return json({
    message: `Successfully added ${total_items} models`,
  })
}