src/routes/api/scrap-models/+server.ts

/** @type {import('./$types').RequestHandler} */

import { json } from '@sveltejs/kit';
// import moment from 'moment';
import prisma from '$lib/prisma';

export async function POST({ request }) {
  const headers = Object.fromEntries(request.headers.entries());

  if (headers["x-hf-token"] !== process.env.SECRET_HF_TOKEN) {
    return Response.json({
      message: "Wrong castle fam :^)"
    }, { status: 401 });
  }

  const response_models = await fetch(`https://huggingface.co/api/models?limit=200&filter=lora%2Cdiffusers&sort=createdAt&full=true&config=true`)
  const new_models = await response_models.json();
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  let index = 0;
  let model_saved = 0;

  for (const model of new_models) {
    if (index % 50 === 0) {
      await new Promise(resolve => setTimeout(resolve, 2000));
    }
    index++

    const existing_model = await prisma.model.findUnique({
      where: {
        id: model.id
      }
    })

    if (existing_model) {
      continue;
    }

    const hasImages = model?.siblings?.filter((sibling: Record<string, string>) => sibling?.rfilename.endsWith(".png") || sibling?.rfilename.endsWith(".jpeg") || sibling?.rfilename.endsWith(".jpg"))
    if (hasImages.length > 0) {
      model.image = hasImages[1]?.rfilename ? `https://huggingface.co/${model.id}/resolve/main/${hasImages[1]?.rfilename}` : `https://huggingface.co/${model.id}/resolve/main/${hasImages[0]?.rfilename}`
    } else {
      const hasReadme = model?.siblings?.find((sibling: Record<string, string>) => sibling?.rfilename === "README.md")
      if (hasReadme) {
        const readmeRes = await fetch(`https://huggingface.co/${model.id}/raw/main/README.md`)
        const readme = await readmeRes.text().catch(() => null)
        if (readme) {
          const imageRegex = /!\[.*\]\((.*)\)/
          let image = readme.match(imageRegex)?.[1]
      
          if (image) {
            image = image.replace(/&#x2F;/g, "/")
            if (image.startsWith("http")) model.image = image
            else model.image = `https://huggingface.co/${model.id}/resolve/main/${image.replace("./", "")}`
          }
        }
      }
    }

    const base_model = model.tags.find((tag: string) => tag.startsWith("base_model:"))?.split(":")[1] ?? null

    await prisma.model.create({
      data: {
        id: model.id,
        ...(
          model?.image?.startsWith("http") ? {
            image: model.image
          } : {
            image: undefined,
          }
        ),
        likes: model.likes,
        downloads: model.downloads,
        createdAt: model.createdAt,
        isPublic: false,
        ...(base_model ? {
          base_model: base_model
        } : {}
        )
      }
    })
    .then(() => {
      model_saved++
    })
    .catch((error) => {
      console.error(error)
    })
  }

  return json({
    message: `Successfully added ${model_saved} models`,
  })
}