add cron to update and scrap

package-lock.json

@@ -18,9 +18,12 @@
 				"cookie": "^0.6.0",
 				"cors": "^2.8.5",
 				"desm": "^1.3.1",
+				"dotenv": "^16.4.1",
 				"express": "^4.18.2",
 				"js-cookie": "^3.0.5",
 				"moment": "^2.30.1",
+				"node-cron": "^3.0.3",
+				"node-fetch": "^3.3.2",
 				"svelte-infinite-scroll": "^2.0.1"
 			},
 			"devDependencies": {
@@ -3801,6 +3804,14 @@
 				"node": ">=4"
 			}
 		},
+		"node_modules/data-uri-to-buffer": {
+			"version": "4.0.1",
+			"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
+			"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
+			"engines": {
+				"node": ">= 12"
+			}
+		},
 		"node_modules/debug": {
 			"version": "4.3.4",
 			"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -3933,6 +3944,17 @@
 				"node": ">=6.0.0"
 			}
 		},
+		"node_modules/dotenv": {
+			"version": "16.4.1",
+			"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.1.tgz",
+			"integrity": "sha512-CjA3y+Dr3FyFDOAMnxZEGtnW9KBR2M0JvvUtXNW+dYJL5ROWxP9DUHCwgFqpMk0OXCc0ljhaNTr2w/kutYIcHQ==",
+			"engines": {
+				"node": ">=12"
+			},
+			"funding": {
+				"url": "https://github.com/motdotla/dotenv?sponsor=1"
+			}
+		},
 		"node_modules/ee-first": {
 			"version": "1.1.1",
 			"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -4380,6 +4402,28 @@
 				"reusify": "^1.0.4"
 			}
 		},
+		"node_modules/fetch-blob": {
+			"version": "3.2.0",
+			"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
+			"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
+			"funding": [
+				{
+					"type": "github",
+					"url": "https://github.com/sponsors/jimmywarting"
+				},
+				{
+					"type": "paypal",
+					"url": "https://paypal.me/jimmywarting"
+				}
+			],
+			"dependencies": {
+				"node-domexception": "^1.0.0",
+				"web-streams-polyfill": "^3.0.3"
+			},
+			"engines": {
+				"node": "^12.20 || >= 14.13"
+			}
+		},
 		"node_modules/file-entry-cache": {
 			"version": "6.0.1",
 			"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
@@ -4478,6 +4522,17 @@
 				"is-callable": "^1.1.3"
 			}
 		},
+		"node_modules/formdata-polyfill": {
+			"version": "4.0.10",
+			"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
+			"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
+			"dependencies": {
+				"fetch-blob": "^3.1.2"
+			},
+			"engines": {
+				"node": ">=12.20.0"
+			}
+		},
 		"node_modules/forwarded": {
 			"version": "0.2.0",
 			"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@@ -5335,6 +5390,52 @@
 				"node": ">= 0.6"
 			}
 		},
+		"node_modules/node-cron": {
+			"version": "3.0.3",
+			"resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz",
+			"integrity": "sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==",
+			"dependencies": {
+				"uuid": "8.3.2"
+			},
+			"engines": {
+				"node": ">=6.0.0"
+			}
+		},
+		"node_modules/node-domexception": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
+			"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
+			"funding": [
+				{
+					"type": "github",
+					"url": "https://github.com/sponsors/jimmywarting"
+				},
+				{
+					"type": "github",
+					"url": "https://paypal.me/jimmywarting"
+				}
+			],
+			"engines": {
+				"node": ">=10.5.0"
+			}
+		},
+		"node_modules/node-fetch": {
+			"version": "3.3.2",
+			"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
+			"integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
+			"dependencies": {
+				"data-uri-to-buffer": "^4.0.0",
+				"fetch-blob": "^3.1.4",
+				"formdata-polyfill": "^4.0.10"
+			},
+			"engines": {
+				"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+			},
+			"funding": {
+				"type": "opencollective",
+				"url": "https://opencollective.com/node-fetch"
+			}
+		},
 		"node_modules/node-releases": {
 			"version": "2.0.14",
 			"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz",
@@ -6995,6 +7096,14 @@
 				}
 			}
 		},
+		"node_modules/web-streams-polyfill": {
+			"version": "3.3.2",
+			"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.2.tgz",
+			"integrity": "sha512-3pRGuxRF5gpuZc0W+EpwQRmCD7gRqcDOMt688KmdlDAgAyaB1XlN0zq2njfDNm44XVdIouE7pZ6GzbdyH47uIQ==",
+			"engines": {
+				"node": ">= 8"
+			}
+		},
 		"node_modules/which": {
 			"version": "2.0.2",
 			"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

package.json

@@ -51,9 +51,12 @@
 		"cookie": "^0.6.0",
 		"cors": "^2.8.5",
 		"desm": "^1.3.1",
+		"dotenv": "^16.4.1",
 		"express": "^4.18.2",
 		"js-cookie": "^3.0.5",
 		"moment": "^2.30.1",
+		"node-cron": "^3.0.3",
+		"node-fetch": "^3.3.2",
 		"svelte-infinite-scroll": "^2.0.1"
 	}
 }

server.js

@@ -1,5 +1,10 @@
 import { handler } from './build/handler.js';
 import express from 'express';
+import cron from 'node-cron';
+import fetch from 'node-fetch';
+import dotenv from 'dotenv';
+
+dotenv.config();
 
 const app = express();
 
@@ -11,4 +16,29 @@ app.use(handler);
 
 app.listen(3000, () => {
     console.log('listening on port 3000');
-});
+});
+
+cron.schedule("0 * * * *", async () => {
+    const request = await fetch(`${process.env.SPACE_HOST}/api/scrap-models`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'x-hf-token': process.env.SECRET_HF_TOKEN
+        },
+    });
+    const response = await request.json();
+    console.info(response?.message)
+});
+
+cron.schedule("0 0 * * *", async () => {
+    const request = await fetch(`${process.env.SPACE_HOST}/api/models`, {
+        method: 'PATCH',
+        headers: {
+            'Content-Type': 'application/json',
+            'x-hf-token': process.env.SECRET_HF_TOKEN
+        },
+    });
+    const response = await request.json();
+    console.info(response?.message)
+});
+

src/routes/api/models/+server.ts

@@ -49,3 +49,41 @@ export async function GET(request : RequestEvent) {
     total_items: total_reposId
   })
 }
+
+export async function PATCH({ request } : RequestEvent) {
+  const headers = Object.fromEntries(request.headers.entries());
+
+  if (headers["x-hf-token"] !== process.env.SECRET_HF_TOKEN) {
+    return Response.json({
+      message: "Wrong castle fam :^)"
+    }, { status: 401 });
+  }
+
+  const models = await prisma.model.findMany({
+    where: {
+      isPublic: true,
+    }
+  })
+
+  let total_updates = 0;
+
+  for (const model of models) {
+    const response = await fetch(`https://huggingface.co/api/models/${model.id}`)
+    const hf_model = await response.json();
+
+    await prisma.model.update({
+      where: {
+        id: model.id
+      },
+      data: {
+        likes: hf_model.likes,
+        downloads: hf_model.downloads,
+      }
+    })
+    total_updates++
+  }
+
+  return json({
+    message: `Updated ${total_updates} models`
+  })
+}

src/routes/api/scrap-models/+server.ts

@@ -1,7 +1,7 @@
 /** @type {import('./$types').RequestHandler} */
 
 import { json } from '@sveltejs/kit';
-// import moment from 'moment';
+import moment from 'moment';
 import prisma from '$lib/prisma';
 
 export async function POST({ request }) {
@@ -13,15 +13,20 @@ export async function POST({ request }) {
     }, { status: 401 });
   }
 
-  const response = await fetch(`https://huggingface.co/api/models?limit=5000&filter=lora%2Cdiffusers&sort=createdAt&full=true&config=true`)
-  const responseData = await response.json();
-
+  const response_models = await fetch(`https://huggingface.co/api/models?limit=200&filter=lora%2Cdiffusers&sort=createdAt&full=true&config=true`)
+  const new_models = await response_models.json();
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   let index = 0;
-  for (const model of responseData) {
+  let model_saved = 0;
+
+  for (const model of new_models) {
     if (index % 50 === 0) {
       await new Promise(resolve => setTimeout(resolve, 2000));
     }
+    index++
+    if (moment(model.createdAt).isBefore(moment().subtract(1, "hour"))) {
+      continue;
+    }
 
     const hasImages = model?.siblings?.filter((sibling: Record<string, string>) => sibling?.rfilename.endsWith(".png") || sibling?.rfilename.endsWith(".jpeg") || sibling?.rfilename.endsWith(".jpg"))
     if (hasImages.length > 0) {
@@ -52,16 +57,13 @@ export async function POST({ request }) {
         image: model.image,
         likes: model.likes,
         downloads: model.downloads,
-        isPublic: true,
+        isPublic: false,
       }
     }).catch(() => {})
-
-    index++
+    model_saved++
   }
 
-  const total_items = await prisma.model.count()
-
   return json({
-    message: `Successfully added ${total_items} models`,
+    message: `Successfully added ${model_saved} models`,
   })
 }