piclets / src /lib /db /trainerScanning.ts
Fraser's picture
LOGS
c0fc1ad
raw
history blame
10.1 kB
import { db } from './index';
import type { TrainerScanProgress } from './schema';
// Initialize trainer scan progress records from paths
export async function initializeTrainerScanProgress(imagePaths: string[]): Promise<void> {
try {
console.log('πŸ” initializeTrainerScanProgress: Starting with', imagePaths.length, 'paths');
console.log('πŸ” initializeTrainerScanProgress: First few paths:', imagePaths.slice(0, 3));
console.log('πŸ” initializeTrainerScanProgress: Creating transaction...');
const tx = db.transaction([TRAINER_SCAN_STORE], 'readwrite');
console.log('πŸ” initializeTrainerScanProgress: Transaction created');
const store = tx.objectStore(TRAINER_SCAN_STORE);
console.log('πŸ” initializeTrainerScanProgress: Got object store');
let processedCount = 0;
let skippedCount = 0;
for (let i = 0; i < imagePaths.length; i++) {
const imagePath = imagePaths[i];
try {
console.log(`πŸ” initializeTrainerScanProgress: Processing path ${i}: "${imagePath}" (type: ${typeof imagePath})`);
if (typeof imagePath !== 'string') {
console.error(`❌ initializeTrainerScanProgress: Path at index ${i} is not a string:`, imagePath);
skippedCount++;
continue;
}
// Extract trainer name and image index from path
// Format: "trainer_images/001_Willow_Snap/image_001.jpg"
console.log(`πŸ” initializeTrainerScanProgress: Splitting path: "${imagePath}"`);
const pathParts = imagePath.split('/');
console.log(`πŸ” initializeTrainerScanProgress: Path parts:`, pathParts);
if (pathParts.length < 3) {
console.warn(`⚠️ Skipping invalid path format: ${imagePath} (length: ${pathParts.length})`);
skippedCount++;
continue;
}
const rawTrainerName = pathParts[1];
const rawImageFile = pathParts[2];
console.log(`πŸ” initializeTrainerScanProgress: Raw parts - trainer: "${rawTrainerName}" (${typeof rawTrainerName}), image: "${rawImageFile}" (${typeof rawImageFile})`);
const trainerName = rawTrainerName?.trim?.(); // Safe call with optional chaining
const imageFile = rawImageFile?.trim?.(); // Safe call with optional chaining
console.log(`πŸ” initializeTrainerScanProgress: Trimmed parts - trainer: "${trainerName}" (${typeof trainerName}), image: "${imageFile}" (${typeof imageFile})`);
if (!trainerName || !imageFile || typeof trainerName !== 'string' || typeof imageFile !== 'string') {
console.warn(`⚠️ Skipping path with missing or invalid parts: ${imagePath}`, {
trainerName,
imageFile,
trainerType: typeof trainerName,
imageType: typeof imageFile
});
skippedCount++;
continue;
}
console.log(`πŸ” initializeTrainerScanProgress: Matching image file: "${imageFile}"`);
const imageMatch = imageFile.match(/image_(\d+)\.jpg/);
const imageIndex = imageMatch ? parseInt(imageMatch[1]) : 1;
console.log(`πŸ” initializeTrainerScanProgress: Image index: ${imageIndex}`);
const remoteUrl = `https://huggingface.co/datasets/Fraser/piclets/resolve/main/${imagePath}`;
console.log(`πŸ” initializeTrainerScanProgress: Remote URL: ${remoteUrl}`);
// Check if this path already exists to avoid duplicates
console.log(`πŸ” initializeTrainerScanProgress: Checking for existing record...`);
const existing = await store.get(imagePath);
console.log(`πŸ” initializeTrainerScanProgress: Existing record:`, existing ? 'found' : 'not found');
if (!existing) {
const progressRecord: Omit<TrainerScanProgress, 'id'> = {
imagePath,
trainerName,
imageIndex,
status: 'pending',
remoteUrl
};
console.log(`πŸ” initializeTrainerScanProgress: Adding record:`, progressRecord);
await store.add(progressRecord);
console.log(`πŸ” initializeTrainerScanProgress: Record added successfully`);
processedCount++;
} else {
// Record exists, don't count as processed but note it
processedCount++;
}
// Log progress every 100 items
if (i % 100 === 0) {
console.log(`πŸ” initializeTrainerScanProgress: Progress: ${i}/${imagePaths.length} (${Math.round((i/imagePaths.length)*100)}%)`);
}
} catch (error) {
console.error(`❌ Failed to process path entry at index ${i}: "${imagePath}"`, error);
console.error('❌ Error details:', {
message: error instanceof Error ? error.message : 'Unknown error',
stack: error instanceof Error ? error.stack : 'No stack trace',
pathValue: imagePath,
pathType: typeof imagePath
});
skippedCount++;
// Continue processing other paths despite this failure
}
}
console.log(`βœ… Trainer scan initialization complete: ${processedCount} records processed, ${skippedCount} skipped`);
console.log('πŸ” initializeTrainerScanProgress: Completing transaction...');
await tx.complete;
console.log('πŸ” initializeTrainerScanProgress: Transaction completed successfully');
} catch (error) {
console.error('❌ initializeTrainerScanProgress: Fatal error during initialization:', error);
console.error('❌ initializeTrainerScanProgress: Error details:', {
message: error instanceof Error ? error.message : 'Unknown error',
stack: error instanceof Error ? error.stack : 'No stack trace'
});
throw error;
}
}
// Get next pending image to process
export async function getNextPendingImage(): Promise<TrainerScanProgress | null> {
const tx = db.transaction([TRAINER_SCAN_STORE], 'readonly');
const store = tx.objectStore(TRAINER_SCAN_STORE);
// Get all pending records
const pendingRecords: TrainerScanProgress[] = [];
let cursor = await store.openCursor();
while (cursor) {
if (cursor.value.status === 'pending') {
pendingRecords.push(cursor.value);
}
cursor = await cursor.continue();
}
// Return the first pending record (if any)
return pendingRecords.length > 0 ? pendingRecords[0] : null;
}
// Update scan progress status
export async function updateScanProgress(
imagePath: string,
updates: Partial<Omit<TrainerScanProgress, 'id' | 'imagePath'>>
): Promise<void> {
const tx = db.transaction([TRAINER_SCAN_STORE], 'readwrite');
const store = tx.objectStore(TRAINER_SCAN_STORE);
const existing = await store.get(imagePath);
if (existing) {
const updated = { ...existing, ...updates };
await store.put(updated);
}
await tx.complete;
}
// Mark image processing as started
export async function markImageProcessingStarted(imagePath: string): Promise<void> {
await updateScanProgress(imagePath, {
status: 'processing',
startedAt: new Date()
});
}
// Mark image processing as completed successfully
export async function markImageProcessingCompleted(
imagePath: string,
picletInstanceId: number
): Promise<void> {
await updateScanProgress(imagePath, {
status: 'completed',
picletInstanceId,
completedAt: new Date()
});
}
// Mark image processing as failed
export async function markImageProcessingFailed(
imagePath: string,
errorMessage: string
): Promise<void> {
await updateScanProgress(imagePath, {
status: 'failed',
errorMessage,
completedAt: new Date()
});
}
// Get scanning statistics
export async function getScanningStats(): Promise<{
total: number;
pending: number;
processing: number;
completed: number;
failed: number;
}> {
const tx = db.transaction([TRAINER_SCAN_STORE], 'readonly');
const store = tx.objectStore(TRAINER_SCAN_STORE);
const stats = {
total: 0,
pending: 0,
processing: 0,
completed: 0,
failed: 0
};
let cursor = await store.openCursor();
while (cursor) {
stats.total++;
const status = cursor.value.status;
stats[status]++;
cursor = await cursor.continue();
}
return stats;
}
// Get all completed scans for a specific trainer
export async function getCompletedScansForTrainer(trainerName: string): Promise<TrainerScanProgress[]> {
const tx = db.transaction([TRAINER_SCAN_STORE], 'readonly');
const store = tx.objectStore(TRAINER_SCAN_STORE);
const results: TrainerScanProgress[] = [];
let cursor = await store.openCursor();
while (cursor) {
const record = cursor.value;
if (record.trainerName === trainerName && record.status === 'completed') {
results.push(record);
}
cursor = await cursor.continue();
}
return results;
}
// Reset all failed scans back to pending (for retry)
export async function resetFailedScans(): Promise<number> {
const tx = db.transaction([TRAINER_SCAN_STORE], 'readwrite');
const store = tx.objectStore(TRAINER_SCAN_STORE);
let resetCount = 0;
let cursor = await store.openCursor();
while (cursor) {
if (cursor.value.status === 'failed') {
const updated = {
...cursor.value,
status: 'pending' as const,
errorMessage: undefined,
startedAt: undefined,
completedAt: undefined
};
await cursor.update(updated);
resetCount++;
}
cursor = await cursor.continue();
}
await tx.complete;
return resetCount;
}
// Get current processing status (for resuming interrupted sessions)
export async function getCurrentProcessingImage(): Promise<TrainerScanProgress | null> {
const tx = db.transaction([TRAINER_SCAN_STORE], 'readonly');
const store = tx.objectStore(TRAINER_SCAN_STORE);
let cursor = await store.openCursor();
while (cursor) {
if (cursor.value.status === 'processing') {
return cursor.value;
}
cursor = await cursor.continue();
}
return null;
}