import * as Sentry from '@sentry/node' ; import { Span } from '@sentry/tracing' ; import { KaitaiStream } from 'kaitai-struct' ; import { Album , Artist , Artwork , Color , EntityFK , Genre , Key , Label , Playlist , PlaylistEntry , Track , } from 'src/entities' ; import RekordboxAnlz from 'src/localdb/kaitai/rekordbox_anlz.ksy' ; import RekordboxPdb from 'src/localdb/kaitai/rekordbox_pdb.ksy' ; import { MetadataORM , Table } from 'src/localdb/orm' ; import { makeCueLoopEntry } from 'src/localdb/utils' ; import { BeatGrid , CueAndLoop , HotcueButton , WaveformHD } from 'src/types' ; import { convertWaveformHDData } from 'src/utils/converters' ; /**
 * The provided function should resolve ANLZ files into buffers. Typically
 * you would just read the file, but in the case of the prolink network, this
 * would handle loading the file over NFS.
 */ type AnlzResolver = ( path : string ) => Promise < Buffer > ; /**
 * Data returned from loading DAT anlz files
 */ type AnlzResponseDAT = { /**
     * Embedded beat grid information
     */ beatGrid : BeatGrid | null ; /**
     * Embedded cue and loop information
     */ cueAndLoops : CueAndLoop [ ] | null ; } ; /**
 * Data returned from loading EXT anlz files
 */ type AnlzResponseEXT = { /**
     * HD Waveform information
     */ waveformHd : WaveformHD | null ; } ; type AnlzResponse = { DAT : AnlzResponseDAT ; EXT : AnlzResponseEXT ; } ; /**
 * Details about the current state of the hydtration task
 */ export type HydrationProgress = { /**
     * The specific table that progress is being reported for
     */ table : string ; /**
     * The total progress steps for this table
     */ total : number ; /**
     * The completed number of progress steps
     */ complete : number ; } ; /**
 * Options to hydrate the database
 */ type Options = { /**
     * The metadata ORM of which the tables will be hydrated
     */ orm : MetadataORM ; /**
     * This buffer should contain the Rekordbox pdb file contents. It will be
     * used to do the hydration
     */ pdbData : Buffer ; /**
     * Sentry tracing span for the parent transaction
     */ span ? : Span ; /**
     * For larger music collections, it may take some time to load everything,
     * especially when limited by IO. When hydration progresses this function
     * will be called.
     */ onProgress ? : ( progress : HydrationProgress ) => void ; } ; /**
 * Given a rekordbox pdb file contents. This function will hydrate the provided
 * database with all entities from the Rekordbox database. This includes all
 * track metadata, including analyzed metadata (such as beatgrids and waveforms).
 */ export async function hydrateDatabase ( { pdbData , span , ... options } ) { const hydrator = new RekordboxHydrator ( options ) ; await hydrator . hydrateFromPdb ( pdbData , span ) ; } /**
 * Loads the ANLZ data of a Track entity from the analyzePath.
 */ export async function loadAnlz < T extends keyof AnlzResponse > ( track , type , anlzResolver ) { const path = `${ track . analyzePath }.${ type }` ; const anlzData = await anlzResolver ( path ) ; const stream = new KaitaiStream ( anlzData ) ; const anlz = new RekordboxAnlz ( stream ) ; const result = { } as AnlzResponse [ T ] ; const resultDat = result as AnlzResponseDAT ; const resultExt = result as AnlzResponseEXT ; for ( const section of anlz . sections ) { if ( section . fourcc === SectionTags . BEAT_GRID ) { resultDat . beatGrid = makeBeatGrid ( section ) ; continue ; } if ( section . fourcc === SectionTags . CUES ) { resultDat . cueAndLoops = makeCueAndLoop ( section ) ; continue ; } if ( section . fourcc === SectionTags . WAVE_COLOR_SCROLL ) { resultExt . waveformHd = makeWaveformHd ( section ) ; continue ; } } return result ; } /**
 * This service provides utilities for translating rekordbox database (pdb_ and
 * analysis (ANLZ) files into the common entity types used in this library.
 */ class RekordboxHydrator { #orm : MetadataORM ; #onProgress : ( progress : HydrationProgress ) => void ; constructor ( { orm , onProgress } ) { this . #orm = orm ; this . #onProgress = onProgress ?? ( ( ) => null ) ; } /**
     * Extract entries from a rekordbox pdb file and hydrate the passed database
     * connection with entities derived from the rekordbox entries.
     */ async hydrateFromPdb ( pdbData , span ? ) { const tx = span ? span . startChild ( { op : 'hydrateFromPdb' } ) : Sentry . startTransaction ( { name : 'hydrateFromPdb' } ) ; const parseTx = tx . startChild ( { op : 'parsePdbData' , data : { size : pdbData . length } } ) ; const stream = new KaitaiStream ( pdbData ) ; const db = new RekordboxPdb ( stream ) ; parseTx . finish ( ) ; const hydrateTx = tx . startChild ( { op : 'hydration' } ) ; await Promise . all ( db . tables . map ( ( table ) => this . hydrateFromTable ( table , hydrateTx ) ) ) ; hydrateTx . finish ( ) ; tx . finish ( ) ; } /**
     * Hydrate the database with entities from the provided RekordboxPdb table.
     * See pdbEntityCreators for how tables are mapped into database entities.
     */ async hydrateFromTable ( table , span ) { const tableName = pdbTables [ table . type ] ; const createObject = pdbEntityCreators [ table . type ] ; const tx = span . startChild ( { op : 'hydrateFromTable' , description : tableName } ) ; if ( createObject === undefined ) { return ; } let totalSaved = 0 ; let totalItems = 0 ; for ( const _ of tableRows ( table ) ) { totalItems ++ ; } tx . setData ( 'items' , totalItems ) ; for ( const row of tableRows ( table ) ) { this . #orm . insertEntity ( tableName , createObject ( row ) ) ; this . #onProgress ( { complete : ++ totalSaved , table : tableName , total : totalItems } ) ; await new Promise ( r => setTimeout ( r , 0 ) ) ; } tx . finish ( ) ; } } /**
 * Utility generator that pages through a table and yields every present row.
 * This flattens the concept of rowGroups and refs.
 */ function * tableRows ( table ) { const { firstPage , lastPage } = table ; let pageRef = firstPage ; do { const page = pageRef . body ; pageRef = page . nextPage ; if ( ! page . isDataPage ) { continue ; } const rows = page . rowGroups . map ( ( group ) => group . rows ) . flat ( ) . filter ( ( row ) => row . present ) ; for ( const row of rows ) { yield row . body ; } } while ( pageRef . index <= lastPage . index ) ; } type IdAndNameEntity = { id : number ; name : string ; } ; const ensureDate = ( date ) => date instanceof Date && ! isNaN ( date . valueOf ( ) ) ? date : undefined ; /**
 * Utility to create a hydrator that hydrates the provided entity with the id
 * and name properties from the row.
 */ const makeIdNameHydrator = < T extends IdAndNameEntity > ( ) => ( row ) => ( { id : row . id , name : row . name . body . text ?? '' , } as T ) ; /**
 * Translates a pdb track row entry to a  Track entity.
  function createTrack ( trackRow ) { const analyzePath = trackRow . analyzePath . body . text ; const track = { id : trackRow . id , title : trackRow . title . body . text , trackNumber : trackRow . trackNumber , discNumber : trackRow . discNumber , duration : trackRow . duration , sampleRate : trackRow . sampleRate , sampleDepth : trackRow . sampleDepth , bitrate : trackRow . bitrate , tempo : trackRow . tempo / 100 , playCount : trackRow . playCount , year : trackRow . year , rating : trackRow . rating , mixName : trackRow . mixName . body . text , comment : trackRow . comment . body . text , autoloadHotcues : trackRow . autoloadHotcues . body . text === 'ON' , kuvoPublic : trackRow . kuvoPublic . body . text === 'ON' , filePath : trackRow . filePath . body . text , fileName : trackRow . filename . body . text , fileSize : trackRow . fileSize , releaseDate : trackRow . releaseDate . body . text , analyzeDate : ensureDate ( new Date ( trackRow . analyzeDate . body . text ) ) , dateAdded : ensureDate ( new Date ( trackRow . dateAdded . body . text ) ) , analyzePath : analyzePath ?. substring ( 0 , analyzePath . length - 4 ) , artworkId : trackRow . artworkId || null , artistId : trackRow . artistId || null , originalArtistId : trackRow . originalArtistId || null , remixerId : trackRow . remixerId || null , composerId : trackRow . composerId || null , albumId : trackRow . albumId || null , labelId : trackRow . labelId || null , genreId : trackRow . genreId || null , colorId : trackRow . colorId || null , keyId : trackRow . keyId || null , beatGrid : null , cueAndLoops : null , waveformHd : null , } ; return track ; } /**
 * Translates a pdb playlist row entry into a  Playlist entity.
  function createPlaylist ( playlistRow ) { const playlist = { id : playlistRow . id , name : playlistRow . name . body . text , isFolder : playlistRow . rawIsFolder !== 0 , parentId : playlistRow . parentId || null , } ; return playlist ; } /**
 * Translates a pdb playlist track entry into a  PlaylistTrack entity.
  function createPlaylistEntry ( playlistTrackRow ) { const entry = { id : playlistTrackRow . id , sortIndex : playlistTrackRow . entryIndex , playlistId : playlistTrackRow . playlistId , trackId : playlistTrackRow . trackId , } ; return entry ; } /**
 * Translates a pdb artwork entry into a  Artwork entity.
  function createArtworkEntry ( artworkRow ) { const art = { id : artworkRow . id , path : artworkRow . path . body . text , } ; return art ; } /**
 * Fill beatgrid data from the ANLZ section
 */ function makeBeatGrid ( data ) { return data . body . beats . map ( ( beat ) => ( { offset : beat . time , bpm : beat . tempo / 100 , count : beat . beatNumber , } ) ) ; } /**
 * Fill cue and loop data from the ANLZ section
 */ function makeCueAndLoop ( data ) { return data . body . cues . map ( ( entry ) => { const button = entry . hotCue === 0 ? false : ( entry . type as HotcueButton ) ; const isCue = entry . type === 0x01 ; const isLoop = entry . type === 0x02 ; const offset = entry . time ; const length = entry . loopTime - offset ; return makeCueLoopEntry ( isCue , isLoop , offset , length , button ) ; } ) ; } /**
 * Fill waveform HD data from the ANLZ section
 */ function makeWaveformHd ( data ) { return convertWaveformHDData ( Buffer . from ( data . body . entries ) ) ; } const { PageType } = RekordboxPdb ; const { SectionTags } = RekordboxAnlz ; /**
 * Maps rekordbox pdb table types to orm table names.
 */ const pdbTables = { [ PageType . TRACKS ] : Table . Track , [ PageType . ARTISTS ] : Table . Artist , [ PageType . GENRES ] : Table . Genre , [ PageType . ALBUMS ] : Table . Album , [ PageType . LABELS ] : Table . Label , [ PageType . COLORS ] : Table . Color , [ PageType . KEYS ] : Table . Key , [ PageType . ARTWORK ] : Table . Artwork , [ PageType . PLAYLIST_TREE ] : Table . Playlist , [ PageType . PLAYLIST_ENTRIES ] : Table . PlaylistEntry , } ; /**
 * Maps rekordbox pdb table types to functions that create entity objects for
 * the passed pdb row.
 */ const pdbEntityCreators = { [ PageType . TRACKS ] : createTrack , [ PageType . ARTISTS ] : makeIdNameHydrator < Artist > ( ) , [ PageType . GENRES ] : makeIdNameHydrator < Genre > ( ) , [ PageType . ALBUMS ] : makeIdNameHydrator < Album > ( ) , [ PageType . LABELS ] : makeIdNameHydrator < Label > ( ) , [ PageType . COLORS ] : makeIdNameHydrator < Color > ( ) , [ PageType . KEYS ] : makeIdNameHydrator < Key > ( ) , [ PageType . ARTWORK ] : createArtworkEntry , [ PageType . PLAYLIST_TREE ] : createPlaylist , [ PageType . PLAYLIST_ENTRIES ] : createPlaylistEntry , } ;