import { AbstractDatastore , AbstractPrefsProvider , BinaryFileData , Datastore , DatastoreCapabilities , DatastoreInfo , DatastoreOverview , DefaultWriteFileOpts , DeleteResult , DocMetaSnapshotEventListener , ErrorListener , InitResult , PrefsProvider , SnapshotResult , WriteFileOpts , WriteOpts } from './Datastore' ; import { isPresent , Preconditions } from 'polar-shared/src/Preconditions' ; import { Logger } from 'polar-shared/src/logger/Logger' ; import { DocMetaFileRef , DocMetaRef } from './DocMetaRef' ; import { FileDeleted , FileHandle , Files } from 'polar-shared/src/util/Files' ; import { FilePaths } from 'polar-shared/src/util/FilePaths' ; import { Directories } from './Directories' ; import fs from 'fs' ; import os from 'os' ; import { Backend } from 'polar-shared/src/datastore/Backend' ; import { DocFileMeta } from 'polar-shared/src/datastore/DocFileMeta' ; import { Optional } from 'polar-shared/src/util/ts/Optional' ; import { Platform , Platforms } from "polar-shared/src/util/Platforms" ; import { DatastoreFiles } from './DatastoreFiles' ; import { DatastoreMutation , DefaultDatastoreMutation } from './DatastoreMutation' ; import { Datastores } from './Datastores' ; import { NULL_FUNCTION } from 'polar-shared/src/util/Functions' ; import { ISODateTimeStrings } from 'polar-shared/src/metadata/ISODateTimeStrings' ; import { Stopwatches } from 'polar-shared/src/util/Stopwatches' ; import { DictionaryPrefs , IPersistentPrefs , StringToPrefDict } from '../util/prefs/Prefs' ; import { DatastoreMutations } from './DatastoreMutations' ; import { IDocInfo } from 'polar-shared/src/metadata/IDocInfo' ; import { IDocMeta } from "polar-shared/src/metadata/IDocMeta" ; import { FileRef } from "polar-shared/src/datastore/FileRef" ; import { Strings } from "polar-shared/src/util/Strings" ; import { Mutexes } from "polar-shared/src/util/Mutexes" ; import { DocMetas } from "../metadata/DocMetas" ; import { GetFileOpts , NetworkLayer } from "polar-shared/src/datastore/IDatastore" ; const log = Logger . create ( ) ; const writeMutex = Mutexes . create ( ) ; export class DiskDatastore extends AbstractDatastore implements Datastore { public readonly id = 'disk' ; public readonly dataDir : string ; public readonly stashDir : string ; public readonly filesDir : string ; public readonly logsDir : string ; public readonly dataDirConfig : DataDirConfig ; public readonly directories : Directories ; private readonly diskPersistentPrefsBacking : DiskPersistentPrefsBacking ; private prefs : DiskPersistentPrefsProviderImpl | undefined ; constructor ( ) { super ( ) ; this . directories = new Directories ( ) ; this . dataDir = this . directories . dataDir ; this . dataDirConfig = this . directories . dataDirConfig ; this . stashDir = this . directories . stashDir ; this . filesDir = this . directories . filesDir ; this . logsDir = this . directories . logsDir ; this . diskPersistentPrefsBacking = new DiskPersistentPrefsBacking ( this . directories ) ; } public async init ( errorListener = NULL_FUNCTION ) { const diskInitResult = await this . directories . init ( ) ; const doInitInfo = async ( ) => { const hasDatastoreInfo = async ( ) => { const datastoreInfo = await this . info ( ) ; return datastoreInfo . isPresent ( ) ; } ; if ( await hasDatastoreInfo ( ) ) { return ; } const stopwatch = Stopwatches . create ( ) ; const docMetaRefs = await this . getDocMetaRefs ( ) ; const addedValues = [ ] ; for ( const docMetaRef of docMetaRefs ) { const data = await this . getDocMeta ( docMetaRef . fingerprint ) ; if ( data ) { try { const docMeta = JSON . parse ( data ) ; if ( docMeta && docMeta . docInfo && docMeta . docInfo . added ) { addedValues . push ( docMeta . docInfo . added ) ; } } catch ( e ) { log . warn ( "Unable to parse doc meta with fingerprint: " + docMetaRef . fingerprint ) ; } } } const created = addedValues . length > 0 ? addedValues . sort ( ) [ 0 ] : ISODateTimeStrings . create ( ) ; const datastoreInfo = { created } ; const msg = "Writing new datastore info: " + JSON . stringify ( datastoreInfo ) ; log . info ( msg ) ; await this . writeInfo ( datastoreInfo ) ; log . info ( msg + " ... " + stopwatch . stop ( ) ) ; } ; await doInitInfo ( ) ; await this . initPrefs ( ) ; return diskInitResult ; } private async initPrefs ( ) { if ( ! this . prefs ) { await this . diskPersistentPrefsBacking . init ( ) ; this . prefs = new DiskPersistentPrefsProviderImpl ( this . diskPersistentPrefsBacking ) ; } } public async stop ( ) { } /**
     * Return true if the DiskDatastore contains a document for the given
     * fingerprint.
     */ public async contains ( fingerprint ) { const docDir = FilePaths . join ( this . dataDir , fingerprint ) ; if ( ! await Files . existsAsync ( docDir ) ) { return false ; } const statePath = FilePaths . join ( docDir , 'state.json' ) ; return await Files . existsAsync ( statePath ) ; } /**
     * Delete the DocMeta file and the underlying doc from the stash.
     *
     */ public async delete ( docMetaFileRef , datastoreMutation = new DefaultDatastoreMutation ( ) ) { const deleteDelegate = async ( ) => { const docDir = FilePaths . join ( this . dataDir , docMetaFileRef . fingerprint ) ; const statePath = FilePaths . join ( docDir , 'state.json' ) ; let deleteFilePromise = Promise . resolve ( ) ; if ( docMetaFileRef . docFile && docMetaFileRef . docFile . name ) { deleteFilePromise = this . deleteFile ( Backend . STASH , docMetaFileRef . docFile ) ; } log . info ( `Deleting statePath ${ statePath } and file: ` , docMetaFileRef . docFile ) ; const deleteStatePathPromise = Files . deleteAsync ( statePath ) ; log . debug ( "Waiting for state delete..." ) ; const docMetaFile = await deleteStatePathPromise ; log . debug ( "Waiting for state delete...done" ) ; log . debug ( "Waiting for file delete..." ) ; await deleteFilePromise ; log . debug ( "Waiting for file delete...done" ) ; return { docMetaFile } ; } ; return await DatastoreMutations . handle ( async ( ) => deleteDelegate ( ) , datastoreMutation , ( ) => true ) ; } /**
     * Get the DocMeta object we currently in the datastore for this given
     * fingerprint or null if it does not exist.
     */ public async getDocMeta ( fingerprint ) { const docDir = FilePaths . join ( this . dataDir , fingerprint ) ; const statePath = FilePaths . join ( docDir , 'state.json' ) ; if ( ! await this . contains ( fingerprint ) ) { return null ; } const statePathStat = await Files . statAsync ( statePath ) ; if ( ! statePathStat . isFile ( ) ) { log . error ( "Path is not a file: " , statePath ) ; return null ; } const canAccess = await Files . accessAsync ( statePath , fs . constants . R_OK | fs . constants . W_OK ) . then ( ( ) => true ) . catch ( ( ) => false ) ; if ( ! canAccess ) { log . error ( "No access: " , statePath ) ; return null ; } const buffer = await Files . readFileAsync ( statePath ) ; return buffer . toString ( 'utf8' ) ; } public async writeFile ( backend , ref , data , opts = new DefaultWriteFileOpts ( ) ) { if ( ! isPresent ( data ) ) { if ( opts . updateMeta ) { return this . getFile ( backend , ref ) ; } else { throw new Error ( "No data present" ) ; } } const meta = opts . meta || { } ; DatastoreFiles . assertSanitizedFileName ( ref ) ; const fileReference = this . createFileReference ( backend , ref ) ; await Files . createDirAsync ( fileReference . dir ) ; type DiskBinaryFileData = FileHandle | Buffer | string | NodeJS . ReadableStream ; const diskData = < DiskBinaryFileData > data ; await Files . writeFileAsync ( fileReference . path , diskData , { existing : 'link' } ) ; await Files . writeFileAsync ( fileReference . metaPath , JSON . stringify ( meta , null , '  ' ) , { atomic : true } ) ; return this . createDatastoreFile ( backend , ref , fileReference ) ; } public getFile ( backend , ref , opts = { } ) { Datastores . assertNetworkLayer ( this , opts . networkLayer ) ; DatastoreFiles . assertSanitizedFileName ( ref ) ; const fileReference = this . createFileReference ( backend , ref ) ; return this . createDatastoreFile ( backend , ref , fileReference ) ; } public containsFile ( backend , ref ) { DatastoreFiles . assertSanitizedFileName ( ref ) ; const fileReference = this . createFileReference ( backend , ref ) ; return Files . existsAsync ( fileReference . path ) ; } public async deleteFile ( backend , ref ) { DatastoreFiles . assertSanitizedFileName ( ref ) ; const fileReference = this . createFileReference ( backend , ref ) ; await Files . removeAsync ( fileReference . path ) ; await Files . removeAsync ( fileReference . metaPath ) ; } /**
     * Write the datastore to disk.
     */ public async write ( fingerprint , data , docInfo , opts = { } ) { const datastoreMutation = opts . datastoreMutation || new DefaultDatastoreMutation ( ) ; const writeDelegate = async ( ) => { await this . handleWriteFile ( opts ) ; Preconditions . assertPresent ( data , "data" ) ; Preconditions . assertTypeOf ( data , "string" , "data" , ( ) => log . error ( "Failed with data: " , data ) ) ; if ( data . length === 0 ) { throw new Error ( "Invalid data" ) ; } if ( data [ 0 ] !== '{' ) { throw new Error ( "Not JSON" ) ; } log . info ( "Performing sync of content into disk datastore" ) ; const docDir = FilePaths . join ( this . dataDir , fingerprint ) ; await Files . createDirAsync ( docDir ) ; log . debug ( "Calling stat on docDir: " + docDir ) ; const stat = await Files . statAsync ( docDir ) ; if ( ! stat . isDirectory ( ) ) { throw new Error ( "Path is not a directory: " + docDir ) ; } const statePath = FilePaths . join ( docDir , "state.json" ) ; log . info ( `Writing data to state file: ${ statePath }` ) ; await Files . writeFileAsync ( statePath , data , { encoding : 'utf8' , atomic : true } ) ; } ; await writeMutex . execute ( async ( ) => { await DatastoreMutations . handle ( async ( ) => writeDelegate ( ) , datastoreMutation , ( ) => true ) ; } ) ; } public async getDocMetaRefs ( ) { if ( ! await Files . existsAsync ( this . dataDir ) ) { return [ ] ; } const fingerprints = await Files . readdirAsync ( this . dataDir ) ; const result = [ ] ; for ( const fingerprint of fingerprints ) { const docMetaDir = FilePaths . join ( this . dataDir , fingerprint ) ; const docMetaDirStat = await Files . statAsync ( docMetaDir ) ; if ( docMetaDirStat . isDirectory ( ) ) { const stateFile = FilePaths . join ( this . dataDir , fingerprint , 'state.json' ) ; const exists = await Files . existsAsync ( stateFile ) ; if ( exists ) { result . push ( { fingerprint , docMetaProvider : async ( ) => { const data = await this . getDocMeta ( fingerprint ) ; return DocMetas . deserialize ( data ! , fingerprint ) ; } } ) ; } } } return result ; } public async snapshot ( docMetaSnapshotEventListener , errorListener = NULL_FUNCTION ) { return Datastores . createCommittedSnapshot ( this , docMetaSnapshotEventListener ) ; } public async createBackup ( ) { const dataDir = this . directories . dataDir ; const now = new Date ( ) ; const ordYear = now . getUTCFullYear ( ) ; const ordMonth = now . getUTCMonth ( ) + 1 ; const ordDay = now . getUTCDate ( ) ; const year = Strings . lpad ( ordYear , '0' , 4 ) ; const month = Strings . lpad ( ordMonth , '0' , 2 ) ; const day = Strings . lpad ( ordDay , '0' , 2 ) ; const backupDir = FilePaths . join ( dataDir , `.backup-${ year }-${ month }-${ day }` ) ; const acceptPredicate = ( path ) => { return path . indexOf ( ".backup-" ) === - 1 ; } ; if ( await Files . existsAsync ( backupDir ) ) { log . warn ( "Not creating backup.  Already exists: " , backupDir ) ; } else { log . notice ( "Creating backup to: " + backupDir ) ; await Files . createDirectorySnapshot ( dataDir , backupDir , acceptPredicate ) ; } } public addDocMetaSnapshotEventListener ( docMetaSnapshotEventListener ) { } /**
     * Get the info from the datastore.
     */ public async info ( ) { const infoPath = FilePaths . join ( this . dataDir , 'info.json' ) ; if ( await Files . existsAsync ( infoPath ) ) { const data = await Files . readFileAsync ( infoPath ) ; try { const result = < DatastoreInfo > JSON . parse ( data . toString ( 'utf-8' ) ) ; return Optional . of ( result ) ; } catch ( e ) { await Files . deleteAsync ( infoPath ) ; log . warn ( "Unable to read info.json file." ) ; return Optional . empty ( ) ; } } return Optional . empty ( ) ; } public async overview ( ) { const docMetaRefs = await this . getDocMetaRefs ( ) ; const datastoreInfo = await this . info ( ) ; const created = datastoreInfo . map ( info => info . created ) . getOrUndefined ( ) ; return { nrDocs : docMetaRefs . length , created } ; } public capabilities ( ) { const networkLayers = new Set < NetworkLayer > ( [ 'local' ] ) ; return { networkLayers , permission : { mode : 'rw' } } ; } private async writeInfo ( datastoreInfo ) { const infoPath = FilePaths . join ( this . dataDir , 'info.json' ) ; const json = JSON . stringify ( datastoreInfo , null , "  " ) ; await Files . writeFileAsync ( infoPath , json , { atomic : true } ) ; } public getPrefs ( ) { if ( this . prefs ) { return this . prefs ; } else { throw new Error ( "No prefs. Not initialized yet." ) ; } } private createDatastoreFile ( backend , ref , fileReference ) { const fileURL = FilePaths . toURL ( fileReference . path ) ; const url = new URL ( fileURL ) ; return { backend , ref , url : url . href , } ; } private createFileReference ( backend , ref ) { let dir ; if ( backend === Backend . STASH ) { dir = FilePaths . join ( this . dataDir , backend . toString ( ) . toLowerCase ( ) ) ; } else { dir = FilePaths . join ( this . filesDir , backend . toString ( ) . toLowerCase ( ) ) ; } const path = FilePaths . join ( dir , ref . name ) ; const metaPath = FilePaths . join ( dir , ref . name + '.meta' ) ; return { dir , path , metaPath } ; } public static getDataDirs ( ) { const userHome = this . getUserHome ( ) ; const platform = Platforms . get ( ) ; return this . getDataDirsForPlatform ( { userHome , platform } ) ; } public static async determineProperDirectory ( directorySet ) { for ( const path of directorySet . paths ) { if ( await Files . existsAsync ( path ) ) { return path ; } } return directorySet . preferredPath ; } /**
     * Get all data dirs for a given platform.  There might be multiple
     * locations per platform depending on earlier versions of Polar so
     * we return all possible directories and we can test which ones exist
     * and use the preferred directory if none exist.
     *
     * The preferred data directories are always in Polar/Data.  The reason we
     * always include /Data is that Electron likes to create a directory name
     * for the app and store chrome data in that directory.  This way the
     * Polar data is sandboxed into its own Data directory seperate from the
     * chromium user profile data.
     *
     */ public static getDataDirsForPlatform ( dirRuntime ) { const { userHome , platform } = dirRuntime ; switch ( platform ) { case Platform . WINDOWS : { const preferredPath = FilePaths . join ( userHome , "Polar" ) ; return { paths : [ FilePaths . join ( userHome , ".polar" ) , preferredPath ] , preferredPath } ; } case Platform . LINUX : { const preferredPath = FilePaths . join ( userHome , ".config" , "Polar" ) ; return { paths : [ FilePaths . join ( userHome , ".polar" ) , ] , preferredPath } ; } case Platform . MACOS : { const preferredPath = FilePaths . join ( userHome , "Library" , "Application Support" , "Polar" ) ; return { paths : [ FilePaths . join ( userHome , ".polar" ) , preferredPath , ] , preferredPath } ; } default : throw new Error ( "Platform not supported: " + platform ) ; } } public static getUserHome ( ) { const ENV_NAME = process . platform === 'win32' ? 'USERPROFILE' : 'HOME' ; let result = process . env [ ENV_NAME ] ; if ( ! result ) { result = os . homedir ( ) ; } return result ; } } /**
 * Information about the directories used in the current runtime / OS.
 */ export interface DirRuntime { readonly userHome : string ; readonly platform : Platform ; } /**
 * A set of directories for a given platform.
 */ export interface DirectorySet { /**
     * All paths that might exist.
     */ readonly paths : string [ ] ; /**
     * The preferred path to use is none currently exist.
     */ readonly preferredPath : string ; } export interface DataDir { /**
     * The path to the data dir.
     */ path : string | undefined | null ; /**
     * How the data dir was configured.
     */ strategy : DirStrategy ; } export interface DataDirConfig { path : string ; /**
     * How the data dir was configured.
     */ strategy : DirStrategy ; } export interface DiskDeleteResult extends DeleteResult { docMetaFile : Readonly < FileDeleted > ; } type DirStrategy = 'env' | 'home' | 'manual' ; interface DiskFileReference { dir : string ; path : string ; metaPath : string ; } export interface DiskInitResult extends InitResult { } export interface InitOptions { /**
     * Perform a snapshot on init if
     */ readonly initialSnapshotRequired : boolean ; } export class DiskPersistentPrefsBacking { private prefs : DiskPersistentPrefs ; private readonly directories : Directories ; private readonly path : string ; constructor ( directories ) { this . directories = directories ; this . prefs = new DiskPersistentPrefs ( this ) ; this . path = FilePaths . create ( this . directories . configDir , "prefs.json" ) ; } public async init ( ) { if ( await Files . existsAsync ( this . path ) ) { log . info ( "Loaded prefs from: " + this . path ) ; const data = await Files . readFileAsync ( this . path ) ; const prefs = JSON . parse ( data . toString ( "utf-8" ) ) ; this . prefs = new DiskPersistentPrefs ( this , prefs ) ; } } public async commit ( ) { const data = JSON . stringify ( this . prefs . toPrefDict ( ) , null , "  " ) ; await Files . writeFileAsync ( this . path , data , { atomic : true } ) ; } public get ( ) { return this . prefs ; } } /**
 * Prefs object just backed by a local dictionary.
 */ export class DiskPersistentPrefs extends DictionaryPrefs implements IPersistentPrefs { private readonly diskPrefsStore : DiskPersistentPrefsBacking ; constructor ( diskPrefsStore , delegate = { } ) { super ( delegate ) ; this . diskPrefsStore = diskPrefsStore ; } public async commit ( ) { return await this . diskPrefsStore . commit ( ) ; } } class DiskPersistentPrefsProviderImpl extends AbstractPrefsProvider { constructor ( private readonly backing ) { super ( ) ; } public get ( ) { return this . createInterceptedPersistentPrefs ( this . backing . get ( ) ) ! ; } }