import * as assert from 'assert' ; import * as iconv from 'iconv-lite' ; import { extname } from 'path' ; import { FieldDescriptor , validateFieldDescriptor } from './field-descriptor' ; import { isValidFileVersion } from './file-version' ; import { CreateOptions , Encoding , normaliseCreateOptions , normaliseOpenOptions , OpenOptions } from './options' ; import { close , open , read , stat , write } from './utils' ; import { createDate , format8CharDate , formatVfpDateTime , parseVfpDateTime , parse8CharDate } from './utils' ; /** Represents a DBF file. */ export class DBFFile { /** Opens an existing DBF file. */ static async open ( path , options ? ) { return openDBF ( path , options ) ; } /** Creates a new DBF file with no records. */ static async create ( path , fields , options ? ) { return createDBF ( path , fields , options ) ; } /** Full path to the DBF file. */ path = '' ; /** Total number of records in the DBF file. (NB: includes deleted records). */ recordCount = 0 ; /** Date of last update as recorded in the DBF file header. */ dateOfLastUpdate ! : Date ; /** Metadata for all fields defined in the DBF file. */ fields = [ ] as FieldDescriptor [ ] ; /**
     * Reads a subset of records from this DBF file. If the `includeDeletedRecords` option is set, then deleted records
     * are included in the results, otherwise they are skipped. Deleted records have the property `[DELETED]: true`,
     * using the `DELETED` symbol exported from this library.
     */ readRecords ( maxCount = 10000000 ) { return readRecordsFromDBF ( this , maxCount ) ; } /** Appends the specified records to this DBF file. */ appendRecords ( records ) { return appendRecordsToDBF ( this , records ) ; } _readMode = 'strict' as 'strict' | 'loose' ; _encoding = '' as Encoding ; _includeDeletedRecords = false ; _recordsRead = 0 ; _headerLength = 0 ; _recordLength = 0 ; _memoPath ? = '' ; _version ? = 0 ; } /** Symbol used for detecting deleted records when the `includeDeletedRecords` option is used. */ export const DELETED = Symbol ( ) ; async function openDBF ( path , opts ? ) { let options = normaliseOpenOptions ( opts ) ; let fd = 0 ; try { fd = await open ( path , 'r' ) ; let buffer = Buffer . alloc ( 32 ) ; await read ( fd , buffer , 0 , 32 , 0 ) ; let fileVersion = buffer . readUInt8 ( 0 ) ; let lastUpdateY = buffer . readUInt8 ( 1 ) ; let lastUpdateM = buffer . readUInt8 ( 2 ) ; let lastUpdateD = buffer . readUInt8 ( 3 ) ; const dateOfLastUpdate = createDate ( lastUpdateY + 1900 , lastUpdateM , lastUpdateD ) ; let recordCount = buffer . readInt32LE ( 4 ) ; let headerLength = buffer . readInt16LE ( 8 ) ; let recordLength = buffer . readInt16LE ( 10 ) ; let memoPath ; if ( options . readMode !== 'loose' && ! isValidFileVersion ( fileVersion ) ) { throw new Error ( `File '${ path }' has unknown/unsupported dBase version: ${ fileVersion }.` ) ; } if ( fileVersion === 0x83 || fileVersion === 0x8b ) { for ( const ext of [ '.dbt' , '.DBT' ] ) { memoPath = path . slice ( 0 , - extname ( path ) . length ) + ext ; let foundMemoFile = await stat ( memoPath ) . catch ( ( ) => 'missing' ) !== 'missing' ; if ( foundMemoFile ) break ; memoPath = undefined ; } if ( options . readMode !== 'loose' && ! memoPath ) { throw new Error ( `Memo file not found for file '${ path }'.` ) ; } } if ( fileVersion === 0x30 ) { const dbExt = extname ( path ) . toLowerCase ( ) ; const memoExt = dbExt == '.dbf' ? '.fpt' : `.${ dbExt . substr ( 1 , 2 ) }t` ; for ( const ext of [ memoExt , memoExt . toUpperCase ( ) ] ) { memoPath = path . slice ( 0 , - extname ( path ) . length ) + ext ; let foundMemoFile = await stat ( memoPath ) . catch ( ( ) => 'missing' ) !== 'missing' ; if ( foundMemoFile ) break ; memoPath = undefined ; } } let fields = [ ] ; while ( headerLength > 32 + fields . length * 32 ) { await read ( fd , buffer , 0 , 32 , 32 + fields . length * 32 ) ; if ( buffer . readUInt8 ( 0 ) === 0x0D ) break ; let field = { name : iconv . decode ( buffer . slice ( 0 , 10 ) , 'ISO-8859-1' ) . split ( '\0' ) [ 0 ] , type : String . fromCharCode ( buffer [ 0x0B ] ) as FieldDescriptor [ 'type' ] , size : buffer . readUInt8 ( 0x10 ) , decimalPlaces : buffer . readUInt8 ( 0x11 ) } ; if ( options . readMode !== 'loose' ) { validateFieldDescriptor ( field , fileVersion ) ; assert ( fields . every ( f => f . name !== field . name ) , `Duplicate field name: '${ field . name }'` ) ; } fields . push ( field ) ; } await read ( fd , buffer , 0 , 1 , 32 + fields . length * 32 ) ; assert ( buffer [ 0 ] === 0x0d , 'Invalid DBF: Expected header terminator' ) ; const computedRecordLength = calculateRecordLengthInBytes ( fields ) ; if ( options . readMode === 'loose' ) recordLength = computedRecordLength ; assert ( recordLength === computedRecordLength , 'Invalid DBF: Incorrect record length' ) ; let result = new DBFFile ( ) ; result . path = path ; result . recordCount = recordCount ; result . dateOfLastUpdate = dateOfLastUpdate ; result . fields = fields ; result . _readMode = options . readMode ; result . _encoding = options . encoding ; result . _includeDeletedRecords = options . includeDeletedRecords ; result . _recordsRead = 0 ; result . _headerLength = headerLength ; result . _recordLength = recordLength ; result . _memoPath = memoPath ; result . _version = fileVersion ; return result ; } finally { if ( fd ) await close ( fd ) ; } } ; async function createDBF ( path , fields , opts ? ) { let options = normaliseCreateOptions ( opts ) ; let fd = 0 ; try { let fileVersion = options . fileVersion ; validateFieldDescriptors ( fields , fileVersion ) ; if ( fields . some ( f => f . type === 'M' ) ) throw new Error ( `Writing to files with memo fields is not supported.` ) ; fd = await open ( path , 'wx' ) ; let buffer = Buffer . alloc ( 32 ) ; buffer . writeUInt8 ( fileVersion , 0x00 ) ; let now = new Date ( ) ; buffer . writeUInt8 ( now . getFullYear ( ) - 1900 , 0x01 ) ; buffer . writeUInt8 ( now . getMonth ( ) + 1 , 0x02 ) ; buffer . writeUInt8 ( now . getDate ( ) , 0x03 ) ; buffer . writeInt32LE ( 0 , 0x04 ) ; let headerLength = 34 + ( fields . length * 32 ) ; buffer . writeUInt16LE ( headerLength , 0x08 ) ; let recordLength = calculateRecordLengthInBytes ( fields ) ; buffer . writeUInt16LE ( recordLength , 0x0A ) ; buffer . writeUInt32LE ( 0 , 0x0C ) ; buffer . writeUInt32LE ( 0 , 0x10 ) ; buffer . writeUInt32LE ( 0 , 0x14 ) ; buffer . writeUInt32LE ( 0 , 0x18 ) ; buffer . writeUInt32LE ( 0 , 0x1C ) ; await write ( fd , buffer , 0 , 32 , 0 ) ; for ( let i = 0 ; i < fields . length ; ++ i ) { let { name , type , size , decimalPlaces } = fields [ i ] ; iconv . encode ( name , 'ISO-8859-1' ) . copy ( buffer , 0 ) ; for ( let j = name . length ; j < 11 ; ++ j ) { buffer . writeUInt8 ( 0 , j ) ; } buffer . writeUInt8 ( type . charCodeAt ( 0 ) , 0x0B ) ; buffer . writeUInt32LE ( 0 , 0x0C ) ; buffer . writeUInt8 ( size , 0x10 ) ; buffer . writeUInt8 ( decimalPlaces || 0 , 0x11 ) ; buffer . writeUInt16LE ( 0 , 0x12 ) ; buffer . writeUInt8 ( 0x01 , 0x14 ) ; buffer . writeUInt16LE ( 0 , 0x15 ) ; buffer . writeUInt8 ( 0 , 0x17 ) ; buffer . writeUInt32LE ( 0 , 0x18 ) ; buffer . writeUInt32LE ( 0 , 0x1C ) ; buffer . writeUInt8 ( 0 , 0x1F ) ; await write ( fd , buffer , 0 , 32 , 32 + i * 32 ) ; } buffer . writeUInt8 ( 0x0D , 0 ) ; buffer . writeUInt8 ( 0x00 , 1 ) ; buffer . writeUInt8 ( 0x1A , 2 ) ; await write ( fd , buffer , 0 , 3 , 32 + fields . length * 32 ) ; let result = new DBFFile ( ) ; result . path = path ; result . recordCount = 0 ; result . dateOfLastUpdate = createDate ( now . getFullYear ( ) , now . getMonth ( ) + 1 , now . getDate ( ) ) ; result . fields = fields . map ( field => ( { ... field } ) ) ; result . _readMode = 'strict' ; result . _encoding = options . encoding ; result . _recordsRead = 0 ; result . _headerLength = headerLength ; result . _recordLength = recordLength ; return result ; } finally { if ( fd ) await close ( fd ) ; } } ; async function readRecordsFromDBF ( dbf , maxCount ) { let fd = 0 ; let memoFd = 0 ; try { fd = await open ( dbf . path , 'r' ) ; let recordCountPerBuffer = 1000 ; let recordLength = dbf . _recordLength ; let buffer = Buffer . alloc ( recordLength * recordCountPerBuffer ) ; let memoBlockSize = 0 ; let memoFileSize = 0 ; let memoBuf ; if ( dbf . _memoPath ) { memoFd = await open ( dbf . _memoPath , 'r' ) ; if ( dbf . _version === 0x30 ) { await read ( memoFd , buffer , 0 , 2 , 6 ) ; memoBlockSize = buffer . readUInt16BE ( 0 ) || 512 ; } else { await read ( memoFd , buffer , 0 , 4 , 4 ) ; memoBlockSize = ( dbf . _version === 0x8b ? buffer . readInt32LE ( 0 ) : 0 ) || 512 ; } memoBuf = Buffer . alloc ( memoBlockSize ) ; memoFileSize = ( await stat ( dbf . _memoPath ) ) . size ; } let currentPosition = dbf . _headerLength + recordLength * dbf . _recordsRead ; let substrAt = ( start , len , enc ) => iconv . decode ( buffer . slice ( start , start + len ) , enc ) ; let int32At = ( start , len ) => buffer . slice ( start , start + len ) . readInt32LE ( 0 ) ; let records = [ ] ; while ( true ) { let maxRecords1 = dbf . recordCount - dbf . _recordsRead ; let maxRecords2 = maxCount - records . length ; let recordCountToRead = maxRecords1 < maxRecords2 ? maxRecords1 : maxRecords2 ; if ( recordCountToRead > recordCountPerBuffer ) recordCountToRead = recordCountPerBuffer ; if ( recordCountToRead === 0 ) break ; await read ( fd , buffer , 0 , recordLength * recordCountToRead , currentPosition ) ; dbf . _recordsRead += recordCountToRead ; currentPosition += recordLength * recordCountToRead ; for ( let i = 0 , offset = 0 ; i < recordCountToRead ; ++ i ) { let record = { } ; let isDeleted = ( buffer [ offset ++ ] === 0x2a ) ; if ( isDeleted && ! dbf . _includeDeletedRecords ) { offset += recordLength - 1 ; continue ; } for ( let j = 0 ; j < dbf . fields . length ; ++ j ) { let field = dbf . fields [ j ] ; let len = field . size ; let value = null ; let encoding = getEncodingForField ( field , dbf . _encoding ) ; switch ( field . type ) { case 'C' : while ( len > 0 && buffer [ offset + len - 1 ] === 0x20 ) -- len ; value = substrAt ( offset , len , encoding ) ; offset += field . size ; break ; case 'N' : case 'F' : while ( len > 0 && buffer [ offset ] === 0x20 ) ++ offset , -- len ; value = len > 0 ? parseFloat ( substrAt ( offset , len , encoding ) ) : null ; offset += len ; break ; case 'L' : let c = String . fromCharCode ( buffer [ offset ++ ] ) ; value = 'TtYy' . indexOf ( c ) >= 0 ? true : ( 'FfNn' . indexOf ( c ) >= 0 ? false : null ) ; break ; case 'T' : if ( buffer [ offset ] === 0x20 ) { value = null ; } else { const julianDay = buffer . readInt32LE ( offset ) ; const msSinceMidnight = buffer . readInt32LE ( offset + 4 ) + 1 ; value = parseVfpDateTime ( { julianDay , msSinceMidnight } ) ; } offset += 8 ; break ; case 'D' : value = buffer [ offset ] === 0x20 ? null : parse8CharDate ( substrAt ( offset , 8 , encoding ) ) ; offset += 8 ; break ; case 'B' : value = buffer . readDoubleLE ( offset ) ; offset += field . size ; break ; case 'I' : value = buffer . readInt32LE ( offset ) ; offset += field . size ; break ; case 'M' : while ( len > 0 && buffer [ offset ] === 0x20 ) ++ offset , -- len ; if ( len === 0 ) { value = null ; break ; } let blockIndex = dbf . _version === 0x30 ? int32At ( offset , len ) : parseInt ( substrAt ( offset , len , encoding ) ) ; offset += len ; if ( ! memoBuf ) continue ; value = '' ; while ( true ) { await read ( memoFd , memoBuf , 0 , memoBlockSize , blockIndex * memoBlockSize ) ; if ( dbf . _version === 0x83 ) { let eos = memoBuf . indexOf ( '\x1A' ) ; value += iconv . decode ( memoBuf . slice ( 0 , eos === - 1 ? memoBlockSize : eos ) , encoding ) ; if ( eos !== - 1 ) break ; } else if ( dbf . _version === 0x8b ) { let isFirstBlockOfMemo = memoBuf . readInt32LE ( 0 ) === 0x0008FFFF ; if ( isFirstBlockOfMemo ) len = memoBuf . readUInt32LE ( 4 ) - 8 ; let skip = isFirstBlockOfMemo ? 8 : 0 ; let take = Math . min ( len , memoBlockSize - skip ) ; value += iconv . decode ( memoBuf . slice ( skip , skip + take ) , encoding ) ; len -= take ; if ( len === 0 ) break ; } else if ( dbf . _version === 0x30 ) { let skip = 0 ; if ( value === '' ) { const memoType = memoBuf . readInt32BE ( 0 ) ; if ( memoType != 1 ) break ; len = memoBuf . readInt32BE ( 4 ) ; skip = 8 ; } let take = Math . min ( len , memoBlockSize - skip ) ; value += iconv . decode ( memoBuf . slice ( skip , skip + take ) , encoding ) ; len -= take ; if ( len === 0 ) break ; } else { throw new Error ( `Reading version ${ dbf . _version } memo fields is not supported.` ) ; } ++ blockIndex ; if ( blockIndex * memoBlockSize > memoFileSize ) { throw new Error ( `Error reading memo file (read past end).` ) ; } } break ; default : if ( dbf . _readMode === 'strict' ) throw new Error ( `Type '${ field . type }' is not supported` ) ; if ( dbf . _readMode === 'loose' ) { offset += field . size ; continue ; } } record [ field . name ] = value ; } if ( isDeleted ) record [ DELETED ] = true ; records . push ( record ) ; } } return records ; } finally { if ( fd ) await close ( fd ) ; if ( memoFd ) await close ( memoFd ) ; } } ; async function appendRecordsToDBF ( dbf , records ) { let fd = 0 ; try { fd = await open ( dbf . path , 'r+' ) ; let recordLength = calculateRecordLengthInBytes ( dbf . fields ) ; let buffer = Buffer . alloc ( recordLength + 4 ) ; let currentPosition = dbf . _headerLength + dbf . recordCount * recordLength ; for ( let i = 0 ; i < records . length ; ++ i ) { let record = records [ i ] ; validateRecord ( dbf . fields , record ) ; let offset = 0 ; buffer . writeUInt8 ( 0x20 , offset ++ ) ; for ( let j = 0 ; j < dbf . fields . length ; ++ j ) { let field = dbf . fields [ j ] ; let value = record [ field . name ] ; if ( value === null || typeof value === 'undefined' ) value = '' ; let encoding = getEncodingForField ( field , dbf . _encoding ) ; switch ( field . type ) { case 'C' : let b = iconv . encode ( value , encoding ) ; for ( let k = 0 ; k < field . size ; ++ k ) { let byte = k < b . length ? b [ k ] : 0x20 ; buffer . writeUInt8 ( byte , offset ++ ) ; } break ; case 'N' : case 'F' : value = value . toString ( ) ; value = value . slice ( 0 , field . size ) ; while ( value . length < field . size ) value = ' ' + value ; iconv . encode ( value , encoding ) . copy ( buffer , offset , 0 , field . size ) ; offset += field . size ; break ; case 'L' : buffer . writeUInt8 ( value === '' ? 0x20 : value ? 0x54 : 0x46 , offset ++ ) ; break ; case 'T' : if ( ! value ) { iconv . encode ( '        ' , encoding ) . copy ( buffer , offset , 0 , 8 ) ; } else { const { julianDay , msSinceMidnight } = formatVfpDateTime ( value ) ; buffer . writeInt32LE ( julianDay , offset ) ; buffer . writeInt32LE ( msSinceMidnight , offset + 4 ) ; } offset += 8 ; break ; case 'D' : value = value ? format8CharDate ( value ) : '        ' ; iconv . encode ( value , encoding ) . copy ( buffer , offset , 0 , 8 ) ; offset += 8 ; break ; case 'B' : buffer . writeDoubleLE ( value , offset ) ; offset += field . size ; break ; case 'I' : buffer . writeInt32LE ( value , offset ) ; offset += field . size ; break ; case 'M' : throw new Error ( `Writing to files with memo fields is not supported.` ) ; default : throw new Error ( `Type '${ field . type }' is not supported` ) ; } } await write ( fd , buffer , 0 , recordLength , currentPosition ) ; currentPosition += recordLength ; } buffer . writeUInt8 ( 0x1A , 0 ) ; await write ( fd , buffer , 0 , 1 , currentPosition ) ; dbf . recordCount += records . length ; buffer . writeInt32LE ( dbf . recordCount , 0 ) ; await write ( fd , buffer , 0 , 4 , 0x04 ) ; return dbf ; } finally { if ( fd ) await close ( fd ) ; } } ; function validateFieldDescriptors ( fields , fileVersion ) { if ( fields . length > 2046 ) throw new Error ( 'Too many fields (maximum is 2046)' ) ; for ( let field of fields ) validateFieldDescriptor ( field , fileVersion ) ; } function validateRecord ( fields , record ) { for ( let i = 0 ; i < fields . length ; ++ i ) { let name = fields [ i ] . name , type = fields [ i ] . type ; let value = record [ name ] ; if ( value === null || typeof value === 'undefined' ) continue ; if ( type === 'C' ) { if ( typeof value !== 'string' ) throw new Error ( `${ name }: expected a string` ) ; if ( value . length > 255 ) throw new Error ( `${ name }: text is too long (maximum length is 255 chars)` ) ; } else if ( type === 'N' || type === 'F' || type === 'I' ) { if ( typeof value !== 'number' ) throw new Error ( `${ name }: expected a number` ) ; } else if ( type === 'D' ) { if ( ! ( value instanceof Date ) ) throw new Error ( `${ name }: expected a date` ) ; } else if ( type === 'L' ) { if ( typeof value !== 'boolean' ) throw new Error ( `${ name }: expected a boolean` ) ; } } } function calculateRecordLengthInBytes ( fields ) { let len = 1 ; for ( let i = 0 ; i < fields . length ; ++ i ) len += fields [ i ] . size ; return len ; } function getEncodingForField ( field , encoding ) { if ( typeof encoding === 'string' ) return encoding ; return encoding [ field . name ] || encoding . default ; }