import ArrayBufferSlice from "../../ArrayBufferSlice" ; import { readFileSync , writeFileSync } from "fs" ; import { assert , hexzero , align , readString } from "../../util" ; import * as Pako from 'pako' ; import * as BYML from "../../byml" ; import { TextDecoder } from "util" ; import { Endianness } from "../../endian" ; function fetchDataSync ( path ) { const b = readFileSync ( path ) ; return new ArrayBufferSlice ( b . buffer ) ; } const pathBaseIn = `../../../data/BanjoTooie_Raw` ; const pathBaseOut = `../../../data/BanjoTooie` ; interface FSFile { fileTableOffs : number ; dataOffs : number ; flags : number ; } interface FS { buffer : ArrayBufferSlice ; files : FSFile [ ] ; } function getFileSize ( fs , index ) { return fs . files [ index + 1 ] . dataOffs - fs . files [ index ] . dataOffs ; } function getFileBuffer ( fs , index ) { const fileSize = getFileSize ( fs , index ) ; return fs . buffer . subarray ( fs . files [ index ] . dataOffs , fileSize ) ; } function decompress ( buffer ) { const decompressed = Pako . inflateRaw ( buffer . createTypedArray ( Uint8Array , 2 ) , { raw : true } ) ; return new ArrayBufferSlice ( decompressed . buffer ) ; } interface CRG1File { FileID : number ; Data : ArrayBufferSlice ; } function extractFileAndAppend ( fileTable , fs , fileID ) { const file = fs . files [ fileID ] ; if ( file === undefined ) return - 1 ; const index = fileTable . findIndex ( ( file ) => file . FileID === fileID ) ; if ( index >= 0 ) return index ; const newFile = extractFile ( fs , fileID ) ; if ( newFile === null ) return - 1 ; fileTable . push ( newFile ) ; return fileTable . length - 1 ; } function extractFile ( fs , index ) { const fileBuffer = getFileBuffer ( fs , index ) ; if ( fileBuffer . byteLength === 0 ) return null ; if ( ( fs . files [ index ] . flags & 0xF ) === 0xA ) decryptSetup ( index , fileBuffer ) ; const buffer = ( fs . files [ index ] . flags & 0x10 ) ? decompress ( fileBuffer ) : fileBuffer ; return { FileID : index , Data : buffer } ; } const setupOffset = 0x955 ; const keyBuffer = new Uint8Array ( 14 ) ; function decryptSetup ( index , buffer ) { const source = ( index - setupOffset ) * 0x10001 ; for ( let i = 0 ; i < 14 ; i += 2 ) { keyBuffer [ i ] = ( source >>> i ) & 0xFF ; keyBuffer [ i + 1 ] = 0 ; } cicResponse ( keyBuffer ) ; const view = buffer . createDataView ( ) ; for ( let offs = 0 ; offs < buffer . byteLength ; offs ++ ) { const old = view . getUint8 ( offs ) ; view . setUint8 ( offs , old ^ keyBuffer [ offs % 14 ] ) ; } } function cicResponse ( buffer ) { let acc = 5 ; let carry = true ; for ( let i = 0 ; i < 2 * buffer . length ; i ++ ) { const byte = buffer [ i >>> 1 ] ; let mem = i & 1 ? byte & 0xF : byte >>> 4 ; if ( ! ( acc & 2 ) ) acc += 4 ; acc = ( acc + 9 * mem + 8 ) & 0xF ; mem = acc ; acc = ( 3 * mem + ( carry ? 1 : 7 ) ) & 0xF ; carry = carry ? acc <= mem : acc < mem ; acc = ( ~ acc ) & 0xF ; if ( i & 1 ) buffer [ i >>> 1 ] = ( byte & 0xF0 ) | acc ; else buffer [ i >>> 1 ] = ( acc << 4 ) | ( byte & 0x0F ) ; } } function hasExternalTextures ( file ) { const modelView = file . Data . createDataView ( ) ; const texOffset = modelView . getUint16 ( 0x08 ) ; return modelView . getUint8 ( texOffset + 0x06 ) === 1 ; } const textureFilesStart = 0x1EF6 ; function extractAndAppendTextures ( fileTable , fs , modelIndex ) { const model = fileTable [ modelIndex ] ; const modelView = model . Data . createDataView ( ) ; const texOffset = modelView . getUint16 ( 0x08 ) ; const bufferLength = modelView . getUint32 ( texOffset ) ; const texCount = modelView . getUint16 ( texOffset + 0x04 ) ; const buffer = new ArrayBuffer ( bufferLength ) ; const bytes = new Uint8Array ( buffer ) ; let offs = 0 ; for ( let i = 0 ; i < texCount ; i ++ ) { const index = modelView . getUint32 ( texOffset + 8 * ( i + 1 ) ) ; const tex = extractFile ( fs , index + textureFilesStart ) ! ; const texData = tex . Data . createTypedArray ( Uint8Array ) ; bytes . set ( texData , offs ) ; offs += tex . Data . byteLength ; } fileTable . push ( { FileID : model . FileID | 0x8000 , Data : new ArrayBufferSlice ( buffer ) } ) ; return fileTable . length - 1 ; } function appendModelWithTextures ( fileTable , fs , id ) { const modelIndex = extractFileAndAppend ( fileTable , fs , id ) ; if ( modelIndex >= 0 && fs . files [ id ] . flags === 0x10 && hasExternalTextures ( fileTable [ modelIndex ] ) ) return extractAndAppendTextures ( fileTable , fs , modelIndex ) ; return - 1 ; } interface MapSection { OpaID : number ; XluID : number ; Position : number [ ] ; } function extractMapSections ( fs , fileTable , dllIndex , start ) { const dll = extractDLL ( fs . buffer , dllIndex ) ; const view = dll . data . createDataView ( start ) ; const out = [ ] ; let offs = 0 ; while ( true ) { const Position = [ ] ; Position . push ( view . getFloat32 ( offs + 0x00 ) ) ; Position . push ( view . getFloat32 ( offs + 0x04 ) ) ; Position . push ( view . getFloat32 ( offs + 0x08 ) ) ; if ( Math . abs ( Position [ 0 ] ) < 1 ) break ; const OpaID = view . getUint16 ( offs + 0x26 ) ; const XluID = view . getUint16 ( offs + 0x28 ) ; offs += 0x2C ; appendModelWithTextures ( fileTable , fs , OpaID ) ; appendModelWithTextures ( fileTable , fs , XluID ) ; out . push ( { OpaID , XluID , Position } ) ; } return out ; } function extractMap ( fs , name , sceneID , opaID , xluID = - 1 , opaSky = - 1 , xluSky = - 1 ) { const fileTable = [ ] ; const crg1 = { Name : name , SceneID : sceneID , SetupFileID : sceneID + setupOffset , Files : fileTable , OpaGeoFileID : opaID , OpaGeoTextures : - 1 , XluGeoFileID : xluID , XluGeoTextures : - 1 , OpaSkyboxFileID : opaSky , OpaSkyboxTextures : - 1 , XluSkyboxFileID : xluSky , XluSkyboxTextures : - 1 , } ; extractFileAndAppend ( fileTable , fs , crg1 . SetupFileID ) ; crg1 . OpaGeoTextures = appendModelWithTextures ( fileTable , fs , opaID ) ; crg1 . XluGeoTextures = appendModelWithTextures ( fileTable , fs , xluID ) ; crg1 . OpaSkyboxTextures = appendModelWithTextures ( fileTable , fs , opaSky ) ; crg1 . XluSkyboxTextures = appendModelWithTextures ( fileTable , fs , xluSky ) ; let sections = [ ] ; if ( sceneID === 0x1A7 ) sections = extractMapSections ( fs , fileTable , 0x296 , 0x6D0 ) ; else if ( sceneID === 0x1A8 ) sections = extractMapSections ( fs , fileTable , 0x297 , 0x2A0 ) ; else if ( sceneID === 0x1A9 ) sections = extractMapSections ( fs , fileTable , 0x298 , 0x2A0 ) ; if ( sections . length > 0 ) ( crg1 as any ) . Sections = sections ; const data = BYML . write ( crg1 , BYML . FileType . CRG1 ) ; writeFileSync ( `${ pathBaseOut }/${ hexzero ( sceneID , 2 ) . toUpperCase ( ) }_arc.crg1` , Buffer . from ( data ) ) ; } interface Animation { duration : number ; id : number ; flags : number ; } function extractActor ( fs , id , actorDLL , chosenFunc ) { const view = actorDLL . data . createDataView ( ) ; let actorOffs = actorDLL . offset ; while ( true ) { const instr = view . getUint32 ( actorOffs ) ; if ( instr === 0x3C020000 || instr === 0x03e00008 || ( instr >>> 12 ) === 0x27BDF || ( view . getUint32 ( actorOffs - 4 ) === 0 && actorOffs > actorDLL . offset ) || instr >>> 8 === 0x3C0E80 || instr === 0x000470C0 || instr === 0x3C0142B4 || instr === 0xAFA40000 || instr === 0x908F0064 || instr === 0x3C010000 ) break ; actorOffs += 0x10 ; } const funcOffset = view . getUint32 ( 0x28 + chosenFunc ) ; assert ( view . getUint32 ( actorOffs + funcOffset + 0x04 ) === 0x03E00008 ) ; const defStart = actorOffs + view . getUint16 ( actorOffs + funcOffset + 0x0A ) ; const fileTable = [ ] ; const modelIndex = view . getInt16 ( defStart + 0x04 ) ; if ( modelIndex >= 0 ) appendModelWithTextures ( fileTable , fs , modelIndex ) ; const FirstAnimation = view . getInt16 ( defStart + 0x06 ) ; const animationTable = view . getUint32 ( defStart + 0x08 ) + actorOffs ; const Animations = [ ] ; if ( animationTable > 0 && animationTable < defStart ) { for ( let offs = animationTable ; offs < defStart ; offs += 8 ) { const duration = view . getFloat32 ( offs + 0x00 ) ; if ( duration !== 0 && duration < .01 ) break ; const id = view . getUint16 ( offs + 0x04 ) ; const flags = view . getUint16 ( offs + 0x06 ) ; extractFileAndAppend ( fileTable , fs , id ) ; Animations . push ( { duration , id , flags } ) ; } } const PairedIDs = [ ] ; const actor = { Name : actorDLL . name , Definition : actorDLL . data . subarray ( defStart , 0x44 ) , Files : fileTable , IsFlipbook : modelIndex < 0 || fs . files [ modelIndex ] . flags !== 0x10 , Animations , FirstAnimation , PairedIDs , } ; if ( actorDLL . name === "chnests" ) PairedIDs . push ( 0x85C ) ; else if ( actorDLL . name === "chjinjo" || actorDLL . name === "chbadjinjo" ) { const ps = [ ] ; for ( let p = 0xC31 ; p <= 0xC39 ; p ++ ) { ps . push ( p ) ; extractFileAndAppend ( fileTable , fs , p ) ; } ( actor as any ) . Palettes = ps ; } else if ( id === 0x438 ) { extractFileAndAppend ( fileTable , fs , modelIndex + 1 ) ; ( actor as any ) . Variants = [ modelIndex , modelIndex + 1 ] ; } const data = BYML . write ( actor , BYML . FileType . CRG1 ) ; writeFileSync ( `${ pathBaseOut }/actor/${ hexzero ( id , 3 ) . toUpperCase ( ) }_arc.crg1` , Buffer . from ( data ) ) ; } function dumpNames ( fs , rom ) { const stringTables = new Map < number , ArrayBufferSlice > ( ) ; const names = new Map < number , string > ( ) ; const dec = new TextDecoder ( ) ; const introText = extractDLL ( rom , 0x29E ) ; const view = introText . data . createDataView ( 0x2D0 ) ; let offs = 0 ; for ( let i = 1 ; i < 199 ; i ++ ) { const entry = view . getUint32 ( offs ) ; const level = entry >>> 22 ; const tableIndex = ( entry >> 6 ) & 0xFFFF ; const index = entry & 0x3F ; if ( ! stringTables . has ( tableIndex ) ) stringTables . set ( tableIndex , extractFile ( fs , tableIndex ) ! . Data ) ; const table = stringTables . get ( tableIndex ) ! ; const txtView = table . createDataView ( ) ; const nameCount = txtView . getUint16 ( 0x02 ) ; let txtOffs = 4 ; for ( let j = 0 ; j < nameCount ; j ++ ) { const x = txtView . getUint8 ( txtOffs + 0x00 ) & 0x7F ; const length = txtView . getUint8 ( txtOffs + 0x01 ) ; if ( x == index ) { names . set ( level , readString ( table , txtOffs + 2 , length - 1 ) ) ; break ; } else { txtOffs += length + 2 ; } } offs += 4 ; } return names ; } interface DLLData { name : string ; index : number ; offset : number ; data : ArrayBufferSlice ; } const DLLStart = 0x1e899b0 ; function extractDLL ( rom , index ) { const view = rom . createDataView ( ) ; const start = view . getUint32 ( DLLStart + index * 4 ) ; const end = view . getUint32 ( DLLStart + index * 4 + 4 ) ; const nameLength = view . getUint8 ( DLLStart + start + 0x0E ) ; const raw = rom . subarray ( DLLStart + start + 0x10 , end - start - 0x10 ) ; const data = decompress ( raw ) ; const dllView = data . createDataView ( ) ; let offset = 0x28 ; while ( dllView . getUint8 ( offset ) === 0 ) offset += 4 ; const name = readString ( data , offset , - 1 , true ) ; assert ( name . length === nameLength - 1 , "bad name" ) ; offset = align ( offset + name . length , 0x10 ) ; return { name , index , offset , data , } ; } function main ( ) { const romData = fetchDataSync ( `${ pathBaseIn }/rom.z64` ) ; const view = romData . createDataView ( ) ; const files = [ ] ; for ( let fsTableIdx = 0x5188 ; fsTableIdx < 0x12B24 ; fsTableIdx += 0x04 ) { const ptr = view . getUint32 ( fsTableIdx + 0x00 ) >>> 6 ; const flags = view . getUint8 ( fsTableIdx + 0x03 ) ; const dataOffs = 0x12B24 + ptr ; files . push ( { fileTableOffs : fsTableIdx , dataOffs , flags } ) ; } const fs = { buffer : romData , files } ; const gcmapDLL = extractDLL ( romData , 0x29F ) ; const mapView = gcmapDLL . data . createDataView ( 0x10C0 ) ; const gcskyDLL = extractDLL ( romData , 0x29A ) ; const skyView = gcskyDLL . data . createDataView ( ) ; for ( let offs = 0 ; offs < 0xC78 ; offs += 0xE ) { const id = mapView . getUint16 ( offs + 0x00 ) ; const opa = mapView . getUint16 ( offs + 0x02 ) ; const xlu = mapView . getUint16 ( offs + 0x04 ) ; let opaSky = - 1 ; let xluSky = - 1 ; for ( let sky = 0x570 ; sky < 0xB58 ; sky += 0x24 ) { if ( skyView . getUint16 ( sky + 0x00 ) === id ) { const skyA = skyView . getUint16 ( sky + 0x04 ) ; if ( skyA > 0 ) { assert ( skyView . getFloat32 ( sky + 0x08 ) === 1 ) ; opaSky = skyA ; } const skyB = skyView . getUint16 ( sky + 0x14 ) ; if ( skyB > 0 ) { assert ( skyView . getFloat32 ( sky + 0x18 ) === 1 ) ; xluSky = skyB ; } } } extractMap ( fs , "" , id , opa > 0 ? opa : - 1 , xlu > 0 ? xlu : - 1 , opaSky , xluSky ) ; } const dllBlock = decompress ( romData . subarray ( 0x1E42550 , 0x44726 ) ) ; const dllView = dllBlock . createDataView ( ) ; const gemarkersDLL = extractDLL ( romData , 0x318 ) ; const markerView = gemarkersDLL . data . createDataView ( 0xE0 ) ; let offs = 0 ; for ( let i = 0xb6 ; i < 0x546 ; i ++ ) { const marker = markerView . getUint32 ( offs ) ; offs += 4 ; if ( marker === 0 || marker >= 0x8008A980 ) continue ; const dispatch = dllView . getUint32 ( marker - 0x800815C0 ) ; const actorDLL = extractDLL ( romData , ( dispatch >>> 6 ) - 1 ) ; extractActor ( fs , i , actorDLL , dllView . getUint8 ( marker + 7 - 0x800815C0 ) ) ; } const gsproplookupDLL = extractDLL ( romData , 0x2DB ) ; const fileTable = [ ] ; const modelList = gsproplookupDLL . data . createTypedArray ( Uint16Array , 0x90 , 90 , Endianness . BIG_ENDIAN ) ; const flipbookList = gsproplookupDLL . data . createTypedArray ( Uint16Array , 0x144 , 48 , Endianness . BIG_ENDIAN ) ; for ( let id of modelList ) extractFileAndAppend ( fileTable , fs , id ) ; for ( let id of flipbookList ) extractFileAndAppend ( fileTable , fs , id ) ; const crg = { Models : gsproplookupDLL . data . subarray ( 0x90 , modelList . byteLength ) , Flipbooks : gsproplookupDLL . data . subarray ( 0x144 , flipbookList . byteLength ) , Files : fileTable , } ; const data = BYML . write ( crg , BYML . FileType . CRG1 ) ; writeFileSync ( `${ pathBaseOut }/static_arc.crg1` , Buffer . from ( data ) ) ; for ( let id of [ 0x85C ] ) { const file = extractFile ( fs , id ) ! ; const data = BYML . write ( file , BYML . FileType . CRG1 ) ; writeFileSync ( `${ pathBaseOut }/file/${ hexzero ( id , 3 ) . toUpperCase ( ) }_arc.crg1` , Buffer . from ( data ) ) ; } } main ( ) ;