feat(compartment-mapper): Consistent hashing
MODIFY
packages/compartment-mapper/NEWS.md
packages/compartment-mapper/NEWS.md
User-visible changes to the compartment mapper:
-# Next releaase
+# Next release
- The calling convention between SES and StaticModuleRecords has changed and
this impacts the code generated by the bundler.
+- Adds a support for consistent hashing (SHA-512) of applications:
+ - `nodeReadPowers(fs, crypto)` produces the necessary capabilities for
+ hashing, when passed the Node.js `crypto` module.
+ - `writeArchive` and `makeArchive` accept a `computeSha512` capability and
+ use it to populate the `compartment-map.json` included within the archive
+ with the SHA-512 of every module in the archive. This ensures that the
+ hash of `compartment-map.json` in a pair of archives is consistent only if
+ every file is consistent.
+ - `importArchive`, `loadArchive`, and `parseArchive` all optionally accept a
+ `computeSha512` capability, use it to verify the integrity of the archive
+ and verify the `expectedSha512` of the contained `compartment-map.json`
+ `importArchive` and `loadArchive` receive the hash function as a read
+ power. `parseArchive` receives the hash function as an option since it
+ doesn't receive read powers.
+ - `hashLocation` produces the hash of an application off the filesystem,
+ suitable for validating that an archive with the same hash was generated
+ from identical files.
+- Also adds `mapLocation`, which produces the compartment map that _would_ be
+ in the corresponding archive for a package.
+- Ensures that IO errors on Node.js include a meaningful stack trace.
# 0.4.1 (2021-16-19)

MODIFY
packages/compartment-mapper/archive.js
packages/compartment-mapper/archive.js
-export { makeArchive, writeArchive } from './src/archive.js';
+export {
+ makeArchive,
+ writeArchive,
+ mapLocation,
+ hashLocation,
+} from './src/archive.js';

MODIFY
packages/compartment-mapper/index.js
packages/compartment-mapper/index.js
// @ts-check
export { loadLocation, importLocation } from './src/import.js';
-export { makeArchive, writeArchive } from './src/archive.js';
+export {
+ makeArchive,
+ writeArchive,
+ mapLocation,
+ hashLocation,
+} from './src/archive.js';
export {
parseArchive,
loadArchive,

MODIFY
packages/compartment-mapper/src/archive.js
packages/compartment-mapper/src/archive.js
/** @typedef {import('./types.js').ParseFn} ParseFn */
/** @typedef {import('./types.js').ReadFn} ReadFn */
/** @typedef {import('./types.js').ReadPowers} ReadPowers */
+/** @typedef {import('./types.js').HashPowers} HashPowers */
/** @typedef {import('./types.js').Sources} Sources */
/** @typedef {import('./types.js').WriteFn} WriteFn */
@@ -90,11 +91,12 @@ const translateCompartmentMap = (compartments, sources, renames) => {
if (compartmentSources) {
for (const name of keys(compartmentSources).sort()) {
const source = compartmentSources[name];
- const { location, parser, exit } = source;
+ const { location, parser, exit, sha512 } = source;
modules[name] = {
location,
parser,
exit,
+ sha512,
};
}
}
@@ -151,12 +153,12 @@ const addSourcesToArchive = async (archive, sources) => {
* @param {ReadFn | ReadPowers} powers
* @param {string} moduleLocation
* @param {ArchiveOptions} [options]
- * @returns {Promise<Uint8Array>}
+ * @returns {Promise<{archiveSources: Sources, archiveCompartmentMapBytes: Uint8Array}>}
*/
-export const makeArchive = async (powers, moduleLocation, options) => {
+const digestLocation = async (powers, moduleLocation, options) => {
const { moduleTransforms, modules: exitModules = {}, dev = false } =
options || {};
- const { read } = unpackReadPowers(powers);
+ const { read, computeSha512 } = unpackReadPowers(powers);
const {
packageLocation,
packageDescriptorText,
@@ -184,6 +186,7 @@ export const makeArchive = async (powers, moduleLocation, options) => {
compartments,
entry: { compartment: entryCompartmentName, module: entryModuleSpecifier },
} = compartmentMap;
+
/** @type {Sources} */
const sources = {};
@@ -193,6 +196,7 @@ export const makeArchive = async (powers, moduleLocation, options) => {
sources,
compartments,
exitModules,
+ computeSha512,
);
// Induce importHook to record all the necessary modules to import the given module specifier.
@@ -231,6 +235,25 @@ export const makeArchive = async (powers, moduleLocation, options) => {
archiveCompartmentMapText,
);
+ return {
+ archiveCompartmentMapBytes,
+ archiveSources,
+ };
+};
+
+/**
+ * @param {ReadFn | ReadPowers} powers
+ * @param {string} moduleLocation
+ * @param {ArchiveOptions} [options]
+ * @returns {Promise<Uint8Array>}
+ */
+export const makeArchive = async (powers, moduleLocation, options) => {
+ const { archiveCompartmentMapBytes, archiveSources } = await digestLocation(
+ powers,
+ moduleLocation,
+ options,
+ );
+
const archive = writeZip();
await archive.write('compartment-map.json', archiveCompartmentMapBytes);
await addSourcesToArchive(archive, archiveSources);
@@ -238,6 +261,37 @@ export const makeArchive = async (powers, moduleLocation, options) => {
return archive.snapshot();
};
+/**
+ * @param {ReadFn | ReadPowers} powers
+ * @param {string} moduleLocation
+ * @param {ArchiveOptions} [options]
+ * @returns {Promise<Uint8Array>}
+ */
+export const mapLocation = async (powers, moduleLocation, options) => {
+ const { archiveCompartmentMapBytes } = await digestLocation(
+ powers,
+ moduleLocation,
+ options,
+ );
+ return archiveCompartmentMapBytes;
+};
+
+/**
+ * @param {HashPowers} powers
+ * @param {string} moduleLocation
+ * @param {ArchiveOptions} [options]
+ * @returns {Promise<string>}
+ */
+export const hashLocation = async (powers, moduleLocation, options) => {
+ const { archiveCompartmentMapBytes } = await digestLocation(
+ powers,
+ moduleLocation,
+ options,
+ );
+ const { computeSha512 } = powers;
+ return computeSha512(archiveCompartmentMapBytes);
+};
+
/**
* @param {WriteFn} write
* @param {ReadFn | ReadPowers} readPowers

MODIFY
packages/compartment-mapper/src/import-archive.js
packages/compartment-mapper/src/import-archive.js
/** @typedef {import('./types.js').CompartmentMapDescriptor} CompartmentMapDescriptor */
/** @typedef {import('./types.js').ExecuteFn} ExecuteFn */
/** @typedef {import('./types.js').ReadFn} ReadFn */
+/** @typedef {import('./types.js').ReadPowers} ReadPowers */
+/** @typedef {import('./types.js').HashFn} HashFn */
+/** @typedef {import('./types.js').LoadArchiveOptions} LoadArchiveOptions */
/** @typedef {import('./types.js').ExecuteOptions} ExecuteOptions */
import { readZip } from '@endo/zip';
@@ -17,6 +20,7 @@ import { parsePreCjs } from './parse-pre-cjs.js';
import { parseJson } from './parse-json.js';
import { parsePreMjs } from './parse-pre-mjs.js';
import { parseLocatedJson } from './json.js';
+import { unpackReadPowers } from './powers.js';
// q as in quote for strings in error messages.
const q = JSON.stringify;
@@ -40,9 +44,15 @@ const parserForLanguage = {
* @param {ArchiveReader} archive
* @param {Record<string, CompartmentDescriptor>} compartments
* @param {string} archiveLocation
+ * @param {HashFn} [computeSha512]
* @returns {ArchiveImportHookMaker}
*/
-const makeArchiveImportHookMaker = (archive, compartments, archiveLocation) => {
+const makeArchiveImportHookMaker = (
+ archive,
+ compartments,
+ archiveLocation,
+ computeSha512,
+) => {
// per-assembly:
/** @type {ArchiveImportHookMaker} */
const makeImportHook = packageLocation => {
@@ -69,6 +79,20 @@ const makeArchiveImportHookMaker = (archive, compartments, archiveLocation) => {
}
const moduleLocation = `${packageLocation}/${module.location}`;
const moduleBytes = await archive.read(moduleLocation);
+
+ if (computeSha512 !== undefined && module.sha512 !== undefined) {
+ const sha512 = computeSha512(moduleBytes);
+ if (sha512 !== module.sha512) {
+ throw new Error(
+ `Module ${q(module.location)} of package ${q(
+ packageLocation,
+ )} in archive ${q(
+ archiveLocation,
+ )} failed a SHA-512 integrity check`,
+ );
+ }
+ }
+
// eslint-disable-next-line no-await-in-loop
const { record } = await parse(
moduleBytes,
@@ -85,19 +109,41 @@ const makeArchiveImportHookMaker = (archive, compartments, archiveLocation) => {
/**
* @param {Uint8Array} archiveBytes
- * @param {string} archiveLocation
+ * @param {string} [archiveLocation]
+ * @param {Object} [options]
+ * @param {string} [options.expectedSha512]
+ * @param {HashFn} [options.computeSha512]
* @returns {Promise<Application>}
*/
-export const parseArchive = async (archiveBytes, archiveLocation) => {
- const archive = await readZip(archiveBytes, archiveLocation);
+export const parseArchive = async (
+ archiveBytes,
+ archiveLocation = '<unknown>',
+ options = {},
+) => {
+ const { computeSha512 = undefined, expectedSha512 = undefined } = options;
+ const archive = await readZip(archiveBytes, archiveLocation);
const compartmentMapBytes = await archive.read('compartment-map.json');
+
+ if (expectedSha512 !== undefined) {
+ if (computeSha512 === undefined) {
+ throw new Error(
+ `Cannot verify expectedSha512 without also providing computeSha512, for archive ${archiveLocation}`,
+ );
+ }
+ const sha512 = computeSha512(compartmentMapBytes);
+ if (sha512 !== expectedSha512) {
+ throw new Error(
+ `Archive compartment map failed a SHA-512 integrity check, expected ${expectedSha512}, got ${sha512}, for archive ${archiveLocation}`,
+ );
+ }
+ }
+
const compartmentMapText = textDecoder.decode(compartmentMapBytes);
const compartmentMap = /** @type {CompartmentMapDescriptor} */ (parseLocatedJson(
compartmentMapText,
'compartment-map.json',
));
-
// TODO validate compartmentMap instead of leaning hard on the above type
// assertion.
@@ -119,6 +165,7 @@ export const parseArchive = async (archiveBytes, archiveLocation) => {
archive,
compartments,
archiveLocation,
+ computeSha512,
);
const compartment = assemble(compartmentMap, {
makeImportHook,
@@ -137,22 +184,32 @@ export const parseArchive = async (archiveBytes, archiveLocation) => {
};
/**
- * @param {ReadFn} read
+ * @param {ReadFn | ReadPowers} readPowers
* @param {string} archiveLocation
+ * @param {LoadArchiveOptions} [options]
* @returns {Promise<Application>}
*/
-export const loadArchive = async (read, archiveLocation) => {
+export const loadArchive = async (
+ readPowers,
+ archiveLocation,
+ options = {},
+) => {
+ const { read, computeSha512 } = unpackReadPowers(readPowers);
+ const { expectedSha512 } = options;
const archiveBytes = await read(archiveLocation);
- return parseArchive(archiveBytes, archiveLocation);
+ return parseArchive(archiveBytes, archiveLocation, {
+ computeSha512,
+ expectedSha512,
+ });
};
/**
- * @param {ReadFn} read
+ * @param {ReadFn | ReadPowers} readPowers
* @param {string} archiveLocation
- * @param {ExecuteOptions} options
+ * @param {ExecuteOptions & LoadArchiveOptions} options
* @returns {Promise<Object>}
*/
-export const importArchive = async (read, archiveLocation, options) => {
- const archive = await loadArchive(read, archiveLocation);
+export const importArchive = async (readPowers, archiveLocation, options) => {
+ const archive = await loadArchive(readPowers, archiveLocation, options);
return archive.import(options);
};

MODIFY
packages/compartment-mapper/src/import-hook.js
packages/compartment-mapper/src/import-hook.js
/** @typedef {import('ses').ImportHook} ImportHook */
/** @typedef {import('ses').StaticModuleType} StaticModuleType */
/** @typedef {import('./types.js').ReadFn} ReadFn */
+/** @typedef {import('./types.js').HashFn} HashFn */
/** @typedef {import('./types.js').Sources} Sources */
/** @typedef {import('./types.js').CompartmentDescriptor} CompartmentDescriptor */
/** @typedef {import('./types.js').ImportHookMaker} ImportHookMaker */
@@ -42,6 +43,7 @@ const resolveLocation = (rel, abs) => new URL(rel, abs).toString();
* @param {Sources} sources
* @param {Record<string, CompartmentDescriptor>} compartments
* @param {Record<string, never>} exitModules
+ * @param {HashFn=} computeSha512
* @returns {ImportHookMaker}
*/
export const makeImportHookMaker = (
@@ -50,6 +52,7 @@ export const makeImportHookMaker = (
sources = {},
compartments = {},
exitModules = {},
+ computeSha512 = undefined,
) => {
// per-assembly:
/** @type {ImportHookMaker} */
@@ -142,6 +145,11 @@ export const makeImportHookMaker = (
record = concreteRecord;
}
+ let sha512;
+ if (computeSha512 !== undefined) {
+ sha512 = computeSha512(transformedBytes);
+ }
+
const packageRelativeLocation = moduleLocation.slice(
packageLocation.length,
);
@@ -150,6 +158,7 @@ export const makeImportHookMaker = (
parser,
bytes: transformedBytes,
record,
+ sha512,
};
return record;
}

MODIFY
packages/compartment-mapper/src/node-powers.js
packages/compartment-mapper/src/node-powers.js
// @ts-check
/** @typedef {import('./types.js').ReadPowers} ReadPowers */
+/** @typedef {import('./types.js').HashFn} HashFn */
/** @typedef {import('./types.js').WritePowers} WritePowers */
/**
* @param {typeof import('fs')} fs
+ * @param {typeof import('crypto')} [crypto]
* @returns {ReadPowers}
*/
-export const makeNodeReadPowers = fs => {
+export const makeNodeReadPowers = (fs, crypto = undefined) => {
/**
* @param {string} location
*/
- const read = async location =>
- fs.promises.readFile(new URL(location).pathname);
+ const read = async location => {
+ try {
+ return await fs.promises.readFile(new URL(location).pathname);
+ } catch (error) {
+ throw new Error(error.message);
+ }
+ };
/**
* There are two special things about the canonical function the compartment
@@ -45,7 +52,16 @@ export const makeNodeReadPowers = fs => {
}
};
- return { read, canonical };
+ /** @type {HashFn=} */
+ const computeSha512 = crypto
+ ? bytes => {
+ const hash = crypto.createHash('sha512');
+ hash.update(bytes);
+ return hash.digest().toString('hex');
+ }
+ : undefined;
+
+ return { read, canonical, computeSha512 };
};
/**
@@ -57,8 +73,13 @@ export const makeNodeWritePowers = fs => {
* @param {string} location
* @param {Uint8Array} data
*/
- const write = async (location, data) =>
- fs.promises.writeFile(new URL(location).pathname, data);
+ const write = async (location, data) => {
+ try {
+ return await fs.promises.writeFile(new URL(location).pathname, data);
+ } catch (error) {
+ throw new Error(error.message);
+ }
+ };
return { write };
};

MODIFY
packages/compartment-mapper/src/types.js
packages/compartment-mapper/src/types.js
@@ -54,6 +54,7 @@ export const moduleJSDocTypes = true;
* @property {string} [module]
* @property {string} [location]
* @property {Language} [parser]
+ * @property {string} [sha512] in base 16, hex
* @property {string} [exit]
*/
@@ -111,9 +112,16 @@ export const moduleJSDocTypes = true;
* @returns {Promise<string>} canonical location
*/
+/**
+ * @callback HashFn
+ * @param {Uint8Array} bytes
+ * @returns {string} hash
+ */
+
/**
* @typedef {Object} Application
* @property {ExecuteFn} import
+ * @property {string} [sha512]
*/
/**
@@ -131,6 +139,14 @@ export const moduleJSDocTypes = true;
* @typedef {Object} ReadPowers
* @property {ReadFn} read
* @property {CanonicalFn} canonical
+ * @property {HashFn} [computeSha512]
+ */
+
+/**
+ * @typedef {Object} HashPowers
+ * @property {ReadFn} read
+ * @property {CanonicalFn} canonical
+ * @property {HashFn} computeSha512
*/
/**
@@ -170,6 +186,11 @@ export const moduleJSDocTypes = true;
* }>}
*/
+/**
+ * @typedef {Object} LoadArchiveOptions
+ * @property {string} [expectedSha512]
+ */
+
/**
* @typedef {Object} ExecuteOptions
* @property {Object} [globals]
@@ -222,6 +243,7 @@ export const moduleJSDocTypes = true;
* @typedef {Object} ModuleSource
* @property {string} [location]
* @property {Uint8Array} [bytes]
+ * @property {string} [sha512] in base16, hex
* @property {Language} [parser]
* @property {string} [exit]
* @property {StaticModuleType} [record]

MODIFY
packages/compartment-mapper/test/app.agar-make.js
packages/compartment-mapper/test/app.agar-make.js
import 'ses';
import fs from 'fs';
+import crypto from 'crypto';
import { writeArchive } from '../archive.js';
import { makeNodeReadPowers, makeNodeWritePowers } from '../src/node-powers.js';
-const readPowers = makeNodeReadPowers(fs);
+const readPowers = makeNodeReadPowers(fs, crypto);
const { write } = makeNodeWritePowers(fs);
const fixture = new URL(

MODIFY
packages/compartment-mapper/test/test-main.js
packages/compartment-mapper/test/test-main.js
import 'ses';
import fs from 'fs';
+import crypto from 'crypto';
import test from 'ava';
+import { ZipReader, ZipWriter } from '@endo/zip';
import {
loadLocation,
importLocation,
@@ -9,6 +11,7 @@ import {
parseArchive,
loadArchive,
importArchive,
+ hashLocation,
} from '../index.js';
import { makeNodeReadPowers } from '../src/node-powers.js';
@@ -17,7 +20,7 @@ const fixture = new URL(
import.meta.url,
).toString();
const archiveFixture = new URL('app.agar', import.meta.url).toString();
-const readPowers = makeNodeReadPowers(fs);
+const readPowers = makeNodeReadPowers(fs, crypto);
const globals = {
globalProperty: 42,
@@ -292,3 +295,109 @@ test('no transitive dev dependencies', async t => {
},
);
});
+
+test('makeArchive / parseArchive, but with sha512 corruption of a module', async t => {
+ t.plan(2);
+ await setup();
+
+ const { computeSha512 } = readPowers;
+
+ let i = 0;
+ const corruptSha512 = bytes => {
+ i += 1;
+ const hash = computeSha512(bytes);
+ if (i === 1) {
+ return `b4d${hash}`;
+ }
+ return hash;
+ };
+
+ const archive = await makeArchive(
+ {
+ ...readPowers,
+ computeSha512: corruptSha512,
+ },
+ fixture,
+ {
+ modules,
+ dev: true,
+ },
+ );
+
+ const application = await parseArchive(archive, 'app.agar', {
+ computeSha512,
+ });
+
+ t.assert(i > 0, 'no files were hashed');
+
+ await t.throwsAsync(
+ () =>
+ application.import({
+ globals,
+ globalLexicals,
+ modules,
+ Compartment,
+ }),
+ {
+ message: /failed a SHA-512 integrity check/,
+ },
+ );
+});
+
+test('makeArchive / parseArchive, but with sha512 corruption of a compartment map', async t => {
+ t.plan(1);
+ await setup();
+
+ const expectedSha512 = await hashLocation(readPowers, fixture, {
+ modules,
+ dev: true,
+ });
+
+ const archive = await makeArchive(readPowers, fixture, {
+ modules,
+ dev: true,
+ });
+
+ const reader = new ZipReader(archive);
+ const writer = new ZipWriter();
+ writer.files = reader.files;
+ // Corrupt compartment map
+ writer.write('compartment-map.json', new TextEncoder().encode('{}'));
+ const corruptArchive = writer.snapshot();
+
+ const { computeSha512 } = readPowers;
+
+ await t.throwsAsync(
+ () =>
+ parseArchive(corruptArchive, 'app.agar', {
+ computeSha512,
+ expectedSha512,
+ }),
+ {
+ message: /compartment map failed a SHA-512 integrity check/,
+ },
+ );
+});
+
+test('makeArchive / parseArchive / hashArchive consistency', async t => {
+ t.plan(0);
+ await setup();
+
+ const expectedSha512 = await hashLocation(readPowers, fixture, {
+ modules,
+ dev: true,
+ });
+
+ const archiveBytes = await makeArchive(readPowers, fixture, {
+ modules,
+ dev: true,
+ });
+
+ const { computeSha512 } = readPowers;
+ await parseArchive(archiveBytes, 'memory:app.agar', {
+ modules,
+ dev: true,
+ computeSha512,
+ expectedSha512,
+ });
+});

feat(compartment-mapper): Consistent hashing