feat(compartment-mapper): Add module transforms
MODIFY
packages/compartment-mapper/NEWS.md
packages/compartment-mapper/NEWS.md
User-visible changes to the compartment mapper:
+## Next release
+
+* Applications may now have asynchronous module transforms, per language.
+ When applied to archive creation, the transformed sources appear in the
+ archive.
+
## 0.2.3 (2020-11-05)
* Embellishes all calls to methods named `import` to work around SES-shim

MODIFY
packages/compartment-mapper/src/archive.js
packages/compartment-mapper/src/archive.js
@@ -89,7 +89,10 @@ const translateCompartmentMap = (compartments, sources, renames) => {
*/
const renameSources = (sources, renames) => {
return fromEntries(
- entries(sources).map(([name, compartment]) => [renames[name], compartment]),
+ entries(sources).map(([name, compartmentSources]) => [
+ renames[name],
+ compartmentSources,
+ ]),
);
};
@@ -116,9 +119,12 @@ const addSourcesToArchive = async (archive, sources) => {
/**
* @param {ReadFn} read
* @param {string} moduleLocation
+ * @param {Object} [options]
+ * @param {ModuleTransforms} [options.moduleTransforms]
* @returns {Promise<Uint8Array>}
*/
-export const makeArchive = async (read, moduleLocation) => {
+export const makeArchive = async (read, moduleLocation, options) => {
+ const { moduleTransforms } = options || {};
const {
packageLocation,
packageDescriptorText,
@@ -159,6 +165,7 @@ export const makeArchive = async (read, moduleLocation) => {
const compartment = assemble(compartmentMap, {
resolve,
makeImportHook,
+ moduleTransforms,
});
await compartment.load(entryModuleSpecifier);
@@ -197,13 +204,15 @@ export const makeArchive = async (read, moduleLocation) => {
* @param {ReadFn} read
* @param {string} archiveLocation
* @param {string} moduleLocation
+ * @param {ArchiveOptions} [options]
*/
export const writeArchive = async (
write,
read,
archiveLocation,
moduleLocation,
+ options,
) => {
- const archiveBytes = await makeArchive(read, moduleLocation);
+ const archiveBytes = await makeArchive(read, moduleLocation, options);
await write(archiveLocation, archiveBytes);
};

MODIFY
packages/compartment-mapper/src/assemble.js
packages/compartment-mapper/src/assemble.js
@@ -164,6 +164,7 @@ export const assemble = (
globals = {},
globalLexicals = {},
transforms = [],
+ moduleTransforms = {},
__shimTransforms__ = [],
modules: exitModules = {},
Compartment = defaultCompartment,
@@ -188,7 +189,7 @@ export const assemble = (
// The `moduleMapHook` writes back to the compartment map.
compartmentDescriptor.modules = modules;
- const parse = mapParsers(parsers, types);
+ const parse = mapParsers(parsers, types, moduleTransforms);
const importHook = makeImportHook(location, parse);
const moduleMapHook = makeModuleMapHook(
compartments,

MODIFY
packages/compartment-mapper/src/import-archive.js
packages/compartment-mapper/src/import-archive.js
@@ -9,7 +9,7 @@ import * as json from './json.js';
// q as in quote for strings in error messages.
const q = JSON.stringify;
-const decoder = new TextDecoder();
+const textDecoder = new TextDecoder();
/**
* @callback ArchiveImportHookMaker
@@ -50,13 +50,14 @@ const makeArchiveImportHookMaker = (archive, compartments, archiveLocation) => {
}
const moduleLocation = `${packageLocation}/${module.location}`;
const moduleBytes = await archive.read(moduleLocation);
- const moduleSource = decoder.decode(moduleBytes);
- return parse(
- moduleSource,
+ // eslint-disable-next-line no-await-in-loop
+ const { record } = await parse(
+ moduleBytes,
moduleSpecifier,
`file:///${moduleLocation}`,
packageLocation,
- ).record;
+ );
+ return record;
};
return importHook;
};
@@ -72,7 +73,7 @@ export const parseArchive = async (archiveBytes, archiveLocation) => {
const archive = await readZip(archiveBytes, archiveLocation);
const compartmentMapBytes = await archive.read('compartment-map.json');
- const compartmentMapText = decoder.decode(compartmentMapBytes);
+ const compartmentMapText = textDecoder.decode(compartmentMapBytes);
const compartmentMap = /** @type {CompartmentMapDescriptor} */ (json.parse(
compartmentMapText,
'compartment-map.json',
@@ -81,10 +82,7 @@ export const parseArchive = async (archiveBytes, archiveLocation) => {
// TODO validate compartmentMap instead of leaning hard on the above type
// assertion.
- /**
- * @param {ExecuteOptions} options
- * @returns {Promise<Object>}
- */
+ /** @type {ExecuteFn} */
const execute = options => {
const {
globals,
@@ -93,7 +91,7 @@ export const parseArchive = async (archiveBytes, archiveLocation) => {
transforms,
__shimTransforms__,
Compartment,
- } = options;
+ } = options || {};
const {
compartments,
entry: { module: moduleSpecifier },

MODIFY
packages/compartment-mapper/src/import-hook.js
packages/compartment-mapper/src/import-hook.js
@@ -4,8 +4,6 @@ import { parseExtension } from './extension.js';
// q, as in quote, for quoting strings in error messages.
const q = JSON.stringify;
-const decoder = new TextDecoder();
-
const { freeze } = Object;
/**
@@ -82,15 +80,14 @@ export const makeImportHookMaker = (
_error => undefined,
);
if (moduleBytes !== undefined) {
- const moduleSource = decoder.decode(moduleBytes);
-
- const envelope = parse(
- moduleSource,
+ // eslint-disable-next-line no-await-in-loop
+ const envelope = await parse(
+ moduleBytes,
candidateSpecifier,
moduleLocation,
packageLocation,
);
- const { parser } = envelope;
+ const { parser, bytes: transformedBytes } = envelope;
const { record: concreteRecord } = envelope;
// Facilitate a redirect if the returned record has a different
@@ -116,7 +113,7 @@ export const makeImportHookMaker = (
packageSources[candidateSpecifier] = {
location: packageRelativeLocation,
parser,
- bytes: moduleBytes,
+ bytes: transformedBytes,
};
return record;
}

MODIFY
packages/compartment-mapper/src/import.js
packages/compartment-mapper/src/import.js
// @ts-check
/* eslint no-shadow: "off" */
+import './types.js';
import { compartmentMapForNodeModules } from './node-modules.js';
import { search } from './search.js';
import { assemble } from './assemble.js';
@@ -10,9 +11,12 @@ import * as json from './json.js';
/**
* @param {ReadFn} read
* @param {string} moduleLocation
+ * @param {ArchiveOptions} [options]
* @returns {Promise<Application>}
*/
-export const loadLocation = async (read, moduleLocation) => {
+export const loadLocation = async (read, moduleLocation, options) => {
+ const { moduleTransforms = {} } = options || {};
+
const {
packageLocation,
packageDescriptorText,
@@ -34,6 +38,7 @@ export const loadLocation = async (read, moduleLocation) => {
moduleSpecifier,
);
+ /** @type {ExecuteFn} */
const execute = async (options = {}) => {
const {
globals,
@@ -50,6 +55,7 @@ export const loadLocation = async (read, moduleLocation) => {
globalLexicals,
modules,
transforms,
+ moduleTransforms,
__shimTransforms__,
Compartment,
});
@@ -64,12 +70,12 @@ export const loadLocation = async (read, moduleLocation) => {
/**
* @param {ReadFn} read
* @param {string} moduleLocation
- * @param {ExecuteOptions} options
+ * @param {ExecuteOptions & ArchiveOptions} [options]
* @returns {Promise<Object>} the object of the imported modules exported
* names.
*/
export const importLocation = async (read, moduleLocation, options = {}) => {
- const application = await loadLocation(read, moduleLocation);
+ const application = await loadLocation(read, moduleLocation, options);
// Call import by property to bypass SES censoring for dynamic import.
// eslint-disable-next-line dot-notation
return application['import'](options);

MODIFY
packages/compartment-mapper/src/parse.js
packages/compartment-mapper/src/parse.js
import { parseExtension } from './extension.js';
import * as json from './json.js';
+const textDecoder = new TextDecoder();
+
const { entries, freeze, fromEntries } = Object;
/**
@@ -22,15 +24,28 @@ const q = JSON.stringify;
// verification.
/** @type {ParseFn} */
-export const parseMjs = (source, _specifier, location, _packageLocation) => {
+export const parseMjs = async (
+ bytes,
+ _specifier,
+ location,
+ _packageLocation,
+) => {
+ const source = textDecoder.decode(bytes);
return {
parser: 'mjs',
+ bytes,
record: new StaticModuleRecord(source, location),
};
};
/** @type {ParseFn} */
-export const parseJson = (source, _specifier, location, _packageLocation) => {
+export const parseJson = async (
+ bytes,
+ _specifier,
+ location,
+ _packageLocation,
+) => {
+ const source = textDecoder.decode(bytes);
/** @type {Readonly<Array<string>>} */
const imports = freeze([]);
/**
@@ -41,54 +56,84 @@ export const parseJson = (source, _specifier, location, _packageLocation) => {
};
return {
parser: 'json',
+ bytes,
record: freeze({ imports, execute }),
};
};
+/** @type {Record<string, ParseFn>} */
+export const parserForLanguage = {
+ mjs: parseMjs,
+ json: parseJson,
+};
+
/**
- * @param {Object<string, ParseFn>} extensions - maps a file extension to the
- * corresponding parse function.
- * @param {Object<string, string>} types - In a rare case, the type of a module
- * is implied by package.json and should not be inferred from its extension.
+ * @param {Record<string, string>} languageForExtension - maps a file extension
+ * to the corresponding language.
+ * @param {Record<string, string>} languageForModuleSpecifier - In a rare case,
+ * the type of a module is implied by package.json and should not be inferred
+ * from its extension.
+ * @param {ModuleTransforms} transforms
* @returns {ParseFn}
*/
-export const makeExtensionParser = (extensions, types) => {
- return (source, specifier, location, packageLocation) => {
- let extension;
- if (Object(types) === types && has(types, specifier)) {
- extension = types[specifier];
+export const makeExtensionParser = (
+ languageForExtension,
+ languageForModuleSpecifier,
+ transforms,
+) => {
+ return async (bytes, specifier, location, packageLocation) => {
+ let language;
+ if (
+ Object(languageForModuleSpecifier) === languageForModuleSpecifier &&
+ has(languageForModuleSpecifier, specifier)
+ ) {
+ language = languageForModuleSpecifier[specifier];
} else {
- extension = parseExtension(location);
+ const extension = parseExtension(location);
+ if (!has(languageForExtension, extension)) {
+ throw new Error(
+ `Cannot parse module ${specifier} at ${location}, no parser configured for extension ${extension}`,
+ );
+ }
+ language = languageForExtension[extension];
}
- if (!has(extensions, extension)) {
+
+ if (has(transforms, language)) {
+ ({ bytes, parser: language } = await transforms[language](
+ bytes,
+ specifier,
+ location,
+ packageLocation,
+ ));
+ }
+
+ if (!has(parserForLanguage, language)) {
throw new Error(
- `Cannot parse module ${specifier} at ${location}, no parser configured for that extension`,
+ `Cannot parse module ${specifier} at ${location}, no parser configured for the language ${language}`,
);
}
- const parse = extensions[extension];
- return parse(source, specifier, location, packageLocation);
+ const parse = parserForLanguage[language];
+ return parse(bytes, specifier, location, packageLocation);
};
};
-/** @type {Record<string, ParseFn>} */
-export const parserForLanguage = {
- mjs: parseMjs,
- json: parseJson,
-};
-
/**
- * @param {Object<string, ParserDescriptor>} parsers
- * @param {Object<string, string>} types - In a rare case, the type of a module
+ * @param {Record<string, ParserDescriptor>} parsers
+ * @param {Record<string, string>} languageForModuleSpecifier - In a rare case, the type of a module
* is implied by package.json and should not be inferred from its extension.
+ * @param {ModuleTransforms} transforms
* @returns {ParseFn}
*/
-export const mapParsers = (parsers, types) => {
- const parserForExtension = [];
+export const mapParsers = (
+ parsers,
+ languageForModuleSpecifier,
+ transforms = {},
+) => {
+ const languageForExtension = [];
const errors = [];
for (const [extension, language] of entries(parsers)) {
if (has(parserForLanguage, language)) {
- const parser = parserForLanguage[language];
- parserForExtension.push([extension, parser]);
+ languageForExtension.push([extension, language]);
} else {
errors.push(`${q(language)} for extension ${q(extension)}`);
}
@@ -96,5 +141,9 @@ export const mapParsers = (parsers, types) => {
if (errors.length > 0) {
throw new Error(`No parser available for language: ${errors.join(', ')}`);
}
- return makeExtensionParser(fromEntries(parserForExtension), types);
+ return makeExtensionParser(
+ fromEntries(languageForExtension),
+ languageForModuleSpecifier,
+ transforms,
+ );
};

MODIFY
packages/compartment-mapper/src/types.js
packages/compartment-mapper/src/types.js
/**
* @callback ExecuteFn
- * @param {ExecuteOptions} options
+ * @param {ExecuteOptions} [options]
* @returns {Promise<Object>}
*/
/**
* @callback ParseFn
- * @param {string} source
+ * @param {Uint8Array} bytes
* @param {string} specifier
* @param {string} location
* @param {string} packageLocation
- * @returns {{
+ * @returns {Promise<{
+ * bytes: Uint8Array,
* parser: ParserDescriptor,
* record: FinalStaticModuleType,
- * }}
+ * }>}
*/
/**
/**
* @typedef {ExecuteOptions & Object} AssemblyOptions
* @property {AssembleImportHook} makeImportHook
+ * @property {ModuleTransforms} [moduleTransforms]
+ */
+
+/**
+ * @typedef {Record<string, ModuleTransform>} ModuleTransforms
+ */
+
+/**
+ * @callback ModuleTransform
+ * @param {Uint8Array} bytes
+ * @param {string} specifier
+ * @param {string} location
+ * @param {string} packageLocation
+ * @returns {Promise<{bytes: Uint8Array, parser: ParserDescriptor}>}
*/
// /////////////////////////////////////////////////////////////////////////////
* @property {ParserDescriptor} [parser]
* @property {string} [exit]
*/
+
+/**
+ * @typedef {Object} Artifact
+ * @property {Uint8Array} bytes
+ * @property {ParserDescriptor} parser
+ */
+
+/**
+ * @typedef {Object} ArchiveOptions
+ * @property {ModuleTransforms} [moduleTransforms]
+ */

MODIFY
packages/compartment-mapper/test/test-transform.js
packages/compartment-mapper/test/test-transform.js
@@ -3,6 +3,8 @@ import 'ses';
import fs from 'fs';
import test from 'ava';
import { loadLocation } from '../src/main.js';
+import { makeArchive } from '../src/archive.js';
+import { parseArchive } from '../src/import-archive.js';
test('transforms applied to evaluation', async t => {
t.plan(1);
@@ -28,3 +30,135 @@ test('transforms applied to evaluation', async t => {
const { default: value } = namespace;
t.is(value, 'heyo', 'code evaluated in compartment is transforemd');
});
+
+test('transforms applied to evaluation of archives', async t => {
+ t.plan(1);
+
+ const fixture = new URL(
+ 'node_modules/evaluator/evaluator.js',
+ import.meta.url,
+ ).toString();
+ const read = async location =>
+ fs.promises.readFile(new URL(location).pathname);
+
+ const archive = await makeArchive(read, fixture);
+ const application = await parseArchive(archive, fixture);
+ const { namespace } = await application.import({
+ globals: {
+ code: '"hello"',
+ },
+ transforms: [
+ function transform(source) {
+ return source.replace(/ll/g, 'y');
+ },
+ ],
+ });
+ const { default: value } = namespace;
+ t.is(value, 'heyo', 'code evaluated in compartment is transforemd');
+});
+
+test('module transforms applied while importing from file system', async t => {
+ t.plan(1);
+
+ const fixture = new URL(
+ 'node_modules/avery/avery.js',
+ import.meta.url,
+ ).toString();
+ const read = async location =>
+ fs.promises.readFile(new URL(location).pathname);
+
+ const application = await loadLocation(read, fixture, {
+ moduleTransforms: {
+ async mjs(sourceBytes) {
+ const source = new TextDecoder().decode(sourceBytes);
+ const object = source.toLowerCase();
+ const objectBytes = new TextEncoder().encode(object);
+ return { bytes: objectBytes, parser: 'mjs' };
+ },
+ },
+ });
+ const { namespace } = await application.import();
+ const { avery } = namespace;
+ // avery was Avery in the source
+ t.is(avery, 'avery', 'code entering archive is transformed');
+});
+
+test('module transforms applied while constructing archives', async t => {
+ t.plan(1);
+
+ const fixture = new URL(
+ 'node_modules/avery/avery.js',
+ import.meta.url,
+ ).toString();
+ const read = async location =>
+ fs.promises.readFile(new URL(location).pathname);
+
+ const archive = await makeArchive(read, fixture, {
+ moduleTransforms: {
+ async mjs(sourceBytes) {
+ const source = new TextDecoder().decode(sourceBytes);
+ const object = source.toLowerCase();
+ const objectBytes = new TextEncoder().encode(object);
+ return { bytes: objectBytes, parser: 'mjs' };
+ },
+ },
+ });
+ const application = await parseArchive(archive, fixture);
+ const { namespace } = await application.import();
+ const { avery } = namespace;
+ // avery was Avery in the source
+ t.is(avery, 'avery', 'code entering archive is transformed');
+});
+
+test('module transforms apply only to the intended language', async t => {
+ t.plan(1);
+
+ const fixture = new URL(
+ 'node_modules/avery/avery.js',
+ import.meta.url,
+ ).toString();
+ const read = async location =>
+ fs.promises.readFile(new URL(location).pathname);
+
+ const archive = await makeArchive(read, fixture, {
+ moduleTransforms: {
+ async json() {
+ // Erase JSON modules. (There are no JSON modules.)
+ t.fail();
+ const objectBytes = new Uint8Array();
+ return { bytes: objectBytes, parser: 'mjs' };
+ },
+ },
+ });
+ const application = await parseArchive(archive, fixture);
+ const { namespace } = await application.import();
+ const { avery } = namespace;
+ t.is(avery, 'Avery', 'non-JSON module is not transformed');
+});
+
+test('module transforms can be used to translate JSON to JS', async t => {
+ t.plan(2);
+
+ const fixture = new URL(
+ 'node_modules/typeparsers/json.json',
+ import.meta.url,
+ ).toString();
+ const read = async location =>
+ fs.promises.readFile(new URL(location).pathname);
+
+ const archive = await makeArchive(read, fixture, {
+ moduleTransforms: {
+ async json(sourceBytes) {
+ t.is(1, 1); // plan + 1
+ const source = new TextDecoder().decode(sourceBytes);
+ const object = `export default ${source.trim()};\n`;
+ const objectBytes = new TextEncoder().encode(object);
+ return { bytes: objectBytes, parser: 'mjs' };
+ },
+ },
+ });
+ const application = await parseArchive(archive, fixture);
+ const { namespace } = await application.import();
+ const { default: value } = namespace;
+ t.is(value, 42, 'JSON module transformed');
+});

feat(compartment-mapper): Add module transforms (#625)