feat(compartment-mapper): Introduce rudimentary bundler
MODIFY
packages/compartment-mapper/NEWS.md
packages/compartment-mapper/NEWS.md
@@ -6,6 +6,9 @@ User-visible changes to the compartment mapper:
Babel.
* The Compartment Mapper now produces archives containing SES-shim
pre-compiled StaticModuleRecords for ESM instead of the source.
+* The Compartment Mapper can now produce bundles of concatenated modules but
+ without Compartments and only supporting ESM but not supporting live
+ bindings.
* *BREAKING*: Archives created for the previous version will no longer work.
The `importArchive` feature only supports pre-compiled ESM and CJS.
* *BREAKING*: This release parallels a breaking upgrade for SES to version

ADD
packages/compartment-mapper/src/bundle.js
+// @ts-check
+/* eslint no-shadow: 0 */
+
+import { resolve } from './node-module-specifier.js';
+import { compartmentMapForNodeModules } from './node-modules.js';
+import { search } from './search.js';
+import { link } from './assemble.js';
+import { makeImportHookMaker } from './import-hook.js';
+import { parseJson } from './parse-json.js';
+import { parseArchiveCjs } from './parse-archive-cjs.js';
+import { parseArchiveMjs } from './parse-archive-mjs.js';
+import { parseLocatedJson } from './json.js';
+
+const textEncoder = new TextEncoder();
+
+/** quotes strings */
+const q = JSON.stringify;
+
+/** @type {Record<string, ParseFn>} */
+export const parserForLanguage = {
+ mjs: parseArchiveMjs,
+ cjs: parseArchiveCjs,
+ json: parseJson,
+};
+
+/**
+ * @param {Record<string, CompartmentDescriptor>} compartmentDescriptors
+ * @param {Record<string, CompartmentSources>} compartmentSources
+ * @param {Record<string, ResolveHook>} compartmentResolvers
+ * @param {string} entryCompartmentName
+ * @param {string} entryModuleSpecifier
+ */
+const sortedModules = (
+ compartmentDescriptors,
+ compartmentSources,
+ compartmentResolvers,
+ entryCompartmentName,
+ entryModuleSpecifier,
+) => {
+ const modules = [];
+ const seen = new Set();
+
+ /**
+ * @param {string} compartmentName
+ * @param {string} moduleSpecifier
+ */
+ const recur = (compartmentName, moduleSpecifier) => {
+ const key = `${compartmentName}#${moduleSpecifier}`;
+ if (seen.has(key)) {
+ return key;
+ }
+ seen.add(key);
+
+ const resolve = compartmentResolvers[compartmentName];
+ const source = compartmentSources[compartmentName][moduleSpecifier];
+ if (source) {
+ const { record, parser } = source;
+ if (record) {
+ const { imports = [], reexports = [] } = record;
+ const resolvedImports = {};
+ for (const importSpecifier of [...imports, ...reexports]) {
+ const resolvedSpecifier = resolve(importSpecifier, moduleSpecifier);
+ resolvedImports[importSpecifier] = recur(
+ compartmentName,
+ resolvedSpecifier,
+ );
+ }
+
+ modules.push({
+ key,
+ compartmentName,
+ moduleSpecifier,
+ parser,
+ record,
+ resolvedImports,
+ });
+
+ return key;
+ }
+ } else {
+ const descriptor =
+ compartmentDescriptors[compartmentName].modules[moduleSpecifier];
+ if (descriptor) {
+ const {
+ compartment: aliasCompartmentName,
+ module: aliasModuleSpecifier,
+ } = descriptor;
+ if (
+ aliasCompartmentName !== undefined &&
+ aliasModuleSpecifier !== undefined
+ ) {
+ return recur(aliasCompartmentName, aliasModuleSpecifier);
+ }
+ }
+ }
+
+ throw new Error(
+ `Cannot bundle: cannot follow module import ${moduleSpecifier} in compartment ${compartmentName}`,
+ );
+ };
+
+ recur(entryCompartmentName, entryModuleSpecifier);
+
+ return modules;
+};
+
+/**
+ * @param {ReadFn} read
+ * @param {string} moduleLocation
+ * @param {Object} [options]
+ * @param {ModuleTransforms} [options.moduleTransforms]
+ * @returns {Promise<string>}
+ */
+export const makeBundle = async (read, moduleLocation, options) => {
+ const { moduleTransforms } = options || {};
+ const {
+ packageLocation,
+ packageDescriptorText,
+ packageDescriptorLocation,
+ moduleSpecifier,
+ } = await search(read, moduleLocation);
+
+ /** @type {Set<string>} */
+ const tags = new Set();
+
+ const packageDescriptor = parseLocatedJson(
+ packageDescriptorText,
+ packageDescriptorLocation,
+ );
+ const compartmentMap = await compartmentMapForNodeModules(
+ read,
+ packageLocation,
+ tags,
+ packageDescriptor,
+ moduleSpecifier,
+ );
+
+ const {
+ compartments,
+ entry: { compartment: entryCompartmentName, module: entryModuleSpecifier },
+ } = compartmentMap;
+ /** @type {Sources} */
+ const sources = {};
+
+ const makeImportHook = makeImportHookMaker(
+ read,
+ packageLocation,
+ sources,
+ compartments,
+ );
+
+ // Induce importHook to record all the necessary modules to import the given module specifier.
+ const { compartment, resolvers } = link(compartmentMap, {
+ resolve,
+ makeImportHook,
+ moduleTransforms,
+ parserForLanguage,
+ });
+ await compartment.load(entryModuleSpecifier);
+
+ const modules = sortedModules(
+ compartmentMap.compartments,
+ sources,
+ resolvers,
+ entryCompartmentName,
+ entryModuleSpecifier,
+ );
+
+ // Create an index of modules so we can resolve import specifiers to the
+ // index of the corresponding functor.
+ const modulesByKey = {};
+ for (let index = 0; index < modules.length; index += 1) {
+ const module = modules[index];
+ module.index = index;
+ modulesByKey[module.key] = module;
+ }
+ for (const module of modules) {
+ module.indexedImports = Object.fromEntries(
+ Object.entries(module.resolvedImports).map(([importSpecifier, key]) => [
+ importSpecifier,
+ modulesByKey[key].index,
+ ]),
+ );
+ }
+
+ // Only support mjs format.
+ const problems = modules
+ .filter(module => module.parser !== 'premjs')
+ .map(
+ ({ moduleSpecifier, compartmentName, parser }) =>
+ `module ${moduleSpecifier} in compartment ${compartmentName} in language ${parser}`,
+ );
+ if (problems.length) {
+ throw new Error(
+ `Can only bundle applications that only have ESM (.mjs-type) modules, got ${problems.join(
+ ', ',
+ )}`,
+ );
+ }
+
+ const bundle = `\
+(functors => {
+ function cell(name, value = undefined) {
+ const observers = [];
+ function set(newValue) {
+ value = newValue;
+ for (const observe of observers) {
+ observe(value);
+ }
+ }
+ function get() {
+ return value;
+ }
+ function observe(observe) {
+ observers.push(observe);
+ observe(value);
+ }
+ return { get, set, observe, enumerable: true };
+ }
+
+ const cells = [${''.concat(
+ ...modules.map(
+ ({ record: { __fixedExportMap__, __liveExportMap__ } }) => `{
+ ${''.concat(
+ ...Object.keys(__fixedExportMap__).map(
+ exportName => `${exportName}: cell(${q(exportName)}),\n`,
+ ),
+ )}
+ ${''.concat(
+ ...Object.keys(__liveExportMap__).map(
+ exportName => `${exportName}: cell(${q(exportName)}),\n`,
+ ),
+ )}
+ },`,
+ ),
+ )}];
+
+ ${''.concat(
+ ...modules.flatMap(({ index, indexedImports, record: { reexports } }) =>
+ reexports.map(
+ (/* @type {string} */ importSpecifier) => `\
+ Object.defineProperties(cells[${index}], Object.getOwnPropertyDescriptors(cells[${indexedImports[importSpecifier]}]));
+ `,
+ ),
+ ),
+ )}
+
+ const namespaces = cells.map(cells => Object.create(null, cells));
+
+ for (let index = 0; index < namespaces.length; index += 1) {
+ cells[index]['*'] = cell('*', namespaces[index]);
+ }
+
+ ${''.concat(
+ ...modules.map(
+ ({
+ index,
+ indexedImports,
+ record: { __liveExportMap__, __fixedExportMap__ },
+ }) => `\
+ functors[${index}]({
+ imports(map) {
+ ${''.concat(
+ ...Object.entries(indexedImports).map(
+ ([importName, importIndex]) => `\
+ for (const [name, observers] of map.get(${q(
+ importName,
+ )}).entries()) {
+ const cell = cells[${importIndex}][name];
+ if (cell === undefined) {
+ throw new ReferenceError(\`Cannot import name \${name}\`);
+ }
+ for (const observer of observers) {
+ cell.observe(observer);
+ }
+ }
+ `,
+ ),
+ )}
+ },
+ liveVar: {
+ ${''.concat(
+ ...Object.entries(__liveExportMap__).map(
+ ([exportName, [importName]]) => `\
+ ${importName}: cells[${index}].${exportName}.set,
+ `,
+ ),
+ )}
+ },
+ onceVar: {
+ ${''.concat(
+ ...Object.entries(__fixedExportMap__).map(
+ ([exportName, [importName]]) => `\
+ ${importName}: cells[${index}].${exportName}.set,
+ `,
+ ),
+ )}
+ },
+ });
+ `,
+ ),
+ )}
+
+})([
+ ${''.concat(
+ ...modules.map(
+ ({ record: { __syncModuleProgram__ } }) =>
+ `${__syncModuleProgram__}\n,\n`,
+ ),
+ )}
+]);
+`;
+
+ return bundle;
+};
+
+/**
+ * @param {WriteFn} write
+ * @param {ReadFn} read
+ * @param {string} bundleLocation
+ * @param {string} moduleLocation
+ * @param {ArchiveOptions} [options]
+ */
+export const writeBundle = async (
+ write,
+ read,
+ bundleLocation,
+ moduleLocation,
+ options,
+) => {
+ const bundleString = await makeBundle(read, moduleLocation, options);
+ const bundleBytes = textEncoder.encode(bundleString);
+ await write(bundleLocation, bundleBytes);
+};

MODIFY
packages/compartment-mapper/src/main.js
packages/compartment-mapper/src/main.js
@@ -4,3 +4,4 @@ export { makeArchive, writeArchive } from './archive.js';
export { parseArchive, loadArchive, importArchive } from './import-archive.js';
export { search } from './search.js';
export { compartmentMapForNodeModules } from './node-modules.js';
+export { makeBundle, writeBundle } from './bundle.js';

MODIFY
packages/compartment-mapper/src/types.js
packages/compartment-mapper/src/types.js
* package.json, there is a corresponding module descriptor.
*
* @typedef {Object} ModuleDescriptor
- * @property {string} [compartment]
+ * @property {string=} [compartment]
* @property {string} [module]
* @property {string} [location]
* @property {Language} [parser]

ADD
packages/compartment-mapper/test/node_modules/bundle/import-all-from-me.js
+export const c = 'sea';
+export const i = 'eye';
+export const k = 'que';
+export const q = 'cue';
+export const u = 'you';
+export const y = 'why';

ADD
packages/compartment-mapper/test/node_modules/bundle/import-and-export-all.js
+export const red = '#f00';
+export const green = '#0f0';
+export const blue = '#00f';

ADD
packages/compartment-mapper/test/node_modules/bundle/import-and-reexport-name-from-me.js
+export const qux = 'qux';

ADD
packages/compartment-mapper/test/node_modules/bundle/import-default-export-from-me.js
+export default 'foo';

ADD
packages/compartment-mapper/test/node_modules/bundle/import-named-export-and-rename.js
+export const color = 'blue';

ADD
packages/compartment-mapper/test/node_modules/bundle/import-named-exports-from-me.js
+const fizz = 'fizz';
+const bizz = 'bizz';
+const buzz = 'buzz';
+export { fizz, bizz, buzz };

ADD
packages/compartment-mapper/test/node_modules/bundle/main.js
+/* global print */
+
+import foo from './import-default-export-from-me.js';
+print(foo);
+
+import * as bar from './import-all-from-me.js';
+print(bar);
+
+import { fizz, buzz } from './import-named-exports-from-me.js';
+print(fizz);
+print(buzz);
+
+import { color as colour } from './import-named-export-and-rename.js';
+print(colour);
+
+import { qux } from './reexport-name.js';
+print(qux);
+
+import * as colors from './reexport-all.js';
+print(colors);

ADD
packages/compartment-mapper/test/node_modules/bundle/package.json
+{
+ "name": "bundle",
+ "parsers": {"js": "mjs"},
+ "main": "./main.js"
+}

ADD
packages/compartment-mapper/test/node_modules/bundle/reexport-all.js
+export * from './import-and-export-all.js';

ADD
packages/compartment-mapper/test/node_modules/bundle/reexport-name.js
+export { qux } from './import-and-reexport-name-from-me.js';

ADD
packages/compartment-mapper/test/test-bundle.js
+import 'ses';
+import fs from 'fs';
+import test from 'ava';
+import { makeBundle, makeArchive, parseArchive } from '../src/main.js';
+
+const fixture = new URL(
+ 'node_modules/bundle/main.js',
+ import.meta.url,
+).toString();
+
+const read = async location => fs.promises.readFile(new URL(location).pathname);
+
+const expectedLog = [
+ 'foo',
+ {
+ c: 'sea',
+ i: 'eye',
+ q: 'cue',
+ k: 'que',
+ u: 'you',
+ y: 'why',
+ },
+ 'fizz',
+ 'buzz',
+ 'blue',
+ 'qux',
+ {
+ red: '#f00',
+ green: '#0f0',
+ blue: '#00f',
+ },
+];
+
+test('bundles work', async t => {
+ const bundle = await makeBundle(read, fixture);
+ t.log(bundle);
+ const log = [];
+ const print = entry => {
+ log.push(entry);
+ };
+ const compartment = new Compartment({ print });
+ compartment.evaluate(bundle);
+ t.deepEqual(log, expectedLog);
+});
+
+test('equivalent archive behaves the same as bundle', async t => {
+ const log = [];
+ const print = entry => {
+ log.push(entry);
+ };
+
+ const archive = await makeArchive(read, fixture);
+ const application = await parseArchive(archive, fixture);
+ await application.import({
+ globals: { print },
+ });
+ t.deepEqual(log, expectedLog);
+});

feat(compartment-mapper): Introduce rudimentary bundler