refactor: improve TypeScript custom transformers (#97)

.eslintignore

@@ -10,5 +10,6 @@ node_modules
 /packages/emnapi/lib
 /packages/core/src/index.js
 /packages/**/test/**/input
+/packages/**/test/**/actual
 /packages/**/test/**/expected
 /out

.github/workflows/main.yml

@@ -2,12 +2,18 @@ name: Build
 
 on:
   push:
+    paths-ignore:
+    - '**/*.md'
+    - '**/docs/**'
     branches:
     - main
     - test-*
     tags:
     - v*
   pull_request:
+    paths-ignore:
+    - '**/*.md'
+    - '**/docs/**'
   workflow_dispatch:
 
 env:

CONTRIBUTING.md

@@ -8,16 +8,52 @@ This doc will explain the structure of this project and some points need to note
 
 - `packages/emnapi` (`devDependencies`)
 
-    The main package of emnapi, including emnapi C headers, CMake configurations and
-    the [Emscripten JavaScript library](https://emscripten.org/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.html#implement-a-c-api-in-javascript) build.
+    The main package of emnapi, including all Node-API implementation, emnapi C headers, CMake configurations, and
+    the [Emscripten JavaScript library](https://emscripten.org/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.html#implement-a-c-api-in-javascript) build. We finally need to build a Emscripten JavaScript library file
+    which is designed for being used in link time. The library file is not a regular JavaScript file,
+    it may contain some macros wrapped in `{{{  }}}` only available in Emscripten, and can not use ES Module format.
+    Fortunately, we can write ESM code during development and then build the final library file through
+    TypeScript custom transformers those are also in this repo.
 
-    The TypeScript `module` compiler option for this package is set to `none`, because we finally need to build
-    a Emscripten JavaScript library file which is designed for being used in link time. The library file is not
-    a regular JavaScript file, it may contain some macros wrapped in `{{{  }}}` only available in Emscripten,
-    and all function body string will be inlined to runtime code, so the module system and closures are not
-    available.
+    For example, write the following source code:
 
-    For example, the `$makeSetValue(...)` in TypeScript source code will be transformed to `{{{ makeSetValue(...) }}}`
+    ```ts
+    import { makeSetValue } from 'emscripten:parse-tools'
+
+    /** @__sig vp */
+    export function foo (result: number) {
+      makeSetValue('result', 0, 0, 'i32')
+    }
+    ```
+
+    This will output:
+
+    ```js
+    function _foo (result) {
+      {{{ makeSetValue('result', 0, 0, 'i32') }}}
+    }
+    addToLibrary({
+      foo: _foo,
+      foo__sig: 'vp'
+    })
+    ```
+
+    It is powered by `packages/rollup-plugin-emscripten-esm-library` and `packages/ts-transform-emscripten-esm-library`.
+
+    In addition, macros are also heavily used in `packages/emnapi` to match the Node.js source as much as possible.
+
+    ```ts
+    import { $CHECK_ARG, $CHECK_ENV_NOT_IN_GC } from '...'
+
+    /** @__sig ipip */
+    export function napi_create_int32 (env: napi_env, value: int32_t, result: Pointer<napi_value>): napi_status {
+      const envObject: Env = $CHECK_ENV_NOT_IN_GC!(env)
+      $CHECK_ARG!(envObject, result)
+      // ...
+    }
+    ```
+
+    Macros are powered by `packages/ts-transform-macro`
 
 - `packages/core` (`dependencies`)
 
@@ -47,9 +83,11 @@ This doc will explain the structure of this project and some points need to note
 - [CMake](https://github.com/Kitware/CMake) `>= 3.13`
 - [ninja-build](https://github.com/ninja-build/ninja)
 
-## Macro
-
-Macro is heavily used in `packages/emnapi`, there are two kinds of macro.
+## Debugging
 
-- `$macroName(...)`: transformed to `{{{ macroName(...) }}}`
-- `$CUSTOM_MACRO!(...)`: powered by `packages/ts-transform-macro`
+- Run `npm run build` to build packages
+- Open `packages/test/**/*.js`
+- Add break points
+- Launch `Launch Test` VSCode launch configuration
+- Input `UV_THREADPOOL_SIZE` environment variable
+- Select target tripple

package.json

@@ -52,8 +52,8 @@
   },
   "workspaces": [
     "packages/ts-transform-macro",
-    "packages/ts-transform-emscripten-parse-tools",
     "packages/ts-transform-emscripten-esm-library",
+    "packages/ts-transform-emscripten-parse-tools",
     "packages/rollup-plugin-emscripten-esm-library",
     "packages/runtime",
     "packages/node",

packages/emnapi/README.md

@@ -109,7 +109,7 @@ Create `hello.c`.
 ```c
 #include <node_api.h>
 
-#define NODE_API_CALL(env, the_call)                                \
+#define NODE_API_CALL(env, the_call)                            \
   do {                                                          \
     if ((the_call) != napi_ok) {                                \
       const napi_extended_error_info *error_info;               \

packages/emnapi/script/build.js

@@ -18,6 +18,10 @@ async function build () {
   const libOut = path.join(path.dirname(libTsconfigPath), './dist/library_napi.js')
   const runtimeRequire = createRequire(path.join(__dirname, '../../runtime/index.js'))
 
+  const runtimeModuleSpecifier = 'emscripten:runtime'
+  const parseToolsModuleSpecifier = 'emscripten:parse-tools'
+  const sharedModuleSpecifier = 'emnapi:shared'
+
   const emnapiRollupBuild = await rollup({
     input: path.join(__dirname, '../src/emscripten/index.ts'),
     treeshake: false,
@@ -37,45 +41,20 @@ async function build () {
             {
               type: 'program',
               factory: require('@emnapi/ts-transform-macro').createTransformerFactory
-            },
-            {
-              type: 'program',
-              factory: () => {
-                return (context) => {
-                  return (src) => {
-                    if (src.isDeclarationFile) return src
-                    const statements = src.statements
-                    const newStatements = statements.filter(s => {
-                      return !(ts.isImportDeclaration(s) && ts.isStringLiteral(s.moduleSpecifier) && (s.moduleSpecifier.text === 'emnapi:emscripten-runtime'))
-                    })
-                    return context.factory.updateSourceFile(src, newStatements)
-                  }
-                }
-              }
             }
           ]
         }
       }),
       rollupAlias({
         entries: [
-          { find: 'emnapi:shared', replacement: path.join(__dirname, '../src/emscripten/init.ts') }
+          { find: sharedModuleSpecifier, replacement: path.join(__dirname, '../src/emscripten/init.ts') }
         ]
       }),
       require('@emnapi/rollup-plugin-emscripten-esm-library').default({
         defaultLibraryFuncsToInclude: ['$emnapiInit'],
         exportedRuntimeMethods: ['emnapiInit'],
-        processDirective: true,
-        modifyOutput (output) {
-          return output
-            .replace(/\$POINTER_SIZE/g, '{{{ POINTER_SIZE }}}')
-            .replace(/\$(from64\(.*?\))/g, '{{{ $1 }}}')
-            .replace(/\$(to64\(.*?\))/g, '{{{ $1 }}}')
-            .replace(/\$(makeGetValue\(.*?\))/g, '{{{ $1 }}}')
-            .replace(/\$(makeSetValue\(.*?\))/g, '{{{ $1 }}}')
-            .replace(/\$(makeDynCall\(.*?\))/g, '{{{ $1 }}}')
-            // .replace(/\$(makeMalloc\(.*?\))/g, '{{{ $1 }}}')
-            .replace(/\$(getUnsharedTextDecoderView\(.*?\))/g, '{{{ $1 }}}')
-        }
+        runtimeModuleSpecifier,
+        parseToolsModuleSpecifier
       })
     ]
   })
@@ -119,7 +98,9 @@ async function build () {
                 return require('@emnapi/ts-transform-emscripten-parse-tools').createTransformerFactory(program, {
                   defines: {
                     MEMORY64: 0
-                  }
+                  },
+                  runtimeModuleSpecifier,
+                  parseToolsModuleSpecifier
                 })
               }
             }
@@ -128,8 +109,8 @@ async function build () {
       }),
       rollupAlias({
         entries: [
-          { find: 'emnapi:shared', replacement: path.join(__dirname, '../src/core/init.ts') },
-          { find: 'emnapi:emscripten-runtime', replacement: path.join(__dirname, '../src/core/init.ts') }
+          { find: sharedModuleSpecifier, replacement: path.join(__dirname, '../src/core/init.ts') },
+          { find: runtimeModuleSpecifier, replacement: path.join(__dirname, '../src/core/init.ts') }
         ]
       })
     ]

packages/emnapi/src/async-work.ts

@@ -1,4 +1,5 @@
 import { emnapiAsyncWorkPoolSize, emnapiNodeBinding, emnapiCtx } from 'emnapi:shared'
+import { makeDynCall } from 'emscripten:parse-tools'
 
 export interface AsyncWork {
   id: number
@@ -95,7 +96,7 @@ export var emnapiAWST = {
       const scope = emnapiCtx.openScope(envObject)
       try {
         (envObject as NodeEnv).callbackIntoModule(true, () => {
-          $makeDynCall('vpip', 'complete')(env, status, data)
+          makeDynCall('vpip', 'complete')(env, status, data)
         })
       } finally {
         emnapiCtx.closeScope(envObject, scope)
@@ -128,7 +129,7 @@ export var emnapiAWST = {
       const execute = work.execute
       work.status = 2
       emnapiCtx.feature.setImmediate(() => {
-        $makeDynCall('vpp', 'execute')(env, data)
+        makeDynCall('vpp', 'execute')(env, data)
         emnapiAWST.queued.delete(id)
         work.status = 3
 

packages/emnapi/src/core/async-work.ts

@@ -1,5 +1,6 @@
 import { emnapiCtx, onCreateWorker, napiModule, emnapiNodeBinding, singleThreadAsyncWork, _emnapi_async_work_pool_size } from 'emnapi:shared'
-import { PThread, ENVIRONMENT_IS_NODE, ENVIRONMENT_IS_PTHREAD, wasmInstance, _free, wasmMemory, _malloc } from 'emnapi:emscripten-runtime'
+import { PThread, ENVIRONMENT_IS_NODE, ENVIRONMENT_IS_PTHREAD, wasmInstance, _free, wasmMemory, _malloc } from 'emscripten:runtime'
+import { POINTER_SIZE, to64, makeDynCall, makeSetValue, from64 } from 'emscripten:parse-tools'
 import { emnapiAWST } from '../async-work'
 import { $CHECK_ENV_NOT_IN_GC, $CHECK_ARG, $CHECK_ENV } from '../macro'
 import { _emnapi_node_emit_async_init, _emnapi_node_emit_async_destroy } from '../node'
@@ -16,10 +17,10 @@ var emnapiAWMT = {
     /* double */ async_id: 8,
     /* double */ trigger_async_id: 16,
     /* napi_env */ env: 24,
-    /* void* */ data: 1 * $POINTER_SIZE + 24,
-    /* napi_async_execute_callback */ execute: 2 * $POINTER_SIZE + 24,
-    /* napi_async_complete_callback */ complete: 3 * $POINTER_SIZE + 24,
-    end: 4 * $POINTER_SIZE + 24
+    /* void* */ data: 1 * POINTER_SIZE + 24,
+    /* napi_async_execute_callback */ execute: 2 * POINTER_SIZE + 24,
+    /* napi_async_complete_callback */ complete: 3 * POINTER_SIZE + 24,
+    end: 4 * POINTER_SIZE + 24
   },
   init () {
     emnapiAWMT.unusedWorkers = []
@@ -107,7 +108,7 @@ var emnapiAWMT = {
       for (let i = 0; i < n; ++i) {
         // eslint-disable-next-line @typescript-eslint/no-unused-vars
         const arg = args[i]
-        _free($to64('arg'))
+        _free(to64('arg'))
       }
       throw err
     }
@@ -209,7 +210,7 @@ var emnapiAWMT = {
     const callback = (): void => {
       if (!complete) return
       (envObject as NodeEnv).callbackIntoModule(true, () => {
-        $makeDynCall('vpip', 'complete')(env, status, data)
+        makeDynCall('vpip', 'complete')(env, status, data)
       })
     }
 
@@ -254,7 +255,7 @@ export var napi_create_async_work = singleThreadAsyncWork
 
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     const id = emnapiAWST.create(env, resourceObject, resourceName, execute, complete, data)
-    $makeSetValue('result', 0, 'id', '*')
+    makeSetValue('result', 0, 'id', '*')
     return envObject.clearLastError()
   }
   : function (env: napi_env, resource: napi_value, resource_name: napi_value, execute: number, complete: number, data: number, result: number): napi_status {
@@ -272,21 +273,21 @@ export var napi_create_async_work = singleThreadAsyncWork
     $CHECK_ARG!(envObject, resource_name)
 
     const sizeofAW = emnapiAWMT.offset.end
-    const aw = _malloc($to64('sizeofAW'))
+    const aw = _malloc(to64('sizeofAW'))
     if (!aw) return envObject.setLastError(napi_status.napi_generic_failure)
     new Uint8Array(wasmMemory.buffer).subarray(aw, aw + sizeofAW).fill(0)
     const s = envObject.ensureHandleId(resourceObject)
     const resourceRef = emnapiCtx.createReference(envObject, s, 1, Ownership.kUserland as any)
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     const resource_ = resourceRef.id
-    $makeSetValue('aw', 0, 'resource_', '*')
+    makeSetValue('aw', 0, 'resource_', '*')
     _emnapi_node_emit_async_init(s, resource_name, -1, aw + emnapiAWMT.offset.async_id)
-    $makeSetValue('aw', 'emnapiAWMT.offset.env', 'env', '*')
-    $makeSetValue('aw', 'emnapiAWMT.offset.execute', 'execute', '*')
-    $makeSetValue('aw', 'emnapiAWMT.offset.complete', 'complete', '*')
-    $makeSetValue('aw', 'emnapiAWMT.offset.data', 'data', '*')
-    $from64('result')
-    $makeSetValue('result', 0, 'aw', '*')
+    makeSetValue('aw', 'emnapiAWMT.offset.env', 'env', '*')
+    makeSetValue('aw', 'emnapiAWMT.offset.execute', 'execute', '*')
+    makeSetValue('aw', 'emnapiAWMT.offset.complete', 'complete', '*')
+    makeSetValue('aw', 'emnapiAWMT.offset.data', 'data', '*')
+    from64('result')
+    makeSetValue('result', 0, 'aw', '*')
     return envObject.clearLastError()
   }
 
@@ -313,7 +314,7 @@ export var napi_delete_async_work = singleThreadAsyncWork
       _emnapi_node_emit_async_destroy(asyncId, triggerAsyncId)
     }
 
-    _free($to64('work') as number)
+    _free(to64('work') as number)
     return envObject.clearLastError()
   }
 
@@ -373,7 +374,7 @@ function executeAsyncWork (work: number): void {
   const execute = emnapiAWMT.getExecute(work)
   const env = emnapiAWMT.getEnv(work)
   const data = emnapiAWMT.getData(work)
-  $makeDynCall('vpp', 'execute')(env, data)
+  makeDynCall('vpp', 'execute')(env, data)
   const postMessage = napiModule.postMessage!
   postMessage({
     __emnapi__: {

packages/emnapi/src/core/async.ts

@@ -1,7 +1,8 @@
 /* eslint-disable @typescript-eslint/no-floating-promises */
 
 import { napiModule } from 'emnapi:shared'
-import { ENVIRONMENT_IS_NODE, wasmMemory, ENVIRONMENT_IS_PTHREAD, PThread } from 'emnapi:emscripten-runtime'
+import { ENVIRONMENT_IS_NODE, wasmMemory, ENVIRONMENT_IS_PTHREAD, PThread } from 'emscripten:runtime'
+import { POINTER_SIZE, makeDynCall, makeGetValue, to64 } from 'emscripten:parse-tools'
 import { _emnapi_set_immediate, _emnapi_next_tick } from '../util'
 
 function emnapiGetWorkerByPthreadPtr (pthreadPtr: number): any {
@@ -76,10 +77,10 @@ export function _emnapi_is_main_browser_thread (): number {
 /** @__sig vppi */
 export function _emnapi_after_uvthreadpool_ready (callback: number, q: number, type: number): void {
   if (uvThreadpoolReady.ready) {
-    $makeDynCall('vpi', 'callback')($to64('q'), type)
+    makeDynCall('vpi', 'callback')(to64('q'), type)
   } else {
     uvThreadpoolReady.then(() => {
-      $makeDynCall('vpi', 'callback')($to64('q'), type)
+      makeDynCall('vpi', 'callback')(to64('q'), type)
     })
   }
 }
@@ -88,7 +89,7 @@ export function _emnapi_after_uvthreadpool_ready (callback: number, q: number, t
 export function _emnapi_tell_js_uvthreadpool (threads: number, size: number): void {
   const p = [] as Array<Promise<void>>
   for (let i = 0; i < size; i++) {
-    const pthreadPtr = $makeGetValue('threads', 'i * ' + POINTER_SIZE, '*')
+    const pthreadPtr = makeGetValue('threads', 'i * ' + POINTER_SIZE, '*')
     const worker = emnapiGetWorkerByPthreadPtr(pthreadPtr)
     p.push(new Promise<void>((resolve) => {
       const handler = function (e: any): void {