Merge pull request #605 from YOU54F/feat/optional_arch_specific_binaries

Feat/optional arch specific binaries

Author: YOU54F

.github/workflows/build-and-test.yml

@@ -64,33 +64,36 @@ jobs:
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        if: github.ref == 'refs/heads/master'
         with:
           fetch-depth: 0
 
       - name: Use Node.js ${{ env.NODE_VERSION }}
         uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
+        if: github.ref == 'refs/heads/master'
         with:
           node-version: ${{ env.NODE_VERSION }}
 
-      - if: runner.os == 'Windows'
+      - if: ${{ runner.os == 'Windows' && github.ref == 'refs/heads/master'}}
         run: echo "ONLY_DOWNLOAD_PACT_FOR_WINDOWS=true" >> $GITHUB_ENV
 
-      - if: ${{ matrix.docker == true && matrix.alpine == true }}
+      - if: ${{ matrix.docker == true && matrix.alpine == true && github.ref == 'refs/heads/master'}}
         name: prebuild linux ${{ matrix.arch }} musl
         run: docker run -v $PWD:/home --platform linux/${{ matrix.arch }} --rm node:20-alpine bin/sh -c 'apk add bash && cd /home && bash -c "/home/script/ci/prebuild-alpine.sh" && rm -rf ffi node_modules'
 
-      - if: ${{ matrix.docker == true && matrix.alpine != true }}
+      - if: ${{ matrix.docker == true && matrix.alpine != true && github.ref == 'refs/heads/master' }}
         name: prebuild linux ${{ matrix.arch }}
         run: docker run -v $PWD:/home --platform linux/${{ matrix.arch }} --rm node:20 bin/bash -c 'cd /home && /home/script/ci/prebuild.sh && rm -rf ffi node_modules'
 
       - run: sudo chown -R $(id -u):$(id -g) prebuilds
-        if: ${{ matrix.docker == true }}
+        if: ${{ matrix.docker == true && github.ref == 'refs/heads/master' }}
 
       - run: ./script/ci/prebuild.sh
-        if: ${{ matrix.docker != true }}
+        if: ${{ matrix.docker != true && github.ref == 'refs/heads/master'}}
 
       - name: Upload prebuild for ${{ runner.os }}-${{ runner.arch }}
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
+        if: github.ref == 'refs/heads/master'
         with:
           path: prebuilds/*.tar.gz
           name: artifact-${{ matrix.docker == true && matrix.alpine == true && 'linux-musl' || matrix.docker == true && matrix.alpine == false && 'linux' || matrix.os }}-${{ matrix.arch }}
@@ -156,7 +159,12 @@ jobs:
           fetch-depth: 0
 
       - name: Download prebuilds
+        if: github.ref == 'refs/heads/master'
         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+      - run: FETCH_ASSETS=true REPO=pact-foundation/pact-js-core ./script/ci/check-release-libs.sh --fetch-assets
+        if: github.ref != 'refs/heads/master'
+        env:
+            GITHUB_TOKEN: ${{ github.token }}
 
       - name: Use Node.js ${{ env.NODE_VERSION }}
         uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
@@ -173,11 +181,11 @@ jobs:
 
       - if: ${{ matrix.docker == true && matrix.alpine == true && matrix.arch == 'amd64' && matrix.os == 'ubuntu-latest' }}
         name: test linux amd64 musl
-        run: docker run -v $PWD:/home --platform linux/${{ matrix.arch }} --rm node:${{ matrix.node-version }}-alpine bin/sh -c 'apk add bash curl gcompat file && cd /home && /home/script/ci/unpack-and-test.sh'
+        run: docker run -v $PWD:/home --platform linux/${{ matrix.arch }} --rm node:${{ matrix.node-version }}-alpine bin/sh -c 'apk add jq gettext-envsubst bash curl gcompat file && cd /home && /home/script/ci/unpack-and-test.sh'
 
       - if: ${{ matrix.docker == true && matrix.alpine == true && matrix.arch == 'arm64' && matrix.os == 'ubuntu-24.04-arm' }}
         name: test linux arm64 musl
-        run: docker run -v $PWD:/home --platform linux/${{ matrix.arch }} --rm node:${{ matrix.node-version }}-alpine bin/sh -c 'apk add bash curl file protoc protobuf-dev && cd /home && /home/script/ci/unpack-and-test.sh'
+        run: docker run -v $PWD:/home --platform linux/${{ matrix.arch }} --rm node:${{ matrix.node-version }}-alpine bin/sh -c 'apk add jq gettext-envsubst bash curl file protoc protobuf-dev && cd /home && /home/script/ci/unpack-and-test.sh'
 
   release_dry_run:
     runs-on: ubuntu-latest

.gitignore

@@ -72,3 +72,6 @@ reports
 tmp
 .tmp
 test/__testoutput__
+
+# platform-arch specific packages
+@pact-foundation/*

.npmignore

@@ -84,10 +84,17 @@ script
 binding.gyp
 native
 
-.cirrus
 .gitattributes
 DEVELOPER.md
 RELEASING.md
 test.js
 tsconfig.build.json
-tsconfig.json
+tsconfig.json
+
+# Standalone Binaries - Published as seperate packages
+@pact-foundation/
+
+# Cross packaging files
+Makefile
+package.json.tmpl
+prebuilds

DEVELOPER.md

@@ -4,13 +4,25 @@ Pact-Js-Core uses FFI bindings from the pact-reference project, which are prebui
 
 Do this and you should be 👌👌👌:
 
-```
+```sh
 bash script/ci/prebuild.sh
-npm ci --ignore-scripts
+supported_platforms=$(./script/ci/build-opt-dependencies.sh determine_platform) 
+./script/ci/build-opt-dependencies.sh build
+./script/ci/build-opt-dependencies.sh link
 npm run build
-npm test
+npm run test
 ```
 
+set supported platform to one of these values
+
+- `linux-x64-glibc`
+- `linux-arm64-glibc`
+- `linux-x64-musl`
+- `linux-arm64-musl`
+- `darwin-x64`
+- `darwin-arm64`
+- `windows-x64`
+
 _notes_ - 
 
 As a developer, you need to run `bash script/ci/prebuild.sh` to
@@ -32,61 +44,103 @@ Alternatively you can run the following, which will not create a prebuild, but i
 bash script/download-libs.sh
 npm ci
 npm run build
-npm test
+npm run test
 ```
 
-### Linux x86_64 Task
+## Creating Platform specific packages
 
-#### Pre Reqs
+We create cross-platform and architecture binaries which are published individually to NPM, and consumed in this project.
 
-1. x86_64 Machine
-   1. ARM64 Mac - If you have Rosetta (MacOS)
+### Download prebuilt binaries for all platforms
 
-### CI Locally
+```sh
+./script/ci/build_opt_dependencies.sh libs v15.2.1
+```
 
-1. Docker/Podman
-2. Act
+Tag is optional and defaults to latest
+
+This will run the following script, which will grab the latest prebuilt binaries from GitHub.
 
 ```sh
-act --container-architecture linux/amd64 -W .github/workflows/build-and-test.yml --artifact-server-path tmp
+FETCH_ASSETS=true ./script/ci/check-release-libs.sh --fetch-assets -t v15.2.1
 ```
 
-### MacOS ARM64 Task
+### Building all platform specific npm packages
 
-#### Pre Reqs
+```sh
+./script/ci/build_opt_dependencies.sh build
+```
+
+### Building individual platform specific npm package
 
-1. Arm64 Mac
-2. Cirrus-Cli
-3. Tart.run
+Supported platforms are
 
+- linux-x64-glibc
+- linux-arm64-glibc
+- linux-x64-musl
+- linux-arm64-musl
+- darwin-x64
+- darwin-arm64
+- windows-x64
+
+You can detect your platform with
 
 ```sh
-cirrus run --output github-actions macos_arm --artifacts-dir tmp
+./script/ci/build-opt-dependencies.sh determine_platform
 ```
 
-### Linux ARM64 Task
+You can build with one
 
-#### Pre Reqs
+```sh
+supported_platforms=$(./script/ci/build-opt-dependencies.sh determine_platform) ./script/ci/build-opt-dependencies.sh build
+```
 
-1. Arm64 Machine
+or all
 
-### CI Locally
+```sh
+./script/ci/build-opt-dependencies.sh build
+```
 
-1. Arm64 Machine
-2. Docker / Podman
-3. Cirrus-Cli
+### Linking arch specific package, for your local build
 
+Make link will try to link all available packages, for all available platforms, and will link any that apply
 
 ```sh
-cirrus run --output github-actions linux_arm --artifacts-dir tmp
+./script/ci/build-opt-dependencies.sh
 ```
 
-#### Publishing Assets
+You can scope it with `supported_platforms`
+
+```sh
+supported_platforms=$(./script/ci/build-opt-dependencies.sh determine_platform) ./script/ci/build-opt-dependencies.sh link
+```
 
-MacOS ARM64
+### Publishing packages
 
-`cirrus run --output github-actions macos_arm --artifacts-dir tmp --environment GITHUB_TOKEN=$GITHUB_TOKEN --environment CIRRUS_RELEASE=test --environment CIRRUS_REPO_FULL_NAME=pact-foundation/pact-js-core;`
+Dry run publishing optional packages, (default)
+
+```sh
+./script/ci/build-opt-dependencies.sh publish
+```
 
-Linux ARM64
+Publishing packages with `--dry-run` option removed.
 
-`cirrus run --output github-actions linux_arm --artifacts-dir tmp --environment GITHUB_TOKEN=$GITHUB_TOKEN --environment CIRRUS_RELEASE=test --environment CIRRUS_REPO_FULL_NAME=pact-foundation/pact-js-core;`
+```sh
+PUBLISH=true ./script/ci/build-opt-dependencies.sh publish
+```
+
+### Linux x86_64 Task
+
+#### Pre Reqs
+
+1. x86_64 Machine
+   1. ARM64 Mac - If you have Rosetta (MacOS)
+
+### CI Locally
+
+1. Docker/Podman
+2. Act
+
+```sh
+act --container-architecture linux/amd64 -W .github/workflows/build-and-test.yml --artifact-server-path tmp
+```

package-lock.json

@@ -44,6 +44,7 @@
   },
   "dependencies": {
     "check-types": "7.4.0",
+    "detect-libc": "^2.0.3",
     "node-gyp-build": "^4.6.0",
     "pino": "^8.7.0",
     "pino-pretty": "^9.1.1",
@@ -160,5 +161,14 @@
       }
     ]
   },
-  "snyk": true
+  "snyk": true,
+  "optionalDependencies": {
+    "@pact-foundation/pact-core-darwin-arm64": "15.2.1",
+    "@pact-foundation/pact-core-darwin-x64": "15.2.1",
+    "@pact-foundation/pact-core-linux-arm64-glibc": "15.2.1",
+    "@pact-foundation/pact-core-linux-arm64-musl": "15.2.1",
+    "@pact-foundation/pact-core-linux-x64-glibc": "15.2.1",
+    "@pact-foundation/pact-core-linux-x64-musl": "15.2.1",
+    "@pact-foundation/pact-core-windows-x64": "15.2.1"
+  }
 }

package.json

@@ -0,0 +1,20 @@
+{
+  "name": "${node_pkg}",
+  "version": "${pkg_version}",
+  "description": "Platform/arch specific libpact_ffi binaries for @pact-foundation/pact-core",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/pact-foundation/pact-js-core.git"
+  },
+  "scripts": {},
+  "author": "Yousaf Nabi <you@saf.dev>",
+  "homepage": "https://github.com/pact-foundation/pact-js-core#readme",
+  "os": [
+    "${node_pkg_os}"
+  ],
+  ${libc}
+  "cpu": [
+    "${node_arch}"
+  ]
+}

package.json.tmpl

@@ -19,11 +19,17 @@ fi
 
 node --version
 npm --version
+# Update main package.json optional dependencies versions, with those created earlier
+current_platform=$("$SCRIPT_DIR"/build-opt-dependencies.sh determine_platform)
+supported_platforms="$current_platform" "$SCRIPT_DIR"/build-opt-dependencies.sh update
+# update lockfile post building updated opt deps
+npm ci --ignore-scripts || npm i --ignore-scripts
+# Link os/arch specific npm package, for running os/arch system
 
-npm ci --ignore-scripts
-
+supported_platforms="$current_platform" "$SCRIPT_DIR"/build-opt-dependencies.sh link
+# ensure we test against the linked npm package, not the prebuild
+rm -rf prebuilds
 npm run format:check
 npm run lint
 npm run build
-npm run test
-ls -1
+npm run test