Move oci-import (and SBOM) shell code out of meta.jq into an explicit shell script

.test/meta-commands/out.sh

@@ -97,84 +97,14 @@ docker push 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1
 
 # </pull>
 # <build>
-export BASHBREW_CACHE="${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}"
-gitCache="$BASHBREW_CACHE/git"
-git init --bare "$gitCache"
-_git() { git -C "$gitCache" "$@"; }
-_git config gc.auto 0
-_commit() { _git rev-parse 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592^{commit}'; }
-if ! _commit &> /dev/null; then _git fetch 'https://github.com/docker-library/busybox.git' 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592:' || _git fetch 'refs/heads/dist-amd64:'; fi
-_commit
-mkdir temp
-_git archive --format=tar 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592:latest/glibc/amd64/' | tar -xvC temp
-jq -s '
-	if length != 1 then
-		error("unexpected '\''oci-layout'\'' document count: " + length)
-	else .[0] end
-	| if .imageLayoutVersion != "1.0.0" then
-		error("unsupported imageLayoutVersion: " + .imageLayoutVersion)
-	else . end
-' temp/oci-layout > /dev/null
-jq -s '
-	if length != 1 then
-		error("unexpected '\''index.json'\'' document count: " + length)
-	else .[0] end
-	| if .schemaVersion != 2 then
-		error("unsupported schemaVersion: " + .schemaVersion)
-	else . end
-	| if .manifests | length != 1 then
-		error("expected only one manifests entry, not " + (.manifests | length))
-	else . end
-	| .manifests[0] |= (
-		if .mediaType != "application/vnd.oci.image.manifest.v1+json" then
-			error("unsupported descriptor mediaType: " + .mediaType)
-		else . end
-		| if .size < 0 then
-			error("invalid descriptor size: " + .size)
-		else . end
-		| del(.annotations, .urls)
-		| .annotations = {"org.opencontainers.image.source":"https://github.com/docker-library/busybox.git","org.opencontainers.image.revision":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","org.opencontainers.image.created":"2024-02-28T00:44:18Z","org.opencontainers.image.version":"1.36.1","org.opencontainers.image.url":"https://hub.docker.com/_/busybox","com.docker.official-images.bashbrew.arch":"amd64","org.opencontainers.image.base.name":"scratch"}
-	)
-' temp/index.json > temp/index.json.new
-mv temp/index.json.new temp/index.json
+build='{"buildId":"191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","build":{"img":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","resolved":{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","size":610,"annotations":{"com.docker.official-images.bashbrew.arch":"amd64","org.opencontainers.image.base.name":"scratch","org.opencontainers.image.created":"2024-02-28T00:44:18Z","org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","org.opencontainers.image.revision":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","org.opencontainers.image.source":"https://github.com/docker-library/busybox.git","org.opencontainers.image.url":"https://hub.docker.com/_/busybox","org.opencontainers.image.version":"1.36.1-glibc"},"platform":{"architecture":"amd64","os":"linux"}}],"annotations":{"org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:70a227928672dffb7d24880bad1a705b527fab650f7503c191e48a209c4a0d10"}},"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","arch":"amd64","parents":{},"resolvedParents":{}},"source":{"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","reproducibleGitChecksum":"17e76ce3a5b47357c5724738db231ed2477c94d43df69ce34ae0871c99f7de78","entries":[{"GitRepo":"https://github.com/docker-library/busybox.git","GitFetch":"refs/heads/dist-amd64","GitCommit":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","Directory":"latest/glibc/amd64","File":"index.json","Builder":"oci-import","SOURCE_DATE_EPOCH":1709081058}],"arches":{"amd64":{"tags":["busybox:1.36.1","busybox:1.36","busybox:1","busybox:stable","busybox:latest","busybox:1.36.1-glibc","busybox:1.36-glibc","busybox:1-glibc","busybox:stable-glibc","busybox:glibc"],"archTags":["amd64/busybox:1.36.1","amd64/busybox:1.36","amd64/busybox:1","amd64/busybox:stable","amd64/busybox:latest","amd64/busybox:1.36.1-glibc","amd64/busybox:1.36-glibc","amd64/busybox:1-glibc","amd64/busybox:stable-glibc","amd64/busybox:glibc"],"froms":["scratch"],"lastStageFrom":"scratch","platformString":"linux/amd64","platform":{"architecture":"amd64","os":"linux"},"parents":{"scratch":{"sourceId":null,"pin":null}}}}}}'
+"$BASHBREW_META_SCRIPTS/helpers/oci-import.sh" <<<"$build" temp
 # SBOM
-originalImageManifest="$(jq -r '.manifests[0].digest' temp/index.json)"
-SOURCE_DATE_EPOCH=1709081058 \
-	docker buildx build --progress=plain \
-	--load=false \
-	--provenance=false \
-	--build-arg BUILDKIT_DOCKERFILE_CHECK=skip=all \
-	--sbom=generator="$BASHBREW_BUILDKIT_SBOM_GENERATOR" \
-	--output 'type=oci,tar=false,dest=sbom' \
-	--platform 'linux/amd64' \
-	--build-context "fake=oci-layout://$PWD/temp@$originalImageManifest" \
-	- <<<'FROM fake'
-sbomIndex="$(jq -r '.manifests[0].digest' sbom/index.json)"
-shell="$(jq -r --arg originalImageManifest "$originalImageManifest" '
-	first(
-		.manifests[]
-		| select(.annotations["vnd.docker.reference.type"] == "attestation-manifest")
-	) as $attDesc
-	| @sh "sbomManifest=\($attDesc.digest)",
-		@sh "sbomManifestDesc=\(
-			$attDesc
-			| .annotations["vnd.docker.reference.digest"] = $originalImageManifest
-			| tojson
-		)"
-' "sbom/blobs/${sbomIndex/://}")"
-eval "$shell"
-shell="$(jq -r '
-	"copyBlobs=( \([ .config.digest, .layers[].digest | @sh ] | join(" ")) )"
-' "sbom/blobs/${sbomManifest/://}")"
-eval "$shell"
-copyBlobs+=( "$sbomManifest" )
-for blob in "${copyBlobs[@]}"; do
-	cp "sbom/blobs/${blob/://}" "temp/blobs/${blob/://}"
-done
-jq -r --argjson sbomManifestDesc "$sbomManifestDesc" '.manifests += [ $sbomManifestDesc ]' temp/index.json > temp/index.json.new
-mv temp/index.json.new temp/index.json
+mv temp temp.orig
+"$BASHBREW_META_SCRIPTS/helpers/oci-sbom.sh" <<<"$build" temp.orig temp
+rm -rf temp.orig
 # </build>
 # <push>
-crane push --index temp 'oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f'
+crane push temp 'oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f'
 rm -rf temp
 # </push>

.test/oci-import/in.json

@@ -0,0 +1 @@
+../builds.json

.test/oci-import/out.sh

@@ -0,0 +1,2 @@
+build='{"buildId":"191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","build":{"img":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","resolved":{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","size":610,"annotations":{"com.docker.official-images.bashbrew.arch":"amd64","org.opencontainers.image.base.name":"scratch","org.opencontainers.image.created":"2024-02-28T00:44:18Z","org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","org.opencontainers.image.revision":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","org.opencontainers.image.source":"https://github.com/docker-library/busybox.git","org.opencontainers.image.url":"https://hub.docker.com/_/busybox","org.opencontainers.image.version":"1.36.1-glibc"},"platform":{"architecture":"amd64","os":"linux"}}],"annotations":{"org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:70a227928672dffb7d24880bad1a705b527fab650f7503c191e48a209c4a0d10"}},"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","arch":"amd64","parents":{},"resolvedParents":{}},"source":{"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","reproducibleGitChecksum":"17e76ce3a5b47357c5724738db231ed2477c94d43df69ce34ae0871c99f7de78","entries":[{"GitRepo":"https://github.com/docker-library/busybox.git","GitFetch":"refs/heads/dist-amd64","GitCommit":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","Directory":"latest/glibc/amd64","File":"index.json","Builder":"oci-import","SOURCE_DATE_EPOCH":1709081058}],"arches":{"amd64":{"tags":["busybox:1.36.1","busybox:1.36","busybox:1","busybox:stable","busybox:latest","busybox:1.36.1-glibc","busybox:1.36-glibc","busybox:1-glibc","busybox:stable-glibc","busybox:glibc"],"archTags":["amd64/busybox:1.36.1","amd64/busybox:1.36","amd64/busybox:1","amd64/busybox:stable","amd64/busybox:latest","amd64/busybox:1.36.1-glibc","amd64/busybox:1.36-glibc","amd64/busybox:1-glibc","amd64/busybox:stable-glibc","amd64/busybox:glibc"],"froms":["scratch"],"lastStageFrom":"scratch","platformString":"linux/amd64","platform":{"architecture":"amd64","os":"linux"},"parents":{"scratch":{"sourceId":null,"pin":null}}}}}}'
+"$BASHBREW_META_SCRIPTS/helpers/oci-import.sh" <<<"$build" temp

.test/oci-import/temp/blobs/sha256/166d2948c01a6ec70e44b073b0a4c56a3d7c4a4b8fd390d9ebfcb16a3ecf658e

@@ -0,0 +1,24 @@
+{
+	"schemaVersion": 2,
+	"mediaType": "application/vnd.oci.image.index.v1+json",
+	"manifests": [
+		{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json",
+			"digest": "sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0",
+			"size": 610,
+			"platform": {
+				"os": "linux",
+				"architecture": "amd64"
+			},
+			"annotations": {
+				"com.docker.official-images.bashbrew.arch": "amd64",
+				"org.opencontainers.image.base.name": "scratch",
+				"org.opencontainers.image.created": "2024-02-28T00:44:18Z",
+				"org.opencontainers.image.revision": "d0b7d566eb4f1fa9933984e6fc04ab11f08f4592",
+				"org.opencontainers.image.source": "https://github.com/docker-library/busybox.git",
+				"org.opencontainers.image.url": "https://hub.docker.com/_/busybox",
+				"org.opencontainers.image.version": "1.36.1"
+			}
+		}
+	]
+}

.test/oci-import/temp/blobs/sha256/4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0

@@ -0,0 +1 @@
+../../image-manifest.json

.test/oci-import/temp/blobs/sha256/7b2699543f22d5b8dc8d66a5873eb246767bca37232dee1e7a3b8c9956bceb0c

@@ -0,0 +1 @@
+../../rootfs.tar.gz

.test/oci-import/temp/blobs/sha256/ba5dc23f65d4cc4a4535bce55cf9e63b068eb02946e3422d3587e8ce803b6aab

@@ -0,0 +1 @@
+../../image-config.json

.test/oci-import/temp/image-config.json

@@ -0,0 +1,22 @@
+{
+	"config": {
+		"Cmd": [
+			"sh"
+		]
+	},
+	"created": "2023-05-18T22:34:17Z",
+	"history": [
+		{
+			"created": "2023-05-18T22:34:17Z",
+			"created_by": "BusyBox 1.36.1 (glibc), Debian 12"
+		}
+	],
+	"rootfs": {
+		"type": "layers",
+		"diff_ids": [
+			"sha256:95c4a60383f7b6eb6f7b8e153a07cd6e896de0476763bef39d0f6cf3400624bd"
+		]
+	},
+	"architecture": "amd64",
+	"os": "linux"
+}

.test/oci-import/temp/image-manifest.json

@@ -0,0 +1,20 @@
+{
+	"schemaVersion": 2,
+	"mediaType": "application/vnd.oci.image.manifest.v1+json",
+	"config": {
+		"mediaType": "application/vnd.oci.image.config.v1+json",
+		"digest": "sha256:ba5dc23f65d4cc4a4535bce55cf9e63b068eb02946e3422d3587e8ce803b6aab",
+		"size": 372
+	},
+	"layers": [
+		{
+			"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+			"digest": "sha256:7b2699543f22d5b8dc8d66a5873eb246767bca37232dee1e7a3b8c9956bceb0c",
+			"size": 2152262
+		}
+	],
+	"annotations": {
+		"org.opencontainers.image.url": "https://github.com/docker-library/busybox",
+		"org.opencontainers.image.version": "1.36.1-glibc"
+	}
+}

.test/oci-import/temp/index.json

@@ -0,0 +1,11 @@
+{
+	"schemaVersion": 2,
+	"mediaType": "application/vnd.oci.image.index.v1+json",
+	"manifests": [
+		{
+			"mediaType": "application/vnd.oci.image.index.v1+json",
+			"digest": "sha256:166d2948c01a6ec70e44b073b0a4c56a3d7c4a4b8fd390d9ebfcb16a3ecf658e",
+			"size": 838
+		}
+	]
+}

.test/oci-import/temp/oci-layout

@@ -0,0 +1 @@
+{"imageLayoutVersion":"1.0.0"}

.test/oci-import/test.jq

@@ -0,0 +1,8 @@
+include "meta";
+
+first(.[] | select(normalized_builder == "oci-import"))
+
+| build_command
+
+# TODO find a better way to stop the SBOM bits from being included here
+| sub("(?s)\n+# SBOM.*"; "")

.test/oci-import/test.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+dir="$(dirname "$BASH_SOURCE")"
+
+set -x
+
+cd "$dir"
+
+export BASHBREW_META_SCRIPTS=../..
+
+rm -rf temp
+source out.sh
+
+# TODO this should be part of "oci-import.sh"
+"$BASHBREW_META_SCRIPTS/helpers/oci-validate.sh" temp
+
+# make sure we don't commit the rootfs tarballs
+find temp -type f -size '+1k' -print -delete
+# TODO rely on .gitignore instead so that when the test finishes, we have a valid + complete OCI layout locally (that we can test push code against, for example)?

.test/oci-sort-platforms/test.jq

@@ -43,7 +43,7 @@ include "oci";
 	},
 
 	# buildkit attestations
-	# https://github.com/moby/buildkit/blob/5e0fe2793d529209ad52e811129f644d972ea094/docs/attestations/attestation-storage.md#attestation-manifest-descriptor
+	# https://github.com/moby/buildkit/blob/c6145c2423de48f891862ac02f9b2653864d3c9e/docs/attestations/attestation-storage.md#attestation-manifest-descriptor
 	{
 		architecture: "unknown",
 		os: "unknown",

.test/test.sh

@@ -279,3 +279,10 @@ fi
 
 # also run our "jq" tests (like generating example commands from the "builds.json" we just generated)
 "$dir/jq.sh"
+
+# TODO a new helper to run these by themselves?
+for t in "$dir/"*"/test.sh"; do
+	if [ -x "$t" ]; then
+		"$t"
+	fi
+done

Jenkinsfile.build

@@ -47,6 +47,8 @@ node('multiarch-' + env.BASHBREW_ARCH) { ansiColor('xterm') {
 		))
 	}
 
+	env.BASHBREW_META_SCRIPTS = env.WORKSPACE + '/meta/.scripts'
+
 	dir('.bin') {
 		deleteDir()
 
@@ -80,7 +82,7 @@ node('multiarch-' + env.BASHBREW_ARCH) { ansiColor('xterm') {
 			obj = sh(returnStdout: true, script: '''
 				[ -n "$BUILD_ID" ]
 				shell="$(
-					jq -L.scripts -r '
+					jq -L"$BASHBREW_META_SCRIPTS" -r '
 						include "meta";
 						.[env.BUILD_ID]
 						| select(needs_build and .build.arch == env.BASHBREW_ARCH) # sanity check

deploy.jq

@@ -57,10 +57,10 @@ def deploy_objects:
 					data: {
 						schemaVersion: 2,
 						mediaType: (
-							if $manifests[0]?.mediaType == "application/vnd.docker.distribution.manifest.v2+json" then
-								"application/vnd.docker.distribution.manifest.list.v2+json"
+							if $manifests[0].mediaType == media_type_dockerv2_image then
+								media_type_dockerv2_list
 							else
-								"application/vnd.oci.image.index.v1+json"
+								media_type_oci_index
 							end
 						),
 						manifests: (