Skip to content

Commit 2b04d9e

Browse files
tianontianon
authored
Merge pull request #124 from infosiftr/helpers
Move `oci-import` (and SBOM) shell code out of `meta.jq` into an explicit shell script
2 parents d70af73 + 06ef8ce commit 2b04d9e

25 files changed

+813
-247
lines changed

‎.test/meta-commands/out.sh

+6-76
Original file line numberDiff line numberDiff line change
@@ -97,84 +97,14 @@ docker push 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1
9797

9898
# </pull>
9999
# <build>
100-
export BASHBREW_CACHE="${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}"
101-
gitCache="$BASHBREW_CACHE/git"
102-
git init --bare "$gitCache"
103-
_git() { git -C "$gitCache" "$@"; }
104-
_git config gc.auto 0
105-
_commit() { _git rev-parse 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592^{commit}'; }
106-
if ! _commit &> /dev/null; then _git fetch 'https://github.com/docker-library/busybox.git' 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592:' || _git fetch 'refs/heads/dist-amd64:'; fi
107-
_commit
108-
mkdir temp
109-
_git archive --format=tar 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592:latest/glibc/amd64/' | tar -xvC temp
110-
jq -s '
111-
if length != 1 then
112-
error("unexpected '\''oci-layout'\'' document count: " + length)
113-
else .[0] end
114-
| if .imageLayoutVersion != "1.0.0" then
115-
error("unsupported imageLayoutVersion: " + .imageLayoutVersion)
116-
else . end
117-
' temp/oci-layout > /dev/null
118-
jq -s '
119-
if length != 1 then
120-
error("unexpected '\''index.json'\'' document count: " + length)
121-
else .[0] end
122-
| if .schemaVersion != 2 then
123-
error("unsupported schemaVersion: " + .schemaVersion)
124-
else . end
125-
| if .manifests | length != 1 then
126-
error("expected only one manifests entry, not " + (.manifests | length))
127-
else . end
128-
| .manifests[0] |= (
129-
if .mediaType != "application/vnd.oci.image.manifest.v1+json" then
130-
error("unsupported descriptor mediaType: " + .mediaType)
131-
else . end
132-
| if .size < 0 then
133-
error("invalid descriptor size: " + .size)
134-
else . end
135-
| del(.annotations, .urls)
136-
| .annotations = {"org.opencontainers.image.source":"https://github.com/docker-library/busybox.git","org.opencontainers.image.revision":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","org.opencontainers.image.created":"2024-02-28T00:44:18Z","org.opencontainers.image.version":"1.36.1","org.opencontainers.image.url":"https://hub.docker.com/_/busybox","com.docker.official-images.bashbrew.arch":"amd64","org.opencontainers.image.base.name":"scratch"}
137-
)
138-
' temp/index.json > temp/index.json.new
139-
mv temp/index.json.new temp/index.json
100+
build='{"buildId":"191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","build":{"img":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","resolved":{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","size":610,"annotations":{"com.docker.official-images.bashbrew.arch":"amd64","org.opencontainers.image.base.name":"scratch","org.opencontainers.image.created":"2024-02-28T00:44:18Z","org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","org.opencontainers.image.revision":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","org.opencontainers.image.source":"https://github.com/docker-library/busybox.git","org.opencontainers.image.url":"https://hub.docker.com/_/busybox","org.opencontainers.image.version":"1.36.1-glibc"},"platform":{"architecture":"amd64","os":"linux"}}],"annotations":{"org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:70a227928672dffb7d24880bad1a705b527fab650f7503c191e48a209c4a0d10"}},"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","arch":"amd64","parents":{},"resolvedParents":{}},"source":{"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","reproducibleGitChecksum":"17e76ce3a5b47357c5724738db231ed2477c94d43df69ce34ae0871c99f7de78","entries":[{"GitRepo":"https://github.com/docker-library/busybox.git","GitFetch":"refs/heads/dist-amd64","GitCommit":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","Directory":"latest/glibc/amd64","File":"index.json","Builder":"oci-import","SOURCE_DATE_EPOCH":1709081058}],"arches":{"amd64":{"tags":["busybox:1.36.1","busybox:1.36","busybox:1","busybox:stable","busybox:latest","busybox:1.36.1-glibc","busybox:1.36-glibc","busybox:1-glibc","busybox:stable-glibc","busybox:glibc"],"archTags":["amd64/busybox:1.36.1","amd64/busybox:1.36","amd64/busybox:1","amd64/busybox:stable","amd64/busybox:latest","amd64/busybox:1.36.1-glibc","amd64/busybox:1.36-glibc","amd64/busybox:1-glibc","amd64/busybox:stable-glibc","amd64/busybox:glibc"],"froms":["scratch"],"lastStageFrom":"scratch","platformString":"linux/amd64","platform":{"architecture":"amd64","os":"linux"},"parents":{"scratch":{"sourceId":null,"pin":null}}}}}}'
101+
"$BASHBREW_META_SCRIPTS/helpers/oci-import.sh" <<<"$build" temp
140102
# SBOM
141-
originalImageManifest="$(jq -r '.manifests[0].digest' temp/index.json)"
142-
SOURCE_DATE_EPOCH=1709081058 \
143-
docker buildx build --progress=plain \
144-
--load=false \
145-
--provenance=false \
146-
--build-arg BUILDKIT_DOCKERFILE_CHECK=skip=all \
147-
--sbom=generator="$BASHBREW_BUILDKIT_SBOM_GENERATOR" \
148-
--output 'type=oci,tar=false,dest=sbom' \
149-
--platform 'linux/amd64' \
150-
--build-context "fake=oci-layout://$PWD/temp@$originalImageManifest" \
151-
- <<<'FROM fake'
152-
sbomIndex="$(jq -r '.manifests[0].digest' sbom/index.json)"
153-
shell="$(jq -r --arg originalImageManifest "$originalImageManifest" '
154-
first(
155-
.manifests[]
156-
| select(.annotations["vnd.docker.reference.type"] == "attestation-manifest")
157-
) as $attDesc
158-
| @sh "sbomManifest=\($attDesc.digest)",
159-
@sh "sbomManifestDesc=\(
160-
$attDesc
161-
| .annotations["vnd.docker.reference.digest"] = $originalImageManifest
162-
| tojson
163-
)"
164-
' "sbom/blobs/${sbomIndex/://}")"
165-
eval "$shell"
166-
shell="$(jq -r '
167-
"copyBlobs=( \([ .config.digest, .layers[].digest | @sh ] | join(" ")) )"
168-
' "sbom/blobs/${sbomManifest/://}")"
169-
eval "$shell"
170-
copyBlobs+=( "$sbomManifest" )
171-
for blob in "${copyBlobs[@]}"; do
172-
cp "sbom/blobs/${blob/://}" "temp/blobs/${blob/://}"
173-
done
174-
jq -r --argjson sbomManifestDesc "$sbomManifestDesc" '.manifests += [ $sbomManifestDesc ]' temp/index.json > temp/index.json.new
175-
mv temp/index.json.new temp/index.json
103+
mv temp temp.orig
104+
"$BASHBREW_META_SCRIPTS/helpers/oci-sbom.sh" <<<"$build" temp.orig temp
105+
rm -rf temp.orig
176106
# </build>
177107
# <push>
178-
crane push --index temp 'oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f'
108+
crane push temp 'oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f'
179109
rm -rf temp
180110
# </push>

‎.test/oci-import/in.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../builds.json

‎.test/oci-import/out.sh

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
build='{"buildId":"191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","build":{"img":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","resolved":{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","size":610,"annotations":{"com.docker.official-images.bashbrew.arch":"amd64","org.opencontainers.image.base.name":"scratch","org.opencontainers.image.created":"2024-02-28T00:44:18Z","org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","org.opencontainers.image.revision":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","org.opencontainers.image.source":"https://github.com/docker-library/busybox.git","org.opencontainers.image.url":"https://hub.docker.com/_/busybox","org.opencontainers.image.version":"1.36.1-glibc"},"platform":{"architecture":"amd64","os":"linux"}}],"annotations":{"org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:70a227928672dffb7d24880bad1a705b527fab650f7503c191e48a209c4a0d10"}},"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","arch":"amd64","parents":{},"resolvedParents":{}},"source":{"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","reproducibleGitChecksum":"17e76ce3a5b47357c5724738db231ed2477c94d43df69ce34ae0871c99f7de78","entries":[{"GitRepo":"https://github.com/docker-library/busybox.git","GitFetch":"refs/heads/dist-amd64","GitCommit":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","Directory":"latest/glibc/amd64","File":"index.json","Builder":"oci-import","SOURCE_DATE_EPOCH":1709081058}],"arches":{"amd64":{"tags":["busybox:1.36.1","busybox:1.36","busybox:1","busybox:stable","busybox:latest","busybox:1.36.1-glibc","busybox:1.36-glibc","busybox:1-glibc","busybox:stable-glibc","busybox:glibc"],"archTags":["amd64/busybox:1.36.1","amd64/busybox:1.36","amd64/busybox:1","amd64/busybox:stable","amd64/busybox:latest","amd64/busybox:1.36.1-glibc","amd64/busybox:1.36-glibc","amd64/busybox:1-glibc","amd64/busybox:stable-glibc","amd64/busybox:glibc"],"froms":["scratch"],"lastStageFrom":"scratch","platformString":"linux/amd64","platform":{"architecture":"amd64","os":"linux"},"parents":{"scratch":{"sourceId":null,"pin":null}}}}}}'
2+
"$BASHBREW_META_SCRIPTS/helpers/oci-import.sh" <<<"$build" temp

‎.test/oci-import/temp/blobs/sha256/166d2948c01a6ec70e44b073b0a4c56a3d7c4a4b8fd390d9ebfcb16a3ecf658e

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"schemaVersion": 2,
3+
"mediaType": "application/vnd.oci.image.index.v1+json",
4+
"manifests": [
5+
{
6+
"mediaType": "application/vnd.oci.image.manifest.v1+json",
7+
"digest": "sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0",
8+
"size": 610,
9+
"platform": {
10+
"os": "linux",
11+
"architecture": "amd64"
12+
},
13+
"annotations": {
14+
"com.docker.official-images.bashbrew.arch": "amd64",
15+
"org.opencontainers.image.base.name": "scratch",
16+
"org.opencontainers.image.created": "2024-02-28T00:44:18Z",
17+
"org.opencontainers.image.revision": "d0b7d566eb4f1fa9933984e6fc04ab11f08f4592",
18+
"org.opencontainers.image.source": "https://github.com/docker-library/busybox.git",
19+
"org.opencontainers.image.url": "https://hub.docker.com/_/busybox",
20+
"org.opencontainers.image.version": "1.36.1"
21+
}
22+
}
23+
]
24+
}

‎.test/oci-import/temp/blobs/sha256/4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../image-manifest.json

‎.test/oci-import/temp/blobs/sha256/7b2699543f22d5b8dc8d66a5873eb246767bca37232dee1e7a3b8c9956bceb0c

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../rootfs.tar.gz

‎.test/oci-import/temp/blobs/sha256/ba5dc23f65d4cc4a4535bce55cf9e63b068eb02946e3422d3587e8ce803b6aab

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../image-config.json

‎.test/oci-import/temp/image-config.json

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"config": {
3+
"Cmd": [
4+
"sh"
5+
]
6+
},
7+
"created": "2023-05-18T22:34:17Z",
8+
"history": [
9+
{
10+
"created": "2023-05-18T22:34:17Z",
11+
"created_by": "BusyBox 1.36.1 (glibc), Debian 12"
12+
}
13+
],
14+
"rootfs": {
15+
"type": "layers",
16+
"diff_ids": [
17+
"sha256:95c4a60383f7b6eb6f7b8e153a07cd6e896de0476763bef39d0f6cf3400624bd"
18+
]
19+
},
20+
"architecture": "amd64",
21+
"os": "linux"
22+
}

‎.test/oci-import/temp/image-manifest.json

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"schemaVersion": 2,
3+
"mediaType": "application/vnd.oci.image.manifest.v1+json",
4+
"config": {
5+
"mediaType": "application/vnd.oci.image.config.v1+json",
6+
"digest": "sha256:ba5dc23f65d4cc4a4535bce55cf9e63b068eb02946e3422d3587e8ce803b6aab",
7+
"size": 372
8+
},
9+
"layers": [
10+
{
11+
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
12+
"digest": "sha256:7b2699543f22d5b8dc8d66a5873eb246767bca37232dee1e7a3b8c9956bceb0c",
13+
"size": 2152262
14+
}
15+
],
16+
"annotations": {
17+
"org.opencontainers.image.url": "https://github.com/docker-library/busybox",
18+
"org.opencontainers.image.version": "1.36.1-glibc"
19+
}
20+
}

‎.test/oci-import/temp/index.json

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"schemaVersion": 2,
3+
"mediaType": "application/vnd.oci.image.index.v1+json",
4+
"manifests": [
5+
{
6+
"mediaType": "application/vnd.oci.image.index.v1+json",
7+
"digest": "sha256:166d2948c01a6ec70e44b073b0a4c56a3d7c4a4b8fd390d9ebfcb16a3ecf658e",
8+
"size": 838
9+
}
10+
]
11+
}

‎.test/oci-import/temp/oci-layout

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"imageLayoutVersion":"1.0.0"}

‎.test/oci-import/test.jq

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
include "meta";
2+
3+
first(.[] | select(normalized_builder == "oci-import"))
4+
5+
| build_command
6+
7+
# TODO find a better way to stop the SBOM bits from being included here
8+
| sub("(?s)\n+# SBOM.*"; "")

‎.test/oci-import/test.sh

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
set -Eeuo pipefail
3+
4+
dir="$(dirname "$BASH_SOURCE")"
5+
6+
set -x
7+
8+
cd "$dir"
9+
10+
export BASHBREW_META_SCRIPTS=../..
11+
12+
rm -rf temp
13+
source out.sh
14+
15+
# TODO this should be part of "oci-import.sh"
16+
"$BASHBREW_META_SCRIPTS/helpers/oci-validate.sh" temp
17+
18+
# make sure we don't commit the rootfs tarballs
19+
find temp -type f -size '+1k' -print -delete
20+
# TODO rely on .gitignore instead so that when the test finishes, we have a valid + complete OCI layout locally (that we can test push code against, for example)?

‎.test/oci-sort-platforms/test.jq

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ include "oci";
4343
},
4444

4545
# buildkit attestations
46-
# https://github.com/moby/buildkit/blob/5e0fe2793d529209ad52e811129f644d972ea094/docs/attestations/attestation-storage.md#attestation-manifest-descriptor
46+
# https://github.com/moby/buildkit/blob/c6145c2423de48f891862ac02f9b2653864d3c9e/docs/attestations/attestation-storage.md#attestation-manifest-descriptor
4747
{
4848
architecture: "unknown",
4949
os: "unknown",

‎.test/test.sh

+7
Original file line numberDiff line numberDiff line change
@@ -279,3 +279,10 @@ fi
279279

280280
# also run our "jq" tests (like generating example commands from the "builds.json" we just generated)
281281
"$dir/jq.sh"
282+
283+
# TODO a new helper to run these by themselves?
284+
for t in "$dir/"*"/test.sh"; do
285+
if [ -x "$t" ]; then
286+
"$t"
287+
fi
288+
done

‎Jenkinsfile.build

+3-1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ node('multiarch-' + env.BASHBREW_ARCH) { ansiColor('xterm') {
4747
))
4848
}
4949

50+
env.BASHBREW_META_SCRIPTS = env.WORKSPACE + '/meta/.scripts'
51+
5052
dir('.bin') {
5153
deleteDir()
5254

@@ -80,7 +82,7 @@ node('multiarch-' + env.BASHBREW_ARCH) { ansiColor('xterm') {
8082
obj = sh(returnStdout: true, script: '''
8183
[ -n "$BUILD_ID" ]
8284
shell="$(
83-
jq -L.scripts -r '
85+
jq -L"$BASHBREW_META_SCRIPTS" -r '
8486
include "meta";
8587
.[env.BUILD_ID]
8688
| select(needs_build and .build.arch == env.BASHBREW_ARCH) # sanity check

‎deploy.jq

+3-3
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,10 @@ def deploy_objects:
5757
data: {
5858
schemaVersion: 2,
5959
mediaType: (
60-
if $manifests[0]?.mediaType == "application/vnd.docker.distribution.manifest.v2+json" then
61-
"application/vnd.docker.distribution.manifest.list.v2+json"
60+
if $manifests[0].mediaType == media_type_dockerv2_image then
61+
media_type_dockerv2_list
6262
else
63-
"application/vnd.oci.image.index.v1+json"
63+
media_type_oci_index
6464
end
6565
),
6666
manifests: (

0 commit comments

Comments
 (0)