From 5001faa690ce12374260334beb5f26dfa3f61613 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 3 Nov 2019 14:00:28 -0500 Subject: [PATCH] ci: stop assuming that azure will complete last Each of the 3 CI services now trigger a sr.ht build via the on_master_success script. The sr.ht build checks if all builds have completed successfully by trying to download the JSON file for the particular version. If all builds have completed successfully then this sr.ht job will update the download page. --- .builds/freebsd.yml | 3 ++ ci/azure/linux_script | 4 +- ci/azure/macos_script | 2 +- ci/azure/on_master_success | 10 +++++ ci/azure/pipelines.yml | 16 ++------ ci/azure/update_download_page | 47 ---------------------- ci/azure/windows_upload | 2 +- ci/drone/linux_script | 6 ++- ci/srht/freebsd_script | 8 +++- ci/srht/on_master_success | 33 +++++++++++++++ ci/srht/update_download_page | 75 +++++++++++++++++++++++++++++++++++ 11 files changed, 142 insertions(+), 64 deletions(-) create mode 100755 ci/azure/on_master_success delete mode 100755 ci/azure/update_download_page create mode 100755 ci/srht/on_master_success create mode 100755 ci/srht/update_download_page diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml index 3c0b91afff..16db3a9678 100644 --- a/.builds/freebsd.yml +++ b/.builds/freebsd.yml @@ -3,8 +3,11 @@ packages: - cmake - py27-s3cmd - wget + - curl + - jq secrets: - 6c60aaee-92e7-4e7d-812c-114817689b4d + - 2cd8c203-ba06-4504-be49-1e2b9b71efdd sources: - https://github.com/ziglang/zig tasks: diff --git a/ci/azure/linux_script b/ci/azure/linux_script index b5237dc226..f1c26d6a96 100755 --- a/ci/azure/linux_script +++ b/ci/azure/linux_script @@ -25,6 +25,7 @@ cd build cmake .. -DCMAKE_BUILD_TYPE=Release make -j2 install ./zig build test -Denable-qemu +VERSION="$(./zig version)" if [ "${BUILD_REASON}" != "PullRequest" ]; then ARTIFACTSDIR="$BUILDDIR/artifacts" @@ -44,7 +45,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE" - s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/linux-$VERSION.json" + s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-linux-$VERSION.json" # `set -x` causes these variables to be mangled. # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html @@ -52,4 +53,5 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL" echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM" echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE" + echo "##vso[task.setvariable variable=version;isOutput=true]$VERSION" fi diff --git a/ci/azure/macos_script b/ci/azure/macos_script index f41c036da1..64fee34de4 100755 --- a/ci/azure/macos_script +++ b/ci/azure/macos_script @@ -104,7 +104,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE" - s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/macos-$VERSION.json" + s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-macos-$VERSION.json" # `set -x` causes these variables to be mangled. # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html diff --git a/ci/azure/on_master_success b/ci/azure/on_master_success new file mode 100755 index 0000000000..5354fb0655 --- /dev/null +++ b/ci/azure/on_master_success @@ -0,0 +1,10 @@ +#!/bin/sh + +set -x +set -e + +sudo apt-get update -y +sudo apt-get install -y curl jq + +OAUTH_TOKEN="$(cat "$DOWNLOADSECUREFILE_SECUREFILEPATH")" +./ci/srht/on_master_success "$VERSION" "$OAUTH_TOKEN" diff --git a/ci/azure/pipelines.yml b/ci/azure/pipelines.yml index bcc2f11945..8ce9d8d01f 100644 --- a/ci/azure/pipelines.yml +++ b/ci/azure/pipelines.yml @@ -55,18 +55,10 @@ jobs: pool: vmImage: 'ubuntu-16.04' variables: - macos_tarball: $[ dependencies.BuildMacOS.outputs['main.tarball'] ] - macos_shasum: $[ dependencies.BuildMacOS.outputs['main.shasum'] ] - macos_bytesize: $[ dependencies.BuildMacOS.outputs['main.bytesize'] ] - linux_tarball: $[ dependencies.BuildLinux.outputs['main.tarball'] ] - linux_shasum: $[ dependencies.BuildLinux.outputs['main.shasum'] ] - linux_bytesize: $[ dependencies.BuildLinux.outputs['main.bytesize'] ] - windows_tarball: $[ dependencies.BuildWindows.outputs['main.tarball'] ] - windows_shasum: $[ dependencies.BuildWindows.outputs['main.shasum'] ] - windows_bytesize: $[ dependencies.BuildWindows.outputs['main.bytesize'] ] + version: $[ dependencies.BuildLinux.outputs['main.version'] ] steps: - task: DownloadSecureFile@1 inputs: - secureFile: s3cfg - - script: ci/azure/update_download_page - displayName: 'Update download page' + secureFile: oauth_token + - script: ci/azure/on_master_success + displayName: 'master branch success hook' diff --git a/ci/azure/update_download_page b/ci/azure/update_download_page deleted file mode 100755 index 92df66edfb..0000000000 --- a/ci/azure/update_download_page +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh - -set -x -set -e - -SRCDIR=$(pwd) -rm -rf .git -sudo apt-get update -y -sudo apt-get install -y s3cmd curl jq - -cd "$HOME" -wget "https://ziglang.org/builds/$LINUX_TARBALL" -tar xf $LINUX_TARBALL -ZIGDIR=$(basename -s .tar.xz $LINUX_TARBALL) -ZIG="$ZIGDIR/zig" -LANGREF="$ZIGDIR/langref.html" -VERSION=$($ZIG version) -SRCTARBALLDIR="zig-$VERSION" -export SRC_TARBALL="$SRCTARBALLDIR.tar.xz" -mv "$SRCDIR" "$SRCTARBALLDIR" -tar cfJ "$SRC_TARBALL" "$SRCTARBALLDIR" -export SRC_SHASUM=$(sha256sum $SRC_TARBALL | cut '-d ' -f1) -export SRC_BYTESIZE=$(wc -c < $SRC_TARBALL) - -# the aarch64 build has to be there too -AARCH64_LINUX_JSON=$(curl --fail "https://ziglang.org/builds/aarch64-linux-$VERSION.json" || exit 1) -export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)" -export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)" -export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)" - -# the freebsd build has to be there too -FREEBSD_JSON=$(curl --fail "https://ziglang.org/builds/freebsd-$VERSION.json" || exit 1) -export FREEBSD_TARBALL="$(echo "$FREEBSD_JSON" | jq .tarball -r)" -export FREEBSD_BYTESIZE="$(echo "$FREEBSD_JSON" | jq .size -r)" -export FREEBSD_SHASUM="$(echo "$FREEBSD_JSON" | jq .shasum -r)" - -git clone https://github.com/ziglang/www.ziglang.org --depth 1 -cd www.ziglang.org -export MASTER_DATE=$(date +%Y-%m-%d) -env -"../$ZIG" run update-download-page.zig - -mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg" -s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/ -s3cmd put -P "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate" -s3cmd put -P www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate" -s3cmd put -P www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate" diff --git a/ci/azure/windows_upload b/ci/azure/windows_upload index d5527fa13a..7ed1b7d1f7 100755 --- a/ci/azure/windows_upload +++ b/ci/azure/windows_upload @@ -30,7 +30,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE" - s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/windows-$VERSION.json" + s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-windows-$VERSION.json" # `set -x` causes these variables to be mangled. # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html diff --git a/ci/drone/linux_script b/ci/drone/linux_script index 987b3466ca..03bf0a3e56 100755 --- a/ci/drone/linux_script +++ b/ci/drone/linux_script @@ -8,7 +8,7 @@ BUILDDIR="$(pwd)" DISTDIR="$(pwd)/dist" apk update -apk add py3-pip xz perl-utils +apk add py3-pip xz perl-utils jq curl pip3 install s3cmd # Make the `zig version` number consistent. @@ -60,4 +60,8 @@ if [ -z "$DRONE_PULL_REQUEST" ]; then s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE" s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/$TRIPLEARCH-linux-$VERSION.json" + if [ "$GITBRANCH" = "master" ]; then + cd "$BUILDDIR" + ./ci/srht/on_master_success "$VERSION" "$SRHT_OAUTH_TOKEN" + fi fi diff --git a/ci/srht/freebsd_script b/ci/srht/freebsd_script index 55c06de33f..8e717e4ba6 100755 --- a/ci/srht/freebsd_script +++ b/ci/srht/freebsd_script @@ -85,5 +85,11 @@ if [ -f ~/.s3cfg ]; then echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE" - s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/freebsd-$VERSION.json" + s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-freebsd-$VERSION.json" + + if [ "$GITBRANCH" = "master" ]; then + OAUTH_TOKEN="$(cat ~/.oauth_token)" + cd "$ZIGDIR" + ./ci/srht/on_master_success "$VERSION" "$OAUTH_TOKEN" + fi fi diff --git a/ci/srht/on_master_success b/ci/srht/on_master_success new file mode 100755 index 0000000000..0de9169109 --- /dev/null +++ b/ci/srht/on_master_success @@ -0,0 +1,33 @@ +#!/bin/sh + +# This script must run on a lot of different platforms. +# It assumes that the following things are installed: +# * curl +# * jq + +set -x +set -e + +VERSION="$1" +OAUTH_TOKEN="$2" +YML_FILE="tmp.yml" + +echo "image: ubuntu/xenial" >> "$YML_FILE" +echo "packages:" >> "$YML_FILE" +echo " - s3cmd" >> "$YML_FILE" +echo " - curl" >> "$YML_FILE" +echo " - jq" >> "$YML_FILE" +echo "secrets:" >> "$YML_FILE" +echo " - 6c60aaee-92e7-4e7d-812c-114817689b4d" >> "$YML_FILE" +echo "sources:" >> "$YML_FILE" +echo " - https://github.com/ziglang/zig" >> "$YML_FILE" +echo "tasks:" >> "$YML_FILE" +echo " - build: cd zig && ./ci/srht/update_download_page $VERSION" >> "$YML_FILE" + +jq <$YML_FILE -sR '{ + "manifest": ., + }' | curl \ + -H Authorization:"token $OAUTH_TOKEN" \ + -H Content-Type:application/json \ + -X POST \ + -d @- "https://builds.sr.ht/api/jobs" diff --git a/ci/srht/update_download_page b/ci/srht/update_download_page new file mode 100755 index 0000000000..7e42d3a3bc --- /dev/null +++ b/ci/srht/update_download_page @@ -0,0 +1,75 @@ +#!/bin/sh + +set -x +set -e + +VERSION="$1" +SRCDIR="$(pwd)" +NATIVE_TARBALL="zig-linux-$(uname -m)-$VERSION.tar.xz" + +# Check for all the builds being completed. It's expected that this script is run several times +# before they are all complete. +AARCH64_LINUX_JSON_URL="https://ziglang.org/builds/aarch64-linux-$VERSION.json" +X86_64_LINUX_JSON_URL="https://ziglang.org/builds/x86_64-linux-$VERSION.json" +X86_64_WINDOWS_JSON_URL="https://ziglang.org/builds/x86_64-windows-$VERSION.json" +X86_64_MACOS_JSON_URL="https://ziglang.org/builds/x86_64-macos-$VERSION.json" +X86_64_FREEBSD_JSON_URL="https://ziglang.org/builds/x86_64-freebsd-$VERSION.json" + +# If any of these fail, it's not really this job failing; rather we have detected +# that this job will be called again later when other jobs have completed. +curl --fail -I "$AARCH64_LINUX_JSON_URL" || exit 0 +curl --fail -I "$X86_64_LINUX_JSON_URL" || exit 0 +curl --fail -I "$X86_64_WINDOWS_JSON_URL" || exit 0 +curl --fail -I "$X86_64_MACOS_JSON_URL" || exit 0 +curl --fail -I "$X86_64_FREEBSD_JSON_URL" || exit 0 + +rm -rf .git + +cd "$HOME" +wget "https://ziglang.org/builds/$NATIVE_TARBALL" +tar xf "$NATIVE_TARBALL" +ZIGDIR=$(basename -s .tar.xz $NATIVE_TARBALL) +ZIG="$ZIGDIR/zig" +LANGREF="$ZIGDIR/langref.html" +SRCTARBALLDIR="zig-$VERSION" +export SRC_TARBALL="$SRCTARBALLDIR.tar.xz" +mv "$SRCDIR" "$SRCTARBALLDIR" +tar cfJ "$SRC_TARBALL" "$SRCTARBALLDIR" +export SRC_SHASUM=$(sha256sum $SRC_TARBALL | cut '-d ' -f1) +export SRC_BYTESIZE=$(wc -c < $SRC_TARBALL) + +X86_64_WINDOWS_JSON=$(curl --fail "X86_64_WINDOWS_JSON_URL" || exit 1) +export X86_64_WINDOWS_TARBALL="$(echo "$X86_64_WINDOWS_JSON" | jq .tarball -r)" +export X86_64_WINDOWS_BYTESIZE="$(echo "$X86_64_WINDOWS_JSON" | jq .size -r)" +export X86_64_WINDOWS_SHASUM="$(echo "$X86_64_WINDOWS_JSON" | jq .shasum -r)" + +X86_64_MACOS_JSON=$(curl --fail "X86_64_MACOS_JSON_URL" || exit 1) +export X86_64_MACOS_TARBALL="$(echo "$X86_64_MACOS_JSON" | jq .tarball -r)" +export X86_64_MACOS_BYTESIZE="$(echo "$X86_64_MACOS_JSON" | jq .size -r)" +export X86_64_MACOS_SHASUM="$(echo "$X86_64_MACOS_JSON" | jq .shasum -r)" + +X86_64_LINUX_JSON=$(curl --fail "X86_64_LINUX_JSON_URL" || exit 1) +export X86_64_LINUX_TARBALL="$(echo "$X86_64_LINUX_JSON" | jq .tarball -r)" +export X86_64_LINUX_BYTESIZE="$(echo "$X86_64_LINUX_JSON" | jq .size -r)" +export X86_64_LINUX_SHASUM="$(echo "$X86_64_LINUX_JSON" | jq .shasum -r)" + +AARCH64_LINUX_JSON=$(curl --fail "AARCH64_LINUX_JSON_URL" || exit 1) +export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)" +export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)" +export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)" + +X86_64_FREEBSD_JSON=$(curl --fail "X86_64_FREEBSD_JSON_URL" || exit 1) +export X86_64_FREEBSD_TARBALL="$(echo "$X86_64_FREEBSD_JSON" | jq .tarball -r)" +export X86_64_FREEBSD_BYTESIZE="$(echo "$X86_64_FREEBSD_JSON" | jq .size -r)" +export X86_64_FREEBSD_SHASUM="$(echo "$X86_64_FREEBSD_JSON" | jq .shasum -r)" + +git clone https://github.com/ziglang/www.ziglang.org --depth 1 +cd www.ziglang.org +export MASTER_DATE="$(date +%Y-%m-%d)" +env +"../$ZIG" run update-download-page.zig + +s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/ +s3cmd put -P "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate" +s3cmd put -P www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate" +s3cmd put -P www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"