From 44b5fdf3266f11607313bc9990a876b5a7f9e174 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 9 Feb 2022 18:26:56 -0700 Subject: [PATCH] Revert "ci: azure: split build-and-test step" This reverts commit 846eb701821a3f2af514bbad770478e3276b2d89. This did not properly translate the upload portion of the CI script to powershell which broke our CI pipeline. --- CMakeLists.txt | 2 +- ci/azure/pipelines.yml | 137 ++++--------------------------- ci/azure/windows_msvc_install | 16 ++++ ci/azure/windows_msvc_script.bat | 39 +++++++++ ci/azure/windows_upload | 46 +++++++++++ 5 files changed, 120 insertions(+), 120 deletions(-) create mode 100644 ci/azure/windows_msvc_install create mode 100644 ci/azure/windows_msvc_script.bat create mode 100755 ci/azure/windows_upload diff --git a/CMakeLists.txt b/CMakeLists.txt index 533e03383f..8fd1960518 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,7 @@ if("${ZIG_VERSION}" STREQUAL "") endif() endif() endif() -message(STATUS "Configuring zig version ${ZIG_VERSION}") +message("Configuring zig version ${ZIG_VERSION}") set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)") set(ZIG_STATIC_LLVM off CACHE BOOL "Prefer linking against static LLVM libraries") diff --git a/ci/azure/pipelines.yml b/ci/azure/pipelines.yml index 6bb54e0402..4524ee9fb1 100644 --- a/ci/azure/pipelines.yml +++ b/ci/azure/pipelines.yml @@ -10,7 +10,6 @@ jobs: - script: ci/azure/macos_script name: main displayName: 'Build and test' - - job: BuildMacOS_arm64 pool: vmImage: 'macOS-10.15' @@ -22,130 +21,30 @@ jobs: - script: ci/azure/macos_arm64_script name: main displayName: 'Build' - - job: BuildWindows - timeoutInMinutes: 360 pool: vmImage: 'windows-2019' - variables: - LLVM_CLANG_LLD_URL: 'https://ziglang.org/deps/llvm+clang+lld-13.0.0-x86_64-windows-msvc-release-mt.tar.xz' - LLVM_CLANG_LLD_DIR: 'llvm+clang+lld-13.0.0-x86_64-windows-msvc-release-mt' + timeoutInMinutes: 360 steps: - - pwsh: | - (New-Object Net.WebClient).DownloadFile("$(LLVM_CLANG_LLD_URL)", "${LLVM_CLANG_LLD_DIR}.tar.xz") - & 'C:\Program Files\7-Zip\7z.exe' x "${LLVM_CLANG_LLD_DIR}.tar.xz" - & 'C:\Program Files\7-Zip\7z.exe' x "${LLVM_CLANG_LLD_DIR}.tar" - name: install - displayName: 'Install LLVM/CLANG/LLD' - - - pwsh: | - Set-Variable -Name ZIGBUILDDIR -Value "$(Get-Location)\build" - Set-Variable -Name ZIGINSTALLDIR -Value "$ZIGBUILDDIR\dist" - Set-Variable -Name ZIGPREFIXPATH -Value "$(Get-Location)\$(LLVM_CLANG_LLD_DIR)" - - # Make the `zig version` number consistent. - # This will affect the cmake command below. - git config core.abbrev 9 - git fetch --tags - - mkdir $ZIGBUILDDIR - cd $ZIGBUILDDIR - & "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - cmake .. ` - -Thost=x64 ` - -G "Visual Studio 16 2019" ` - -A x64 ` - -DCMAKE_INSTALL_PREFIX="$ZIGINSTALLDIR" ` - -DCMAKE_PREFIX_PATH="$ZIGPREFIXPATH" ` - -DCMAKE_BUILD_TYPE=Release ` - -DZIG_OMIT_STAGE2=ON 2> out-null - & "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\MSBuild.exe" ` - /maxcpucount /p:Configuration=Release INSTALL.vcxproj - name: build - displayName: 'Build' - - - pwsh: | - Set-Variable -Name ZIGINSTALLDIR -Value "$(Get-Location)\build\dist" - - # Sadly, stage2 is omitted from this build to save memory on the CI server. Once self-hosted is - # built with itself and does not gobble as much memory, we can enable these tests. - #& "$ZIGINSTALLDIR\bin\zig.exe" test "..\test\behavior.zig" -fno-stage1 -fLLVM -I "..\test" 2>&1 - - & "$ZIGINSTALLDIR\bin\zig.exe" build test-toolchain -Dskip-non-native -Dskip-stage2-tests 2>&1 - & "$ZIGINSTALLDIR\bin\zig.exe" build test-std -Dskip-non-native 2>&1 - name: test - displayName: 'Test' - - - pwsh: | - Set-Variable -Name ZIGINSTALLDIR -Value "$(Get-Location)\build\dist" - - & "$ZIGINSTALLDIR\bin\zig.exe" build docs - timeoutInMinutes: 60 - name: doc - displayName: 'Documentation' - + - powershell: | + (New-Object Net.WebClient).DownloadFile("https://github.com/msys2/msys2-installer/releases/download/2022-01-28/msys2-base-x86_64-20220128.sfx.exe", "sfx.exe") + .\sfx.exe -y -o\ + displayName: Download/Extract/Install MSYS2 + - script: | + @REM install updated filesystem package first without dependency checking + @REM because of: https://github.com/msys2/MSYS2-packages/issues/2021 + %CD:~0,2%\msys64\usr\bin\bash -lc "pacman --noconfirm -Sydd filesystem" + displayName: Workaround filesystem dash MSYS2 dependency issue + - script: | + %CD:~0,2%\msys64\usr\bin\bash -lc "pacman --noconfirm -Syuu" + %CD:~0,2%\msys64\usr\bin\bash -lc "pacman --noconfirm -Syuu" + displayName: Update MSYS2 - task: DownloadSecureFile@1 inputs: - name: aws_credentials - secureFile: aws_credentials - - - pwsh: | - Set-Variable -Name ZIGBUILDDIR -Value "$(Get-Location)\build" - $Env:AWS_SHARED_CREDENTIALS_FILE = "$Env:DOWNLOADSECUREFILE_SECUREFILEPATH" - - # Workaround Azure networking issue - # https://github.com/aws/aws-cli/issues/5749 - $Env:AWS_EC2_METADATA_DISABLED = "true" - $Env:AWS_REGION = "us-west-2" - - cd "$ZIGBUILDDIR" - mv ../LICENSE dist/ - mv ../zig-cache/langref.html dist/ - mv dist/bin/zig.exe dist/ - rmdir dist/bin - - # Remove the unnecessary zig dir in $prefix/lib/zig/std/std.zig - mv dist/lib/zig dist/lib2 - rmdir dist/lib - mv dist/lib2 dist/lib - - Set-Variable -Name VERSION -Value $(./dist/zig.exe version) - Set-Variable -Name DIRNAME -Value "zig-windows-x86_64-$VERSION" - Set-Variable -Name TARBALL -Value "$DIRNAME.zip" - mv dist "$DIRNAME" - 7z a "$TARBALL" "$DIRNAME" - - aws s3 cp ` - "$TARBALL" ` - s3://ziglang.org/builds/ ` - --cache-control 'public, max-age=31536000, immutable' - - Set-Variable -Name SHASUM -Value (Get-FileHash "$TARBALL" -Algorithm SHA256 | select-object -ExpandProperty Hash) - Set-Variable -Name BYTESIZE -Value (Get-Item "$TARBALL").length - - Set-Variable -Name JSONFILE -Value "windows-${Env:BUILD_SOURCEBRANCHNAME}.json" - echo $null > $JSONFILE - echo ('{"tarball": "' + $TARBALL + '",') >> $JSONFILE - echo ('"shasum": "' + $SHASUM + '",') >> $JSONFILE - echo ('"size": ' + $BYTESIZE + '}' ) >> $JSONFILE - - aws s3 cp ` - "$JSONFILE" ` - s3://ziglang.org/builds/ ` - --cache-control 'max-age=0, must-revalidate' - - aws s3 cp ` - "$JSONFILE" ` - "s3://ziglang.org/builds/x86_64-windows-${VERSION}.json" - - echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL" - echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM" - echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE" - - name: upload - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - displayName: 'Upload' - + secureFile: s3cfg + - script: ci/azure/windows_msvc_script.bat + name: main + displayName: 'Build and test' - job: OnMasterSuccess dependsOn: - BuildMacOS diff --git a/ci/azure/windows_msvc_install b/ci/azure/windows_msvc_install new file mode 100644 index 0000000000..2df445fe12 --- /dev/null +++ b/ci/azure/windows_msvc_install @@ -0,0 +1,16 @@ +#!/bin/sh + +set -x +set -e + +pacman -Suy --needed --noconfirm +pacman -S --needed --noconfirm wget p7zip python3-pip tar xz + +TARBALL="llvm+clang+lld-13.0.0-x86_64-windows-msvc-release-mt.tar.xz" + +pip install s3cmd +wget -nv "https://ziglang.org/deps/$TARBALL" +# If the first extraction fails, re-try it once; this can happen if the tarball +# contains symlinks that are in the table of contents before the files that +# they point to. +tar -xf $TARBALL || tar --overwrite -xf $TARBALL diff --git a/ci/azure/windows_msvc_script.bat b/ci/azure/windows_msvc_script.bat new file mode 100644 index 0000000000..c61c88093c --- /dev/null +++ b/ci/azure/windows_msvc_script.bat @@ -0,0 +1,39 @@ +@echo on +SET "SRCROOT=%cd%" +SET "PREVPATH=%PATH%" +SET "PREVMSYSTEM=%MSYSTEM%" + +set "PATH=%CD:~0,2%\msys64\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem" +SET "MSYSTEM=MINGW64" +bash -lc "cd ${SRCROOT} && ci/azure/windows_msvc_install" || exit /b +SET "PATH=%PREVPATH%" +SET "MSYSTEM=%PREVMSYSTEM%" + +SET "ZIGBUILDDIR=%SRCROOT%\build" +SET "ZIGINSTALLDIR=%ZIGBUILDDIR%\dist" +SET "ZIGPREFIXPATH=%SRCROOT%\llvm+clang+lld-13.0.0-x86_64-windows-msvc-release-mt" + +call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + +REM Make the `zig version` number consistent. +REM This will affect the cmake command below. +git.exe config core.abbrev 9 +git.exe fetch --unshallow +git.exe fetch --tags + +mkdir %ZIGBUILDDIR% +cd %ZIGBUILDDIR% +cmake.exe .. -Thost=x64 -G"Visual Studio 16 2019" -A x64 "-DCMAKE_INSTALL_PREFIX=%ZIGINSTALLDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release -DZIG_OMIT_STAGE2=ON || exit /b +msbuild /maxcpucount /p:Configuration=Release INSTALL.vcxproj || exit /b + +REM Sadly, stage2 is omitted from this build to save memory on the CI server. Once self-hosted is +REM built with itself and does not gobble as much memory, we can enable these tests. +REM "%ZIGINSTALLDIR%\bin\zig.exe" test "..\test\behavior.zig" -fno-stage1 -fLLVM -I "..\test" || exit /b + +"%ZIGINSTALLDIR%\bin\zig.exe" build test-toolchain -Dskip-non-native -Dskip-stage2-tests || exit /b +"%ZIGINSTALLDIR%\bin\zig.exe" build test-std -Dskip-non-native || exit /b +"%ZIGINSTALLDIR%\bin\zig.exe" build docs || exit /b + +set "PATH=%CD:~0,2%\msys64\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem" +SET "MSYSTEM=MINGW64" +bash -lc "cd ${SRCROOT} && ci/azure/windows_upload" || exit /b diff --git a/ci/azure/windows_upload b/ci/azure/windows_upload new file mode 100755 index 0000000000..9c5e07e5f9 --- /dev/null +++ b/ci/azure/windows_upload @@ -0,0 +1,46 @@ +#!/bin/sh + +set -x +set -e + +if [ "${BUILD_REASON}" != "PullRequest" ]; then + cd "$ZIGBUILDDIR" + + mv ../LICENSE dist/ + mv ../zig-cache/langref.html dist/ + mv dist/bin/zig.exe dist/ + rmdir dist/bin + + # Remove the unnecessary zig dir in $prefix/lib/zig/std/std.zig + mv dist/lib/zig dist/lib2 + rmdir dist/lib + mv dist/lib2 dist/lib + + VERSION=$(dist/zig.exe version) + DIRNAME="zig-windows-x86_64-$VERSION" + TARBALL="$DIRNAME.zip" + mv dist "$DIRNAME" + 7z a "$TARBALL" "$DIRNAME" + + # mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg" + s3cmd -c "$DOWNLOADSECUREFILE_SECUREFILEPATH" put -P --add-header="cache-control: public, max-age=31536000, immutable" "$TARBALL" s3://ziglang.org/builds/ + + SHASUM=$(sha256sum $TARBALL | cut '-d ' -f1) + BYTESIZE=$(wc -c < $TARBALL) + + JSONFILE="windows-$GITBRANCH.json" + touch $JSONFILE + echo "{\"tarball\": \"$TARBALL\"," >>$JSONFILE + echo "\"shasum\": \"$SHASUM\"," >>$JSONFILE + echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE + + s3cmd -c "$DOWNLOADSECUREFILE_SECUREFILEPATH" put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE" + s3cmd -c "$DOWNLOADSECUREFILE_SECUREFILEPATH" put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-windows-$VERSION.json" + + # `set -x` causes these variables to be mangled. + # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html + set +x + echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL" + echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM" + echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE" +fi