From 77031baf982ec5d1d6e3f16252298af2f0325d42 Mon Sep 17 00:00:00 2001
From: LemonBoy <thatlemon@gmail.com>
Date: Fri, 4 Dec 2020 13:04:10 +0100
Subject: [PATCH] ci: Prevent some more CI failures

Many times the CI pipeline fails in the early stages because the APT
repo for LLVM is being updated.
Add a retry mechanism for some more APT commands.
---
 ci/azure/linux_script | 40 +++++++++++++++++++++++-----------------
 1 file changed, 23 insertions(+), 17 deletions(-)

diff --git a/ci/azure/linux_script b/ci/azure/linux_script
index aba1555a7d..45321b1adc 100755
--- a/ci/azure/linux_script
+++ b/ci/azure/linux_script
@@ -3,32 +3,38 @@
 set -x
 set -e
 
+# This parameters we wait at most 2mins, it should be enough to sort out any
+# transient error.
+CMD_MAX_RETRY=12
+CMD_WAIT_TIME=10s
+
+# Execute the given command and, in case of failure, try to execute it again
+# after sleeping for CMD_WAIT_TIME.
+# We give up after retrying CMD_MAX_RETRY times.
+retry() {
+  for i in $(seq 1 "$CMD_MAX_RETRY"); do
+    "$@" && return
+    echo "command \"$@\" failed, retrying..."
+    sleep ${CMD_WAIT_TIME}
+  done
+
+  echo "command \"$@\" failed, giving up..."
+  exit 1
+}
+
 BUILDDIR="$(pwd)"
 
 sudo sh -c 'echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main" >> /etc/apt/sources.list'
-wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+retry 'wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -'
+retry sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
 
 sudo apt-get remove -y llvm-*
 sudo rm -rf /usr/local/*
 
-# Some APT mirrors can be flaky, retry the download instead of failing right
-# away.
-APT_MAX_RETRY=3
-
-for i in $(seq 1 "$APT_MAX_RETRY"); do
-  sudo apt-get update -q
-  sudo apt-get install -y \
+retry sudo apt-get update -q
+retry sudo apt-get install -y \
     libxml2-dev libclang-11-dev llvm-11 llvm-11-dev liblld-11-dev cmake s3cmd \
     gcc-7 g++-7 ninja-build tidy \
-    && break
-  if [ "$i" -eq "$APT_MAX_RETRY" ]; then
-    echo 'apt-get failed, giving up...'
-    exit 1
-  fi
-  echo 'apt-get failed, retrying...'
-  sleep 5s
-done
 
 QEMUBASE="qemu-linux-x86_64-5.1.0"
 wget https://ziglang.org/deps/$QEMUBASE.tar.xz