deploy: add a timeout in the polling waiting script

2 files changed, 36 insertions, 9 deletions

diff --git a/files/clever-wait-deploy.sh b/files/clever-wait-deploy.sh
index 98c5c37..a95c96f 100755
--- a/files/clever-wait-deploy.sh
+++ b/files/clever-wait-deploy.sh
@@ -4,18 +4,35 @@ set -ueo pipefail
 
 VERBOSE=${VERBOSE:-}
 lastCleverActivity=""
+timeout=2300 # 100 seconds less than the Ansible tasks' timeout
+startTime=$(date +%s)
 
 function cleverActivity {
   lastCleverActivity=$(clever activity)
 }
 
 function ensure {
-  if [ "$?" == "1" ]
-  then
-    echo "✗ Deployment failed!"
-  fi
+  local lastReturnCode="$?"
+
   VERBOSE=true
   verbose
+
+  if isNotTimeout
+  then
+    if [ "$lastReturnCode" == "0" ]
+    then
+      echo "✓ Deployment done."
+    else
+      echo "✗ Deployment failed!"
+    fi
+  else
+    echo "⁈ Deployment timeout... Please check clever logs"
+    exit 1
+  fi
+}
+
+function isNotTimeout {
+  [ $(($(date +%s) - startTime)) -lt $timeout ]
 }
 
 trap ensure EXIT ERR
@@ -52,7 +69,7 @@ function check {
   local samplingTime=5
 
   echo "️▫ Waiting for deployment to start..."
-  while inactive "$commit"
+  while inactive "$commit" && isNotTimeout
   do
     verbose
     sleep $samplingTime
@@ -60,14 +77,13 @@ function check {
 
   # Wait for completion
   echo "▪ Deployment in progress..."
-  while deploying "$commit"
+  while deploying "$commit" && isNotTimeout
   do
     verbose
     sleep $samplingTime
   done
 
   deployed "$commit"
-  echo "✓ Deployment done."
 }
 
 function getHeadRev {
diff --git a/tasks/deploy.yml b/tasks/deploy.yml
index df02464..d5e5a45 100644
--- a/tasks/deploy.yml
+++ b/tasks/deploy.yml
@@ -88,10 +88,21 @@
     jid: "{{ long_command.ansible_job_id }}"
   register: job_result
   until: job_result.finished
+  ignore_errors: true
   delay: 30
   retries: 80 # 40 minutes (80 * 30 secs delay)
 
-# Output of waiting script stdout
-- name: Output waiting script logs
+- name: Waiting script logs (stdout)
   debug:
     var: job_result.stdout_lines
+  when: job_result.stdout_lines is defined
+
+- name: Waiting script logs (stderr)
+  debug:
+    var: job_result.stderr_lines
+  when: job_result.stderr_lines is defined
+
+- name: Fail in case of timeout or failure
+  fail:
+    msg: "Deployment failed. Please check logs above."
+  when: not job_result.finished or not job_result.rc == 0