From 5da7f0d812714f9d2af03522d269bd2f462ee939 Mon Sep 17 00:00:00 2001 From: Paul Bonaud Date: Thu, 1 Aug 2019 10:38:53 +0200 Subject: deploy: add a timeout in the polling waiting script --- files/clever-wait-deploy.sh | 30 +++++++++++++++++++++++------- tasks/deploy.yml | 15 +++++++++++++-- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/files/clever-wait-deploy.sh b/files/clever-wait-deploy.sh index 98c5c37..a95c96f 100755 --- a/files/clever-wait-deploy.sh +++ b/files/clever-wait-deploy.sh @@ -4,18 +4,35 @@ set -ueo pipefail VERBOSE=${VERBOSE:-} lastCleverActivity="" +timeout=2300 # 100 seconds less than the Ansible tasks' timeout +startTime=$(date +%s) function cleverActivity { lastCleverActivity=$(clever activity) } function ensure { - if [ "$?" == "1" ] - then - echo "✗ Deployment failed!" - fi + local lastReturnCode="$?" + VERBOSE=true verbose + + if isNotTimeout + then + if [ "$lastReturnCode" == "0" ] + then + echo "✓ Deployment done." + else + echo "✗ Deployment failed!" + fi + else + echo "⁈ Deployment timeout... Please check clever logs" + exit 1 + fi +} + +function isNotTimeout { + [ $(($(date +%s) - startTime)) -lt $timeout ] } trap ensure EXIT ERR @@ -52,7 +69,7 @@ function check { local samplingTime=5 echo "️▫ Waiting for deployment to start..." - while inactive "$commit" + while inactive "$commit" && isNotTimeout do verbose sleep $samplingTime @@ -60,14 +77,13 @@ function check { # Wait for completion echo "▪ Deployment in progress..." - while deploying "$commit" + while deploying "$commit" && isNotTimeout do verbose sleep $samplingTime done deployed "$commit" - echo "✓ Deployment done." } function getHeadRev { diff --git a/tasks/deploy.yml b/tasks/deploy.yml index df02464..d5e5a45 100644 --- a/tasks/deploy.yml +++ b/tasks/deploy.yml @@ -88,10 +88,21 @@ jid: "{{ long_command.ansible_job_id }}" register: job_result until: job_result.finished + ignore_errors: true delay: 30 retries: 80 # 40 minutes (80 * 30 secs delay) -# Output of waiting script stdout -- name: Output waiting script logs +- name: Waiting script logs (stdout) debug: var: job_result.stdout_lines + when: job_result.stdout_lines is defined + +- name: Waiting script logs (stderr) + debug: + var: job_result.stderr_lines + when: job_result.stderr_lines is defined + +- name: Fail in case of timeout or failure + fail: + msg: "Deployment failed. Please check logs above." + when: not job_result.finished or not job_result.rc == 0 -- cgit v1.2.3