aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorPaul Bonaud <paul.bonaud@fretlink.com>2019-08-01 10:38:53 +0200
committerPaul Bonaud <paul.bonaud@fretlink.com>2019-08-01 17:47:02 +0200
commit5da7f0d812714f9d2af03522d269bd2f462ee939 (patch)
tree876a2c852df1334d2435e40c29524f19ad2f67ed
parente749969ff845089853920647b0da039938a2371c (diff)
downloadansible-clever-5da7f0d812714f9d2af03522d269bd2f462ee939.tar.gz
ansible-clever-5da7f0d812714f9d2af03522d269bd2f462ee939.tar.zst
ansible-clever-5da7f0d812714f9d2af03522d269bd2f462ee939.zip
deploy: add a timeout in the polling waiting script
-rwxr-xr-xfiles/clever-wait-deploy.sh30
-rw-r--r--tasks/deploy.yml15
2 files changed, 36 insertions, 9 deletions
diff --git a/files/clever-wait-deploy.sh b/files/clever-wait-deploy.sh
index 98c5c37..a95c96f 100755
--- a/files/clever-wait-deploy.sh
+++ b/files/clever-wait-deploy.sh
@@ -4,18 +4,35 @@ set -ueo pipefail
4 4
5VERBOSE=${VERBOSE:-} 5VERBOSE=${VERBOSE:-}
6lastCleverActivity="" 6lastCleverActivity=""
7timeout=2300 # 100 seconds less than the Ansible tasks' timeout
8startTime=$(date +%s)
7 9
8function cleverActivity { 10function cleverActivity {
9 lastCleverActivity=$(clever activity) 11 lastCleverActivity=$(clever activity)
10} 12}
11 13
12function ensure { 14function ensure {
13 if [ "$?" == "1" ] 15 local lastReturnCode="$?"
14 then 16
15 echo "✗ Deployment failed!"
16 fi
17 VERBOSE=true 17 VERBOSE=true
18 verbose 18 verbose
19
20 if isNotTimeout
21 then
22 if [ "$lastReturnCode" == "0" ]
23 then
24 echo "✓ Deployment done."
25 else
26 echo "✗ Deployment failed!"
27 fi
28 else
29 echo "⁈ Deployment timeout... Please check clever logs"
30 exit 1
31 fi
32}
33
34function isNotTimeout {
35 [ $(($(date +%s) - startTime)) -lt $timeout ]
19} 36}
20 37
21trap ensure EXIT ERR 38trap ensure EXIT ERR
@@ -52,7 +69,7 @@ function check {
52 local samplingTime=5 69 local samplingTime=5
53 70
54 echo "️▫ Waiting for deployment to start..." 71 echo "️▫ Waiting for deployment to start..."
55 while inactive "$commit" 72 while inactive "$commit" && isNotTimeout
56 do 73 do
57 verbose 74 verbose
58 sleep $samplingTime 75 sleep $samplingTime
@@ -60,14 +77,13 @@ function check {
60 77
61 # Wait for completion 78 # Wait for completion
62 echo "▪ Deployment in progress..." 79 echo "▪ Deployment in progress..."
63 while deploying "$commit" 80 while deploying "$commit" && isNotTimeout
64 do 81 do
65 verbose 82 verbose
66 sleep $samplingTime 83 sleep $samplingTime
67 done 84 done
68 85
69 deployed "$commit" 86 deployed "$commit"
70 echo "✓ Deployment done."
71} 87}
72 88
73function getHeadRev { 89function getHeadRev {
diff --git a/tasks/deploy.yml b/tasks/deploy.yml
index df02464..d5e5a45 100644
--- a/tasks/deploy.yml
+++ b/tasks/deploy.yml
@@ -88,10 +88,21 @@
88 jid: "{{ long_command.ansible_job_id }}" 88 jid: "{{ long_command.ansible_job_id }}"
89 register: job_result 89 register: job_result
90 until: job_result.finished 90 until: job_result.finished
91 ignore_errors: true
91 delay: 30 92 delay: 30
92 retries: 80 # 40 minutes (80 * 30 secs delay) 93 retries: 80 # 40 minutes (80 * 30 secs delay)
93 94
94# Output of waiting script stdout 95- name: Waiting script logs (stdout)
95- name: Output waiting script logs
96 debug: 96 debug:
97 var: job_result.stdout_lines 97 var: job_result.stdout_lines
98 when: job_result.stdout_lines is defined
99
100- name: Waiting script logs (stderr)
101 debug:
102 var: job_result.stderr_lines
103 when: job_result.stderr_lines is defined
104
105- name: Fail in case of timeout or failure
106 fail:
107 msg: "Deployment failed. Please check logs above."
108 when: not job_result.finished or not job_result.rc == 0