From fa045db6b7d85f1c5aa53b8fba86958bde4b368b Mon Sep 17 00:00:00 2001 From: Paul Bonaud Date: Tue, 30 Jul 2019 17:57:55 +0200 Subject: deploy: revert back to git push + polling In #19 we removed the polling script to rely on the clever cli's ability to wait on `clever deploy` command. After more than 6 months of experimentation it seems the command is still not reliable (sometimes the command never returns, sometimes it returns even when the deployment has finished..). Thus we are reverting back to our manual process of polling `clever activity` every 5 seconds. The timeout for this polling is set at the Ansible level (not in the bash script) and set for 40 minutes. This should give enough time for any of our applications to deploy. --- files/clever-wait-deploy.sh | 80 ++++++++++++++++++++++++++++++++++++++ tasks/deploy.yml | 93 ++++++++++++++++----------------------------- tasks/setup.yml | 1 + 3 files changed, 113 insertions(+), 61 deletions(-) create mode 100755 files/clever-wait-deploy.sh diff --git a/files/clever-wait-deploy.sh b/files/clever-wait-deploy.sh new file mode 100755 index 0000000..98c5c37 --- /dev/null +++ b/files/clever-wait-deploy.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +set -ueo pipefail + +VERBOSE=${VERBOSE:-} +lastCleverActivity="" + +function cleverActivity { + lastCleverActivity=$(clever activity) +} + +function ensure { + if [ "$?" == "1" ] + then + echo "✗ Deployment failed!" + fi + VERBOSE=true + verbose +} + +trap ensure EXIT ERR + +function deploying { + checkStatus "$1" "IN PROGRESS" +} + +function deployed { + checkStatus "$1" "OK" +} + +function inactive { + checkStatus "$1" "" "-v" +} + +function checkStatus { + local commit="$1" + local status="$2" + + cleverActivity + echo "${lastCleverActivity}" | tail -n1 | grep ${3:+ "${3}"} -q -E "${status}.*DEPLOY.*${commit}" +} + +function verbose { + if [ -n "${VERBOSE}" ]; then + echo -e "\\nLast clever activity:" + echo -e "${lastCleverActivity}\\n" + fi +} + +function check { + local commit="$1" + local samplingTime=5 + + echo "️▫ Waiting for deployment to start..." + while inactive "$commit" + do + verbose + sleep $samplingTime + done + + # Wait for completion + echo "▪ Deployment in progress..." + while deploying "$commit" + do + verbose + sleep $samplingTime + done + + deployed "$commit" + echo "✓ Deployment done." +} + +function getHeadRev { + local chdir="$1/.git" + + git --git-dir="$chdir" rev-parse HEAD +} + +workdir="$(pwd)" +check "$(getHeadRev "${workdir}")" diff --git a/tasks/deploy.yml b/tasks/deploy.yml index 74232a6..df02464 100644 --- a/tasks/deploy.yml +++ b/tasks/deploy.yml @@ -31,18 +31,18 @@ - skip_ansible_lint - name: Deploy to Clever-Cloud - shell: "clever deploy --force" + command: "git push --force git+ssh://git@push-par-clevercloud-customers.services.clever-cloud.com/{{ clever_app }}.git HEAD:refs/heads/master" args: chdir: "{{ clever_app_root }}" - environment: - CONFIGURATION_FILE: "{{ clever_login_file }}" register: clever_deploy - async: 900 # 15 minutes timeout - poll: 30 ignore_errors: true tags: - skip_ansible_lint +################################################################## +# Re-deploy only if its the first clever deploy for that project # +################################################################## + - name: First time push to Clever-Cloud needs a full git clone command: "git fetch --unshallow" args: @@ -55,72 +55,43 @@ - skip_ansible_lint - name: Deploy to Clever-Cloud - shell: "clever deploy --force" + command: "git push --force git+ssh://git@push-par-clevercloud-customers.services.clever-cloud.com/{{ clever_app }}.git HEAD:refs/heads/master" args: chdir: "{{ clever_app_root }}" - environment: - CONFIGURATION_FILE: "{{ clever_login_file }}" - when: clever_deploy is failed + when: + - clever_deploy is failed + - clever_deploy.stderr is defined + - clever_deploy.stderr is search("Failed to read git object") register: clever_deploy - async: 900 # 15 minutes timeout - poll: 30 - ignore_errors: true tags: - skip_ansible_lint -- name: Return deployment logs - debug: - var: clever_deploy.stdout - when: clever_deploy.stdout is defined - -- name: Return deployment errors - debug: - var: clever_deploy.stderr - when: - - clever_deploy is failed - - clever_deploy.stderr is defined +############################################################################## +# Poll deployment status from 'clever activity' command. # +# 'clever deploy' command is not yet 100% reliable to get a blocking command # +# until the deployment is over. Hence the need for a custom waiting script. # +############################################################################## -- name: Retrieve clever activity - command: clever activity +- name: Watch deployment status + command: "{{ ansible_env.HOME }}/{{ clever_user_path }}/clever-wait-deploy.sh" args: chdir: "{{ clever_app_root }}" environment: CONFIGURATION_FILE: "{{ clever_login_file }}" - changed_when: false - register: clever_activity_result - -- name: Display clever activity - debug: - var: clever_activity_result.stdout_lines + async: 2400 # 40 minutes + poll: 0 + register: long_command + changed_when: False -- name: Get current commit sha - command: git show -q --format=format:%H HEAD - args: - chdir: "{{ clever_app_root }}" - warn: False - changed_when: false - register: current_commit_sha +- name: Wait 40 minutes for deployment completion + async_status: + jid: "{{ long_command.ansible_job_id }}" + register: job_result + until: job_result.finished + delay: 30 + retries: 80 # 40 minutes (80 * 30 secs delay) -# #### -# Expects all configuration to be located in the project's repository. -# Making a git commit bound to the same *configuration* and *executable* version. -# ## -- name: Fail if current commit is not the last deployed one - fail: - msg: "The clever deployment failed! Please check latest deploy activity logs above." - when: - - clever_deploy is failed - - clever_deploy.stderr is defined - - clever_deploy.stderr is search("application is up-to-date") - - | - (current_commit_sha.stdout_lines[-1] not in clever_activity_result.stdout_lines[-1]) - or - (clever_activity_valid_deploy_keyword not in clever_activity_result.stdout_lines[-1]) - -- name: Fail on any other deployment errors - fail: - msg: "The clever deployment failed! Please check logs above." - when: - - clever_deploy is failed - - clever_deploy.stderr is defined - - clever_deploy.stderr is not search("application is up-to-date") +# Output of waiting script stdout +- name: Output waiting script logs + debug: + var: job_result.stdout_lines diff --git a/tasks/setup.yml b/tasks/setup.yml index 9d1f8aa..c5abb81 100644 --- a/tasks/setup.yml +++ b/tasks/setup.yml @@ -27,5 +27,6 @@ dest: "{{ ansible_env.HOME }}/{{ clever_user_path }}/{{ item }}" mode: 0755 with_items: + - clever-wait-deploy.sh - clever-set-domain.sh - clever-set-drain.sh -- cgit v1.2.3