From 0a21fb6c2c52ca5cc2dfdfc41ca0a51c0d81296c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Tue, 13 Mar 2018 13:23:17 +0100 Subject: Start to cleanup the files --- modules/role/manifests/cryptoportfolio.pp | 273 ++++++++++----------- .../templates/cryptoportfolio/api_conf.toml.erb | 12 +- .../templates/cryptoportfolio/bot_config.ini.erb | 10 +- .../cryptoportfolio-app.service.erb | 4 +- .../templates/cryptoportfolio/static_conf.env.erb | 6 +- 5 files changed, 145 insertions(+), 160 deletions(-) (limited to 'modules/role') diff --git a/modules/role/manifests/cryptoportfolio.pp b/modules/role/manifests/cryptoportfolio.pp index bec247e..8b4a63b 100644 --- a/modules/role/manifests/cryptoportfolio.pp +++ b/modules/role/manifests/cryptoportfolio.pp @@ -1,4 +1,22 @@ -class role::cryptoportfolio { +class role::cryptoportfolio ( + String $user, + String $group, + String $home, + Optional[String] $env = "prod", + Optional[String] $webhook_url = undef, + String $pg_user, + String $pg_user_replication, + String $pg_db, + Optional[String] $pg_hostname = "localhost", + Optional[String] $pg_port = "5432", + Optional[String] $web_host = undef, + Optional[String] $web_port = "", + Optional[Boolean] $web_ssl = true, + Optional[String] $front_version = undef, + Optional[String] $front_sha256 = undef, + Optional[String] $bot_version = undef, + Optional[String] $bot_sha256 = undef, +) { ensure_resource('exec', 'systemctl daemon-reload', { command => '/usr/bin/systemctl daemon-reload', refreshonly => true @@ -11,37 +29,23 @@ class role::cryptoportfolio { include "profile::apache" include "profile::xmr_stak" - $password_seed = lookup("base_installation::puppet_pass_seed") |$key| { {} } - - $cf_pg_user = "cryptoportfolio" - $cf_pg_user_replication = "cryptoportfolio_replication" - $cf_pg_db = "cryptoportfolio" - $cf_pg_password = generate_password(24, $password_seed, "postgres_cryptoportfolio") - $cf_pg_replication_password = generate_password(24, $password_seed, "postgres_cryptoportfolio_replication") - $cf_pg_hostname = "localhost" - $cf_pg_port = "5432" - $cf_pg_host = "${cf_pg_hostname}:${cf_pg_port}" - - $cf_user = "cryptoportfolio" - $cf_group = "cryptoportfolio" - $cf_home = "/opt/cryptoportfolio" - $cf_env = "prod" - $cf_front_app_host = lookup("base_installation::system_hostname") |$key| { "example.com" } - $cf_front_app_port = "" - $cf_front_app_ssl = "true" - $cf_front_app = "${cf_home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio/Front" + $password_seed = lookup("base_installation::puppet_pass_seed") + + $pg_password = generate_password(24, $password_seed, "postgres_cryptoportfolio") + $pg_replication_password = generate_password(24, $password_seed, "postgres_cryptoportfolio_replication") + $pg_host = "${pg_hostname}:${pg_port}" + + $cf_front_app = "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio/Front" $cf_front_app_api_workdir = "${cf_front_app}/cmd/app" $cf_front_app_api_bin = "${cf_front_app_api_workdir}/cryptoportfolio-app" - $cf_front_app_api_conf = "${cf_home}/conf.toml" + $cf_front_app_api_conf = "${home}/conf.toml" $cf_front_app_api_secret = generate_password(24, $password_seed, "cryptoportfolio_api_secret") $cf_front_app_static_conf = "${cf_front_app}/cmd/web/env/prod.env" - $cf_bot_app = "${cf_home}/bot" - $cf_bot_app_conf = "${cf_home}/bot_config.ini" - $cf_bot_app_reports = "${cf_home}/bot_reports" - - $cf_webhook_url = lookup("cryptoportfolio::slack_webhook") |$key| { "" } + $cf_bot_app = "${home}/bot" + $cf_bot_app_conf = "${home}/bot_config.ini" + $cf_bot_app_reports = "${home}/bot_reports" file { "/var/lib/postgres/data/certs": ensure => directory, @@ -52,21 +56,21 @@ class role::cryptoportfolio { } file { "/var/lib/postgres/data/certs/cert.pem": - source => "file:///etc/letsencrypt/live/$cf_front_app_host/cert.pem", + source => "file:///etc/letsencrypt/live/$web_host/cert.pem", mode => "0600", links => "follow", owner => $::profile::postgresql::pg_user, group => $::profile::postgresql::pg_user, - require => [Letsencrypt::Certonly[$cf_front_app_host], File["/var/lib/postgres/data/certs"]] + require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] } file { "/var/lib/postgres/data/certs/privkey.pem": - source => "file:///etc/letsencrypt/live/$cf_front_app_host/privkey.pem", + source => "file:///etc/letsencrypt/live/$web_host/privkey.pem", mode => "0600", links => "follow", owner => $::profile::postgresql::pg_user, group => $::profile::postgresql::pg_user, - require => [Letsencrypt::Certonly[$cf_front_app_host], File["/var/lib/postgres/data/certs"]] + require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] } postgresql::server::config_entry { "wal_level": @@ -75,52 +79,52 @@ class role::cryptoportfolio { postgresql::server::config_entry { "ssl": value => "on", - require => Letsencrypt::Certonly[$cf_front_app_host], + require => Letsencrypt::Certonly[$web_host], } postgresql::server::config_entry { "ssl_cert_file": value => "/var/lib/postgres/data/certs/cert.pem", - require => Letsencrypt::Certonly[$cf_front_app_host], + require => Letsencrypt::Certonly[$web_host], } postgresql::server::config_entry { "ssl_key_file": value => "/var/lib/postgres/data/certs/privkey.pem", - require => Letsencrypt::Certonly[$cf_front_app_host], + require => Letsencrypt::Certonly[$web_host], } - postgresql::server::db { $cf_pg_db: - user => $cf_pg_user, - password => postgresql_password($cf_pg_user, $cf_pg_password), + postgresql::server::db { $pg_db: + user => $pg_user, + password => postgresql_password($pg_user, $pg_password), } -> - postgresql_psql { "CREATE PUBLICATION ${cf_pg_db}_publication FOR ALL TABLES": - db => $cf_pg_db, - unless => "SELECT 1 FROM pg_catalog.pg_publication WHERE pubname = '${cf_pg_db}_publication'", + postgresql_psql { "CREATE PUBLICATION ${pg_db}_publication FOR ALL TABLES": + db => $pg_db, + unless => "SELECT 1 FROM pg_catalog.pg_publication WHERE pubname = '${pg_db}_publication'", } -> - postgresql::server::role { $cf_pg_user_replication: - db => $cf_pg_db, + postgresql::server::role { $pg_user_replication: + db => $pg_db, replication => true, - password_hash => postgresql_password($cf_pg_user_replication, $cf_pg_replication_password), + password_hash => postgresql_password($pg_user_replication, $pg_replication_password), } -> - postgresql::server::database_grant { $cf_pg_user_replication: - db => $cf_pg_db, + postgresql::server::database_grant { $pg_user_replication: + db => $pg_db, privilege => "CONNECT", - role => $cf_pg_user_replication, + role => $pg_user_replication, } -> - postgresql::server::grant { "all tables in schema:public:$cf_pg_user_replication": - db => $cf_pg_db, - role => $cf_pg_user_replication, + postgresql::server::grant { "all tables in schema:public:$pg_user_replication": + db => $pg_db, + role => $pg_user_replication, privilege => "SELECT", object_type => "ALL TABLES IN SCHEMA", object_name => "public", } -> - postgresql::server::grant { "all sequences in schema:public:$cf_pg_user_replication": - db => $cf_pg_db, - role => $cf_pg_user_replication, + postgresql::server::grant { "all sequences in schema:public:$pg_user_replication": + db => $pg_db, + role => $pg_user_replication, privilege => "SELECT", object_type => "ALL SEQUENCES IN SCHEMA", object_name => "public", @@ -128,16 +132,16 @@ class role::cryptoportfolio { postgresql::server::pg_hba_rule { 'allow localhost TCP access to cryptoportfolio user': type => 'host', - database => $cf_pg_db, - user => $cf_pg_user, + database => $pg_db, + user => $pg_user, address => '127.0.0.1/32', auth_method => 'md5', order => "b0", } postgresql::server::pg_hba_rule { 'allow localhost ip6 TCP access to cryptoportfolio user': type => 'host', - database => $cf_pg_db, - user => $cf_pg_user, + database => $pg_db, + user => $pg_user, address => '::1/128', auth_method => 'md5', order => "b0", @@ -145,83 +149,64 @@ class role::cryptoportfolio { postgresql::server::pg_hba_rule { 'allow TCP access to replication user from immae.eu': type => 'hostssl', - database => $cf_pg_db, - user => $cf_pg_user_replication, + database => $pg_db, + user => $pg_user_replication, address => 'immae.eu', auth_method => 'md5', order => "b0", } - letsencrypt::certonly { $cf_front_app_host: ; - default: * => $::profile::apache::letsencrypt_certonly_default; - } - class { 'apache::mod::headers': } - apache::vhost { $cf_front_app_host: + apache::vhost { $web_host: port => '443', docroot => false, manage_docroot => false, proxy_dest => "http://localhost:8000", request_headers => 'set X-Forwarded-Proto "https"', ssl => true, - ssl_cert => "/etc/letsencrypt/live/$cf_front_app_host/cert.pem", - ssl_key => "/etc/letsencrypt/live/$cf_front_app_host/privkey.pem", - ssl_chain => "/etc/letsencrypt/live/$cf_front_app_host/chain.pem", - require => Letsencrypt::Certonly[$cf_front_app_host], + ssl_cert => "/etc/letsencrypt/live/$web_host/cert.pem", + ssl_key => "/etc/letsencrypt/live/$web_host/privkey.pem", + ssl_chain => "/etc/letsencrypt/live/$web_host/chain.pem", + require => Letsencrypt::Certonly[$web_host], proxy_preserve_host => true; default: * => $::profile::apache::apache_vhost_default; } - user { $cf_user: - name => $cf_user, - ensure => "present", - managehome => true, - home => $cf_home, - system => true, - password => '!!', - } - file { "/usr/local/bin/slack-notify": mode => "0755", source => "puppet:///modules/role/cryptoportfolio/slack-notify.py", } - $front_version = lookup("cryptoportfolio::front_version") |$key| { {} } - $front_sha256 = lookup("cryptoportfolio::front_sha256") |$key| { {} } - - $bot_version = lookup("cryptoportfolio::bot_version") |$key| { {} } - $bot_sha256 = lookup("cryptoportfolio::bot_sha256") |$key| { {} } - unless empty($bot_version) { ensure_packages(["python", "python-pip"]) file { $cf_bot_app: ensure => "directory", mode => "0700", - owner => $cf_user, - group => $cf_group, - require => User[$cf_user], + owner => $user, + group => $group, + require => User["$user:"], } - archive { "${cf_home}/trader_${bot_version}.tar.gz": - path => "${cf_home}/trader_${bot_version}.tar.gz", + archive { "${home}/trader_${bot_version}.tar.gz": + path => "${home}/trader_${bot_version}.tar.gz", source => "https://git.immae.eu/releases/cryptoportfolio/trader/trader_${bot_version}.tar.gz", checksum_type => "sha256", checksum => $bot_sha256, cleanup => false, extract => true, - user => $cf_user, + user => $user, username => $facts["ec2_metadata"]["hostname"], password => generate_password(24, $password_seed, "ldap"), extract_path => $cf_bot_app, - require => [User[$cf_user], File[$cf_bot_app]], + require => [User["$user:"], File[$cf_bot_app]], } ~> exec { "py-cryptoportfolio-dependencies": cwd => $cf_bot_app, - user => $cf_user, - environment => ["HOME=${cf_home}"], + user => $user, + environment => ["HOME=${home}"], command => "/usr/bin/make install", - require => User[$cf_user], + require => User["$user:"], refreshonly => true, before => [ File[$cf_bot_app_conf], @@ -231,53 +216,53 @@ class role::cryptoportfolio { } file { $cf_bot_app_conf: - owner => $cf_user, - group => $cf_group, + owner => $user, + group => $group, mode => "0600", content => template("role/cryptoportfolio/bot_config.ini.erb"), require => [ - User[$cf_user], - Archive["${cf_home}/trader_${bot_version}.tar.gz"], + User["$user:"], + Archive["${home}/trader_${bot_version}.tar.gz"], ], } cron { "py-cryptoportfolio-before": ensure => present, command => "cd $cf_bot_app ; python main.py --config $cf_bot_app_conf --before", - user => "cryptoportfolio", + user => $user, weekday => 7, # Sunday hour => 22, minute => 30, - environment => ["HOME=${cf_home}","PATH=/usr/bin/"], + environment => ["HOME=${home}","PATH=/usr/bin/"], require => [ File[$cf_bot_app_conf], - Archive["${cf_home}/trader_${bot_version}.tar.gz"] + Archive["${home}/trader_${bot_version}.tar.gz"] ], } cron { "py-cryptoportfolio-after": ensure => present, command => "cd $cf_bot_app ; python main.py --config $cf_bot_app_conf --after", - user => "cryptoportfolio", + user => $user, weekday => 1, # Monday hour => 1, minute => 0, - environment => ["HOME=${cf_home}","PATH=/usr/bin/"], + environment => ["HOME=${home}","PATH=/usr/bin/"], require => [ File[$cf_bot_app_conf], - Archive["${cf_home}/trader_${bot_version}.tar.gz"] + Archive["${home}/trader_${bot_version}.tar.gz"] ], } - unless empty($cf_webhook_url) { + unless empty($webhook_url) { exec { "bot-slack-notify": refreshonly => true, environment => [ "P_PROJECT=Trader", - "P_WEBHOOK=${cf_webhook_url}", + "P_WEBHOOK=${webhook_url}", "P_VERSION=${bot_version}", - "P_HOST=${cf_front_app_host}", - "P_HTTPS=${cf_front_app_ssl}", + "P_HOST=${web_host}", + "P_HTTPS=${web_ssl}", ], command => "/usr/local/bin/slack-notify", require => File["/usr/local/bin/slack-notify"], @@ -291,22 +276,22 @@ class role::cryptoportfolio { ensure_packages(["go", "npm", "nodejs", "yarn"]) file { [ - "${cf_home}/go/", - "${cf_home}/go/src", - "${cf_home}/go/src/immae.eu", - "${cf_home}/go/src/immae.eu/Immae", - "${cf_home}/go/src/immae.eu/Immae/Projets", - "${cf_home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies", - "${cf_home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio", + "${home}/go/", + "${home}/go/src", + "${home}/go/src/immae.eu", + "${home}/go/src/immae.eu/Immae", + "${home}/go/src/immae.eu/Immae/Projets", + "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies", + "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio", $cf_front_app]: ensure => "directory", mode => "0700", - owner => $cf_user, - group => $cf_group, - require => User[$cf_user], + owner => $user, + group => $group, + require => User["$user:"], } - file { "${cf_home}/front": + file { "${home}/front": ensure => "link", target => $cf_front_app, before => File[$cf_front_app], @@ -326,28 +311,28 @@ class role::cryptoportfolio { subscribe => [Exec["go-cryptoportfolio-app"], Exec["web-cryptoportfolio-build"]], require => [ File["/etc/systemd/system/cryptoportfolio-app.service"], - Postgresql::Server::Db[$cf_pg_db] + Postgresql::Server::Db[$pg_db] ], } ~> - exec { "dump $cf_pg_db structure": + exec { "dump $pg_db structure": refreshonly => true, user => $::profile::postgresql::pg_user, group => $::profile::postgresql::pg_user, - command => "/usr/bin/pg_dump --schema-only --clean --no-publications $cf_pg_db > /var/lib/postgres/${cf_pg_db}.schema", + command => "/usr/bin/pg_dump --schema-only --clean --no-publications $pg_db > /var/lib/postgres/${pg_db}.schema", } - archive { "${cf_home}/front_${front_version}.tar.gz": - path => "${cf_home}/front_${front_version}.tar.gz", + archive { "${home}/front_${front_version}.tar.gz": + path => "${home}/front_${front_version}.tar.gz", source => "https://git.immae.eu/releases/cryptoportfolio/front/front_${front_version}.tar.gz", checksum_type => "sha256", checksum => $front_sha256, cleanup => false, extract => true, - user => $cf_user, + user => $user, username => $facts["ec2_metadata"]["hostname"], password => generate_password(24, $password_seed, "ldap"), extract_path => $cf_front_app, - require => [User[$cf_user], File[$cf_front_app]], + require => [User["$user:"], File[$cf_front_app]], notify => [ Exec["web-cryptoportfolio-dependencies"], Exec["go-get-dep"], @@ -356,39 +341,39 @@ class role::cryptoportfolio { # Api file { $cf_front_app_api_conf: - owner => $cf_user, - group => $cf_group, + owner => $user, + group => $group, mode => "0600", content => template("role/cryptoportfolio/api_conf.toml.erb"), before => Exec["go-cryptoportfolio-app"], } exec { "go-get-dep": - user => $cf_user, - environment => ["HOME=${cf_home}"], - creates => "${cf_home}/go/bin/dep", + user => $user, + environment => ["HOME=${home}"], + creates => "${home}/go/bin/dep", command => "/usr/bin/go get -u github.com/golang/dep/cmd/dep", refreshonly => true, } ~> exec { "go-cryptoportfolio-dependencies": cwd => $cf_front_app, - user => $cf_user, - environment => ["HOME=${cf_home}"], - command => "${cf_home}/go/bin/dep ensure", + user => $user, + environment => ["HOME=${home}"], + command => "${home}/go/bin/dep ensure", refreshonly => true, } ~> exec { "go-cryptoportfolio-app": cwd => $cf_front_app_api_workdir, - user => $cf_user, - environment => ["HOME=${cf_home}"], + user => $user, + environment => ["HOME=${home}"], command => "/usr/bin/make build", refreshonly => true, } # Static pages file { $cf_front_app_static_conf: - owner => $cf_user, - group => $cf_group, + owner => $user, + group => $group, mode => "0600", content => template("role/cryptoportfolio/static_conf.env.erb"), before => Exec["web-cryptoportfolio-build"], @@ -396,30 +381,30 @@ class role::cryptoportfolio { exec { "web-cryptoportfolio-dependencies": cwd => "${cf_front_app}/cmd/web", - user => $cf_user, - environment => ["HOME=${cf_home}"], + user => $user, + environment => ["HOME=${home}"], command => "/usr/bin/make install", refreshonly => true, require => [Package["npm"], Package["nodejs"], Package["yarn"]] } ~> exec { "web-cryptoportfolio-build": cwd => "${cf_front_app}/cmd/web", - user => $cf_user, - environment => ["HOME=${cf_home}"], + user => $user, + environment => ["HOME=${home}"], path => ["${cf_front_app}/cmd/web/node_modules/.bin/", "/usr/bin"], - command => "/usr/bin/make static ENV=${cf_env}", + command => "/usr/bin/make static ENV=${env}", refreshonly => true, } - unless empty($cf_webhook_url) { + unless empty($webhook_url) { exec { "front-slack-notify": refreshonly => true, environment => [ "P_PROJECT=Front", - "P_WEBHOOK=${cf_webhook_url}", + "P_WEBHOOK=${webhook_url}", "P_VERSION=${front_version}", - "P_HOST=${cf_front_app_host}", - "P_HTTPS=${cf_front_app_ssl}", + "P_HOST=${web_host}", + "P_HTTPS=${web_ssl}", ], command => "/usr/local/bin/slack-notify", require => File["/usr/local/bin/slack-notify"], diff --git a/modules/role/templates/cryptoportfolio/api_conf.toml.erb b/modules/role/templates/cryptoportfolio/api_conf.toml.erb index 13550c9..7a4b66d 100644 --- a/modules/role/templates/cryptoportfolio/api_conf.toml.erb +++ b/modules/role/templates/cryptoportfolio/api_conf.toml.erb @@ -1,15 +1,15 @@ log_level="info" -mode="<%= @cf_env %>" +mode="<%= @env %>" log_out="stdout" [db] -user="<%= @cf_pg_user %>" -password="<%= @cf_pg_password %>" -database="<%= @cf_pg_db %>" -address="<%= @cf_pg_host %>" +user="<%= @pg_user %>" +password="<%= @pg_password %>" +database="<%= @pg_db %>" +address="<%= @pg_host %>" [api] -domain="<%= @cf_front_app_host %>" +domain="<%= @web_host %>" jwt_secret="<%= @cf_front_app_api_secret %>" [app] diff --git a/modules/role/templates/cryptoportfolio/bot_config.ini.erb b/modules/role/templates/cryptoportfolio/bot_config.ini.erb index 30298eb..b0211a6 100644 --- a/modules/role/templates/cryptoportfolio/bot_config.ini.erb +++ b/modules/role/templates/cryptoportfolio/bot_config.ini.erb @@ -1,9 +1,9 @@ [postgresql] -host = <%= @cf_pg_hostname %> -port = <%= @cf_pg_port %> -user = <%= @cf_pg_user %> -password = <%= @cf_pg_password %> -database = <%= @cf_pg_db %> +host = <%= @pg_hostname %> +port = <%= @pg_port %> +user = <%= @pg_user %> +password = <%= @pg_password %> +database = <%= @pg_db %> [app] report_path = <%= @cf_bot_app_reports %> diff --git a/modules/role/templates/cryptoportfolio/cryptoportfolio-app.service.erb b/modules/role/templates/cryptoportfolio/cryptoportfolio-app.service.erb index a521c0e..ed2b908 100644 --- a/modules/role/templates/cryptoportfolio/cryptoportfolio-app.service.erb +++ b/modules/role/templates/cryptoportfolio/cryptoportfolio-app.service.erb @@ -5,8 +5,8 @@ Description=Cryptoportfolio app Type=simple WorkingDirectory=<%= @cf_front_app_api_workdir %> -User=<%= @cf_user %> -Group=<%= @cf_group %> +User=<%= @user %> +Group=<%= @group %> UMask=007 ExecStart=<%= @cf_front_app_api_bin %> -conf <%= @cf_front_app_api_conf %> diff --git a/modules/role/templates/cryptoportfolio/static_conf.env.erb b/modules/role/templates/cryptoportfolio/static_conf.env.erb index db9759d..314ee14 100644 --- a/modules/role/templates/cryptoportfolio/static_conf.env.erb +++ b/modules/role/templates/cryptoportfolio/static_conf.env.erb @@ -1,4 +1,4 @@ -API_HOST="<%= @cf_front_app_host %>" -API_PORT="<%= @cf_front_app_port %>" -API_HTTPS="<%= @cf_front_app_ssl %>" +API_HOST="<%= @web_host %>" +API_PORT="<%= @web_port %>" +API_HTTPS="<%= @web_ssl %>" -- cgit v1.2.3 From a045b9dc12f71c286d4afcb196705f430b6731f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Tue, 13 Mar 2018 14:31:07 +0100 Subject: Cleanup xmr_stak profile --- modules/role/manifests/cryptoportfolio.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'modules/role') diff --git a/modules/role/manifests/cryptoportfolio.pp b/modules/role/manifests/cryptoportfolio.pp index 8b4a63b..5b64787 100644 --- a/modules/role/manifests/cryptoportfolio.pp +++ b/modules/role/manifests/cryptoportfolio.pp @@ -196,7 +196,7 @@ class role::cryptoportfolio ( cleanup => false, extract => true, user => $user, - username => $facts["ec2_metadata"]["hostname"], + username => lookup("base_installation::ldap_cn"), password => generate_password(24, $password_seed, "ldap"), extract_path => $cf_bot_app, require => [User["$user:"], File[$cf_bot_app]], @@ -329,7 +329,7 @@ class role::cryptoportfolio ( cleanup => false, extract => true, user => $user, - username => $facts["ec2_metadata"]["hostname"], + username => lookup("base_installation::ldap_cn"), password => generate_password(24, $password_seed, "ldap"), extract_path => $cf_front_app, require => [User["$user:"], File[$cf_front_app]], -- cgit v1.2.3 From a386ce060c4c49d772bd4d03d6586012a266317e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Tue, 13 Mar 2018 15:00:26 +0100 Subject: Reorder pg_hba rules --- modules/role/manifests/cryptoportfolio.pp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'modules/role') diff --git a/modules/role/manifests/cryptoportfolio.pp b/modules/role/manifests/cryptoportfolio.pp index 5b64787..503620b 100644 --- a/modules/role/manifests/cryptoportfolio.pp +++ b/modules/role/manifests/cryptoportfolio.pp @@ -136,7 +136,7 @@ class role::cryptoportfolio ( user => $pg_user, address => '127.0.0.1/32', auth_method => 'md5', - order => "b0", + order => "05-01", } postgresql::server::pg_hba_rule { 'allow localhost ip6 TCP access to cryptoportfolio user': type => 'host', @@ -144,7 +144,7 @@ class role::cryptoportfolio ( user => $pg_user, address => '::1/128', auth_method => 'md5', - order => "b0", + order => "05-01", } postgresql::server::pg_hba_rule { 'allow TCP access to replication user from immae.eu': @@ -153,7 +153,7 @@ class role::cryptoportfolio ( user => $pg_user_replication, address => 'immae.eu', auth_method => 'md5', - order => "b0", + order => "05-01", } class { 'apache::mod::headers': } -- cgit v1.2.3 From 39e05b4ec79c5dc07417a6cb1f0e1b2dcd111a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Tue, 13 Mar 2018 20:04:52 +0100 Subject: Move cryptoporfolio module to sub-parts --- modules/role/manifests/cryptoportfolio.pp | 385 +-------------------- modules/role/manifests/cryptoportfolio/apache.pp | 17 + modules/role/manifests/cryptoportfolio/bot.pp | 101 ++++++ modules/role/manifests/cryptoportfolio/front.pp | 158 +++++++++ modules/role/manifests/cryptoportfolio/notify.pp | 6 + .../role/manifests/cryptoportfolio/postgresql.pp | 116 +++++++ 6 files changed, 403 insertions(+), 380 deletions(-) create mode 100644 modules/role/manifests/cryptoportfolio/apache.pp create mode 100644 modules/role/manifests/cryptoportfolio/bot.pp create mode 100644 modules/role/manifests/cryptoportfolio/front.pp create mode 100644 modules/role/manifests/cryptoportfolio/notify.pp create mode 100644 modules/role/manifests/cryptoportfolio/postgresql.pp (limited to 'modules/role') diff --git a/modules/role/manifests/cryptoportfolio.pp b/modules/role/manifests/cryptoportfolio.pp index 503620b..799e297 100644 --- a/modules/role/manifests/cryptoportfolio.pp +++ b/modules/role/manifests/cryptoportfolio.pp @@ -17,11 +17,6 @@ class role::cryptoportfolio ( Optional[String] $bot_version = undef, Optional[String] $bot_sha256 = undef, ) { - ensure_resource('exec', 'systemctl daemon-reload', { - command => '/usr/bin/systemctl daemon-reload', - refreshonly => true - }) - include "base_installation" include "profile::tools" @@ -29,387 +24,17 @@ class role::cryptoportfolio ( include "profile::apache" include "profile::xmr_stak" - $password_seed = lookup("base_installation::puppet_pass_seed") - - $pg_password = generate_password(24, $password_seed, "postgres_cryptoportfolio") - $pg_replication_password = generate_password(24, $password_seed, "postgres_cryptoportfolio_replication") - $pg_host = "${pg_hostname}:${pg_port}" - - $cf_front_app = "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio/Front" - $cf_front_app_api_workdir = "${cf_front_app}/cmd/app" - $cf_front_app_api_bin = "${cf_front_app_api_workdir}/cryptoportfolio-app" - $cf_front_app_api_conf = "${home}/conf.toml" - $cf_front_app_api_secret = generate_password(24, $password_seed, "cryptoportfolio_api_secret") - - $cf_front_app_static_conf = "${cf_front_app}/cmd/web/env/prod.env" - - $cf_bot_app = "${home}/bot" - $cf_bot_app_conf = "${home}/bot_config.ini" - $cf_bot_app_reports = "${home}/bot_reports" - - file { "/var/lib/postgres/data/certs": - ensure => directory, - mode => "0700", - owner => $::profile::postgresql::pg_user, - group => $::profile::postgresql::pg_user, - require => File["/var/lib/postgres"], - } - - file { "/var/lib/postgres/data/certs/cert.pem": - source => "file:///etc/letsencrypt/live/$web_host/cert.pem", - mode => "0600", - links => "follow", - owner => $::profile::postgresql::pg_user, - group => $::profile::postgresql::pg_user, - require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] - } - - file { "/var/lib/postgres/data/certs/privkey.pem": - source => "file:///etc/letsencrypt/live/$web_host/privkey.pem", - mode => "0600", - links => "follow", - owner => $::profile::postgresql::pg_user, - group => $::profile::postgresql::pg_user, - require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] - } - - postgresql::server::config_entry { "wal_level": - value => "logical", - } - - postgresql::server::config_entry { "ssl": - value => "on", - require => Letsencrypt::Certonly[$web_host], - } - - postgresql::server::config_entry { "ssl_cert_file": - value => "/var/lib/postgres/data/certs/cert.pem", - require => Letsencrypt::Certonly[$web_host], - } - - postgresql::server::config_entry { "ssl_key_file": - value => "/var/lib/postgres/data/certs/privkey.pem", - require => Letsencrypt::Certonly[$web_host], - } - - postgresql::server::db { $pg_db: - user => $pg_user, - password => postgresql_password($pg_user, $pg_password), - } - -> - postgresql_psql { "CREATE PUBLICATION ${pg_db}_publication FOR ALL TABLES": - db => $pg_db, - unless => "SELECT 1 FROM pg_catalog.pg_publication WHERE pubname = '${pg_db}_publication'", - } - -> - postgresql::server::role { $pg_user_replication: - db => $pg_db, - replication => true, - password_hash => postgresql_password($pg_user_replication, $pg_replication_password), - } - -> - postgresql::server::database_grant { $pg_user_replication: - db => $pg_db, - privilege => "CONNECT", - role => $pg_user_replication, - } - -> - postgresql::server::grant { "all tables in schema:public:$pg_user_replication": - db => $pg_db, - role => $pg_user_replication, - privilege => "SELECT", - object_type => "ALL TABLES IN SCHEMA", - object_name => "public", - } - -> - postgresql::server::grant { "all sequences in schema:public:$pg_user_replication": - db => $pg_db, - role => $pg_user_replication, - privilege => "SELECT", - object_type => "ALL SEQUENCES IN SCHEMA", - object_name => "public", - } - - postgresql::server::pg_hba_rule { 'allow localhost TCP access to cryptoportfolio user': - type => 'host', - database => $pg_db, - user => $pg_user, - address => '127.0.0.1/32', - auth_method => 'md5', - order => "05-01", - } - postgresql::server::pg_hba_rule { 'allow localhost ip6 TCP access to cryptoportfolio user': - type => 'host', - database => $pg_db, - user => $pg_user, - address => '::1/128', - auth_method => 'md5', - order => "05-01", - } + contain "role::cryptoportfolio::postgresql" + contain "role::cryptoportfolio::apache" - postgresql::server::pg_hba_rule { 'allow TCP access to replication user from immae.eu': - type => 'hostssl', - database => $pg_db, - user => $pg_user_replication, - address => 'immae.eu', - auth_method => 'md5', - order => "05-01", - } - - class { 'apache::mod::headers': } - apache::vhost { $web_host: - port => '443', - docroot => false, - manage_docroot => false, - proxy_dest => "http://localhost:8000", - request_headers => 'set X-Forwarded-Proto "https"', - ssl => true, - ssl_cert => "/etc/letsencrypt/live/$web_host/cert.pem", - ssl_key => "/etc/letsencrypt/live/$web_host/privkey.pem", - ssl_chain => "/etc/letsencrypt/live/$web_host/chain.pem", - require => Letsencrypt::Certonly[$web_host], - proxy_preserve_host => true; - default: * => $::profile::apache::apache_vhost_default; - } - - file { "/usr/local/bin/slack-notify": - mode => "0755", - source => "puppet:///modules/role/cryptoportfolio/slack-notify.py", - } + contain "role::cryptoportfolio::notify" unless empty($bot_version) { - ensure_packages(["python", "python-pip"]) - - file { $cf_bot_app: - ensure => "directory", - mode => "0700", - owner => $user, - group => $group, - require => User["$user:"], - } - - archive { "${home}/trader_${bot_version}.tar.gz": - path => "${home}/trader_${bot_version}.tar.gz", - source => "https://git.immae.eu/releases/cryptoportfolio/trader/trader_${bot_version}.tar.gz", - checksum_type => "sha256", - checksum => $bot_sha256, - cleanup => false, - extract => true, - user => $user, - username => lookup("base_installation::ldap_cn"), - password => generate_password(24, $password_seed, "ldap"), - extract_path => $cf_bot_app, - require => [User["$user:"], File[$cf_bot_app]], - } ~> - exec { "py-cryptoportfolio-dependencies": - cwd => $cf_bot_app, - user => $user, - environment => ["HOME=${home}"], - command => "/usr/bin/make install", - require => User["$user:"], - refreshonly => true, - before => [ - File[$cf_bot_app_conf], - Cron["py-cryptoportfolio-before"], - Cron["py-cryptoportfolio-after"], - ] - } - - file { $cf_bot_app_conf: - owner => $user, - group => $group, - mode => "0600", - content => template("role/cryptoportfolio/bot_config.ini.erb"), - require => [ - User["$user:"], - Archive["${home}/trader_${bot_version}.tar.gz"], - ], - } - - cron { "py-cryptoportfolio-before": - ensure => present, - command => "cd $cf_bot_app ; python main.py --config $cf_bot_app_conf --before", - user => $user, - weekday => 7, # Sunday - hour => 22, - minute => 30, - environment => ["HOME=${home}","PATH=/usr/bin/"], - require => [ - File[$cf_bot_app_conf], - Archive["${home}/trader_${bot_version}.tar.gz"] - ], - } - - cron { "py-cryptoportfolio-after": - ensure => present, - command => "cd $cf_bot_app ; python main.py --config $cf_bot_app_conf --after", - user => $user, - weekday => 1, # Monday - hour => 1, - minute => 0, - environment => ["HOME=${home}","PATH=/usr/bin/"], - require => [ - File[$cf_bot_app_conf], - Archive["${home}/trader_${bot_version}.tar.gz"] - ], - } - - unless empty($webhook_url) { - exec { "bot-slack-notify": - refreshonly => true, - environment => [ - "P_PROJECT=Trader", - "P_WEBHOOK=${webhook_url}", - "P_VERSION=${bot_version}", - "P_HOST=${web_host}", - "P_HTTPS=${web_ssl}", - ], - command => "/usr/local/bin/slack-notify", - require => File["/usr/local/bin/slack-notify"], - subscribe => Exec["py-cryptoportfolio-dependencies"], - } - } + contain "role::cryptoportfolio::bot" } # FIXME: restore backup unless empty($front_version) { - ensure_packages(["go", "npm", "nodejs", "yarn"]) - - file { [ - "${home}/go/", - "${home}/go/src", - "${home}/go/src/immae.eu", - "${home}/go/src/immae.eu/Immae", - "${home}/go/src/immae.eu/Immae/Projets", - "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies", - "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio", - $cf_front_app]: - ensure => "directory", - mode => "0700", - owner => $user, - group => $group, - require => User["$user:"], - } - - file { "${home}/front": - ensure => "link", - target => $cf_front_app, - before => File[$cf_front_app], - } - - file { "/etc/systemd/system/cryptoportfolio-app.service": - mode => "0644", - owner => "root", - group => "root", - content => template("role/cryptoportfolio/cryptoportfolio-app.service.erb"), - notify => Exec["systemctl daemon-reload"], - } - - service { 'cryptoportfolio-app': - enable => true, - ensure => "running", - subscribe => [Exec["go-cryptoportfolio-app"], Exec["web-cryptoportfolio-build"]], - require => [ - File["/etc/systemd/system/cryptoportfolio-app.service"], - Postgresql::Server::Db[$pg_db] - ], - } ~> - exec { "dump $pg_db structure": - refreshonly => true, - user => $::profile::postgresql::pg_user, - group => $::profile::postgresql::pg_user, - command => "/usr/bin/pg_dump --schema-only --clean --no-publications $pg_db > /var/lib/postgres/${pg_db}.schema", - } - - archive { "${home}/front_${front_version}.tar.gz": - path => "${home}/front_${front_version}.tar.gz", - source => "https://git.immae.eu/releases/cryptoportfolio/front/front_${front_version}.tar.gz", - checksum_type => "sha256", - checksum => $front_sha256, - cleanup => false, - extract => true, - user => $user, - username => lookup("base_installation::ldap_cn"), - password => generate_password(24, $password_seed, "ldap"), - extract_path => $cf_front_app, - require => [User["$user:"], File[$cf_front_app]], - notify => [ - Exec["web-cryptoportfolio-dependencies"], - Exec["go-get-dep"], - ] - } - - # Api - file { $cf_front_app_api_conf: - owner => $user, - group => $group, - mode => "0600", - content => template("role/cryptoportfolio/api_conf.toml.erb"), - before => Exec["go-cryptoportfolio-app"], - } - - exec { "go-get-dep": - user => $user, - environment => ["HOME=${home}"], - creates => "${home}/go/bin/dep", - command => "/usr/bin/go get -u github.com/golang/dep/cmd/dep", - refreshonly => true, - } ~> - exec { "go-cryptoportfolio-dependencies": - cwd => $cf_front_app, - user => $user, - environment => ["HOME=${home}"], - command => "${home}/go/bin/dep ensure", - refreshonly => true, - } ~> - exec { "go-cryptoportfolio-app": - cwd => $cf_front_app_api_workdir, - user => $user, - environment => ["HOME=${home}"], - command => "/usr/bin/make build", - refreshonly => true, - } - - # Static pages - file { $cf_front_app_static_conf: - owner => $user, - group => $group, - mode => "0600", - content => template("role/cryptoportfolio/static_conf.env.erb"), - before => Exec["web-cryptoportfolio-build"], - } - - exec { "web-cryptoportfolio-dependencies": - cwd => "${cf_front_app}/cmd/web", - user => $user, - environment => ["HOME=${home}"], - command => "/usr/bin/make install", - refreshonly => true, - require => [Package["npm"], Package["nodejs"], Package["yarn"]] - } ~> - exec { "web-cryptoportfolio-build": - cwd => "${cf_front_app}/cmd/web", - user => $user, - environment => ["HOME=${home}"], - path => ["${cf_front_app}/cmd/web/node_modules/.bin/", "/usr/bin"], - command => "/usr/bin/make static ENV=${env}", - refreshonly => true, - } - - unless empty($webhook_url) { - exec { "front-slack-notify": - refreshonly => true, - environment => [ - "P_PROJECT=Front", - "P_WEBHOOK=${webhook_url}", - "P_VERSION=${front_version}", - "P_HOST=${web_host}", - "P_HTTPS=${web_ssl}", - ], - command => "/usr/local/bin/slack-notify", - require => File["/usr/local/bin/slack-notify"], - subscribe => [Exec["go-cryptoportfolio-app"], Exec["web-cryptoportfolio-build"]], - } - } + contain "role::cryptoportfolio::front" } } diff --git a/modules/role/manifests/cryptoportfolio/apache.pp b/modules/role/manifests/cryptoportfolio/apache.pp new file mode 100644 index 0000000..62d5447 --- /dev/null +++ b/modules/role/manifests/cryptoportfolio/apache.pp @@ -0,0 +1,17 @@ +class role::cryptoportfolio::apache inherits role::cryptoportfolio { + class { 'apache::mod::headers': } + apache::vhost { $web_host: + port => '443', + docroot => false, + manage_docroot => false, + proxy_dest => "http://localhost:8000", + request_headers => 'set X-Forwarded-Proto "https"', + ssl => true, + ssl_cert => "/etc/letsencrypt/live/$web_host/cert.pem", + ssl_key => "/etc/letsencrypt/live/$web_host/privkey.pem", + ssl_chain => "/etc/letsencrypt/live/$web_host/chain.pem", + require => Letsencrypt::Certonly[$web_host], + proxy_preserve_host => true; + default: * => $::profile::apache::apache_vhost_default; + } +} diff --git a/modules/role/manifests/cryptoportfolio/bot.pp b/modules/role/manifests/cryptoportfolio/bot.pp new file mode 100644 index 0000000..a15c779 --- /dev/null +++ b/modules/role/manifests/cryptoportfolio/bot.pp @@ -0,0 +1,101 @@ +class role::cryptoportfolio::bot inherits role::cryptoportfolio { + $password_seed = lookup("base_installation::puppet_pass_seed") + + $cf_bot_app = "${home}/bot" + $cf_bot_app_conf = "${home}/bot_config.ini" + $cf_bot_app_reports = "${home}/bot_reports" + + ensure_packages(["python", "python-pip"]) + + file { $cf_bot_app: + ensure => "directory", + mode => "0700", + owner => $user, + group => $group, + require => User["$user:"], + } + + archive { "${home}/trader_${bot_version}.tar.gz": + path => "${home}/trader_${bot_version}.tar.gz", + source => "https://git.immae.eu/releases/cryptoportfolio/trader/trader_${bot_version}.tar.gz", + checksum_type => "sha256", + checksum => $bot_sha256, + cleanup => false, + extract => true, + user => $user, + username => lookup("base_installation::ldap_cn"), + password => generate_password(24, $password_seed, "ldap"), + extract_path => $cf_bot_app, + require => [User["$user:"], File[$cf_bot_app]], + } ~> + exec { "py-cryptoportfolio-dependencies": + cwd => $cf_bot_app, + user => $user, + environment => ["HOME=${home}"], + command => "/usr/bin/make install", + require => User["$user:"], + refreshonly => true, + before => [ + File[$cf_bot_app_conf], + Cron["py-cryptoportfolio-before"], + Cron["py-cryptoportfolio-after"], + ] + } + + $pg_password = generate_password(24, $password_seed, "postgres_cryptoportfolio") + file { $cf_bot_app_conf: + owner => $user, + group => $group, + mode => "0600", + content => template("role/cryptoportfolio/bot_config.ini.erb"), + require => [ + User["$user:"], + Archive["${home}/trader_${bot_version}.tar.gz"], + ], + } + + cron { "py-cryptoportfolio-before": + ensure => present, + command => "cd $cf_bot_app ; python main.py --config $cf_bot_app_conf --before", + user => $user, + weekday => 7, # Sunday + hour => 22, + minute => 30, + environment => ["HOME=${home}","PATH=/usr/bin/"], + require => [ + File[$cf_bot_app_conf], + Archive["${home}/trader_${bot_version}.tar.gz"] + ], + } + + cron { "py-cryptoportfolio-after": + ensure => present, + command => "cd $cf_bot_app ; python main.py --config $cf_bot_app_conf --after", + user => $user, + weekday => 1, # Monday + hour => 1, + minute => 0, + environment => ["HOME=${home}","PATH=/usr/bin/"], + require => [ + File[$cf_bot_app_conf], + Archive["${home}/trader_${bot_version}.tar.gz"] + ], + } + + unless empty($webhook_url) { + exec { "bot-slack-notify": + refreshonly => true, + environment => [ + "P_PROJECT=Trader", + "P_WEBHOOK=${webhook_url}", + "P_VERSION=${bot_version}", + "P_HOST=${web_host}", + "P_HTTPS=${web_ssl}", + ], + command => "/usr/local/bin/slack-notify", + require => File["/usr/local/bin/slack-notify"], + subscribe => Exec["py-cryptoportfolio-dependencies"], + } + } +} + diff --git a/modules/role/manifests/cryptoportfolio/front.pp b/modules/role/manifests/cryptoportfolio/front.pp new file mode 100644 index 0000000..280ef8b --- /dev/null +++ b/modules/role/manifests/cryptoportfolio/front.pp @@ -0,0 +1,158 @@ +class role::cryptoportfolio::front inherits role::cryptoportfolio { + ensure_resource('exec', 'systemctl daemon-reload', { + command => '/usr/bin/systemctl daemon-reload', + refreshonly => true + }) + + $password_seed = lookup("base_installation::puppet_pass_seed") + + $cf_front_app = "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio/Front" + $cf_front_app_api_workdir = "${cf_front_app}/cmd/app" + $cf_front_app_api_bin = "${cf_front_app_api_workdir}/cryptoportfolio-app" + $cf_front_app_api_conf = "${home}/conf.toml" + $cf_front_app_api_secret = generate_password(24, $password_seed, "cryptoportfolio_api_secret") + + $cf_front_app_static_conf = "${cf_front_app}/cmd/web/env/prod.env" + + ensure_packages(["go", "npm", "nodejs", "yarn"]) + + file { [ + "${home}/go/", + "${home}/go/src", + "${home}/go/src/immae.eu", + "${home}/go/src/immae.eu/Immae", + "${home}/go/src/immae.eu/Immae/Projets", + "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies", + "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio", + $cf_front_app]: + ensure => "directory", + mode => "0700", + owner => $user, + group => $group, + require => User["$user:"], + } + + file { "${home}/front": + ensure => "link", + target => $cf_front_app, + before => File[$cf_front_app], + } + + file { "/etc/systemd/system/cryptoportfolio-app.service": + mode => "0644", + owner => "root", + group => "root", + content => template("role/cryptoportfolio/cryptoportfolio-app.service.erb"), + notify => Exec["systemctl daemon-reload"], + } + + service { 'cryptoportfolio-app': + enable => true, + ensure => "running", + subscribe => [File[$cf_front_app_api_conf], Exec["go-cryptoportfolio-app"], Exec["web-cryptoportfolio-build"]], + require => [ + File["/etc/systemd/system/cryptoportfolio-app.service"], + Postgresql::Server::Db[$pg_db] + ], + } ~> + exec { "dump $pg_db structure": + refreshonly => true, + user => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + command => "/usr/bin/pg_dump --schema-only --clean --no-publications $pg_db > /var/lib/postgres/${pg_db}.schema", + } + + archive { "${home}/front_${front_version}.tar.gz": + path => "${home}/front_${front_version}.tar.gz", + source => "https://git.immae.eu/releases/cryptoportfolio/front/front_${front_version}.tar.gz", + checksum_type => "sha256", + checksum => $front_sha256, + cleanup => false, + extract => true, + user => $user, + username => lookup("base_installation::ldap_cn"), + password => generate_password(24, $password_seed, "ldap"), + extract_path => $cf_front_app, + require => [User["$user:"], File[$cf_front_app]], + notify => [ + Exec["web-cryptoportfolio-dependencies"], + Exec["go-get-dep"], + ] + } + + # Api + $pg_password = generate_password(24, $password_seed, "postgres_cryptoportfolio") + $pg_host = "${pg_hostname}:${pg_port}" + file { $cf_front_app_api_conf: + owner => $user, + group => $group, + mode => "0600", + content => template("role/cryptoportfolio/api_conf.toml.erb"), + before => Exec["go-cryptoportfolio-app"], + } + + exec { "go-get-dep": + user => $user, + environment => ["HOME=${home}"], + creates => "${home}/go/bin/dep", + command => "/usr/bin/go get -u github.com/golang/dep/cmd/dep", + refreshonly => true, + } ~> + exec { "go-cryptoportfolio-dependencies": + cwd => $cf_front_app, + user => $user, + environment => ["HOME=${home}"], + command => "${home}/go/bin/dep ensure", + refreshonly => true, + } ~> + exec { "go-cryptoportfolio-app": + cwd => $cf_front_app_api_workdir, + user => $user, + environment => ["HOME=${home}"], + command => "/usr/bin/make build", + refreshonly => true, + } + + # Static pages + file { $cf_front_app_static_conf: + owner => $user, + group => $group, + mode => "0600", + content => template("role/cryptoportfolio/static_conf.env.erb"), + before => Exec["web-cryptoportfolio-build"], + } + + exec { "web-cryptoportfolio-dependencies": + cwd => "${cf_front_app}/cmd/web", + user => $user, + environment => ["HOME=${home}"], + command => "/usr/bin/make install", + refreshonly => true, + require => [Package["npm"], Package["nodejs"], Package["yarn"]] + } ~> + exec { "web-cryptoportfolio-build": + cwd => "${cf_front_app}/cmd/web", + user => $user, + environment => ["HOME=${home}"], + path => ["${cf_front_app}/cmd/web/node_modules/.bin/", "/usr/bin"], + command => "/usr/bin/make static ENV=${env}", + refreshonly => true, + } + + unless empty($webhook_url) { + exec { "front-slack-notify": + refreshonly => true, + environment => [ + "P_PROJECT=Front", + "P_WEBHOOK=${webhook_url}", + "P_VERSION=${front_version}", + "P_HOST=${web_host}", + "P_HTTPS=${web_ssl}", + ], + command => "/usr/local/bin/slack-notify", + require => File["/usr/local/bin/slack-notify"], + subscribe => [Exec["go-cryptoportfolio-app"], Exec["web-cryptoportfolio-build"]], + } + } + +} diff --git a/modules/role/manifests/cryptoportfolio/notify.pp b/modules/role/manifests/cryptoportfolio/notify.pp new file mode 100644 index 0000000..218312c --- /dev/null +++ b/modules/role/manifests/cryptoportfolio/notify.pp @@ -0,0 +1,6 @@ +class role::cryptoportfolio::notify inherits role::cryptoportfolio { + file { "/usr/local/bin/slack-notify": + mode => "0755", + source => "puppet:///modules/role/cryptoportfolio/slack-notify.py", + } +} diff --git a/modules/role/manifests/cryptoportfolio/postgresql.pp b/modules/role/manifests/cryptoportfolio/postgresql.pp new file mode 100644 index 0000000..cc4d2a9 --- /dev/null +++ b/modules/role/manifests/cryptoportfolio/postgresql.pp @@ -0,0 +1,116 @@ +class role::cryptoportfolio::postgresql inherits role::cryptoportfolio { + $password_seed = lookup("base_installation::puppet_pass_seed") + + $pg_password = generate_password(24, $password_seed, "postgres_cryptoportfolio") + $pg_replication_password = generate_password(24, $password_seed, "postgres_cryptoportfolio_replication") + + file { "/var/lib/postgres/data/certs": + ensure => directory, + mode => "0700", + owner => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + require => File["/var/lib/postgres"], + } + + file { "/var/lib/postgres/data/certs/cert.pem": + source => "file:///etc/letsencrypt/live/$web_host/cert.pem", + mode => "0600", + links => "follow", + owner => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] + } + + file { "/var/lib/postgres/data/certs/privkey.pem": + source => "file:///etc/letsencrypt/live/$web_host/privkey.pem", + mode => "0600", + links => "follow", + owner => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] + } + + postgresql::server::config_entry { "wal_level": + value => "logical", + } + + postgresql::server::config_entry { "ssl": + value => "on", + require => Letsencrypt::Certonly[$web_host], + } + + postgresql::server::config_entry { "ssl_cert_file": + value => "/var/lib/postgres/data/certs/cert.pem", + require => Letsencrypt::Certonly[$web_host], + } + + postgresql::server::config_entry { "ssl_key_file": + value => "/var/lib/postgres/data/certs/privkey.pem", + require => Letsencrypt::Certonly[$web_host], + } + + postgresql::server::db { $pg_db: + user => $pg_user, + password => postgresql_password($pg_user, $pg_password), + } + -> + postgresql_psql { "CREATE PUBLICATION ${pg_db}_publication FOR ALL TABLES": + db => $pg_db, + unless => "SELECT 1 FROM pg_catalog.pg_publication WHERE pubname = '${pg_db}_publication'", + } + -> + postgresql::server::role { $pg_user_replication: + db => $pg_db, + replication => true, + password_hash => postgresql_password($pg_user_replication, $pg_replication_password), + } + -> + postgresql::server::database_grant { $pg_user_replication: + db => $pg_db, + privilege => "CONNECT", + role => $pg_user_replication, + } + -> + postgresql::server::grant { "all tables in schema:public:$pg_user_replication": + db => $pg_db, + role => $pg_user_replication, + privilege => "SELECT", + object_type => "ALL TABLES IN SCHEMA", + object_name => "public", + } + -> + postgresql::server::grant { "all sequences in schema:public:$pg_user_replication": + db => $pg_db, + role => $pg_user_replication, + privilege => "SELECT", + object_type => "ALL SEQUENCES IN SCHEMA", + object_name => "public", + } + + postgresql::server::pg_hba_rule { 'allow localhost TCP access to cryptoportfolio user': + type => 'host', + database => $pg_db, + user => $pg_user, + address => '127.0.0.1/32', + auth_method => 'md5', + order => "05-01", + } + postgresql::server::pg_hba_rule { 'allow localhost ip6 TCP access to cryptoportfolio user': + type => 'host', + database => $pg_db, + user => $pg_user, + address => '::1/128', + auth_method => 'md5', + order => "05-01", + } + + postgresql::server::pg_hba_rule { 'allow TCP access to replication user from immae.eu': + type => 'hostssl', + database => $pg_db, + user => $pg_user_replication, + address => 'immae.eu', + auth_method => 'md5', + order => "05-01", + } + +} -- cgit v1.2.3 From d87a489f9585d10f0a185beb59ae16a10f27a7bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Fri, 16 Mar 2018 20:40:27 +0100 Subject: Add backup role --- modules/role/manifests/backup.pp | 122 +++++++++++++++++++++ .../templates/backup/backup_dirname_head.sh.erb | 27 +++++ .../templates/backup/backup_dirname_part.sh.erb | 26 +++++ .../templates/backup/backup_dirname_tail.sh.erb | 4 + modules/role/templates/backup/backup_head.sh.erb | 20 ++++ .../role/templates/backup/backup_immae_eu.sh.erb | 79 +++++++++++++ modules/role/templates/backup/backup_tail.sh.erb | 0 .../templates/backup/ssh_host_changed.info.erb | 4 + .../role/templates/backup/ssh_key_changed.info.erb | 5 + 9 files changed, 287 insertions(+) create mode 100644 modules/role/manifests/backup.pp create mode 100644 modules/role/templates/backup/backup_dirname_head.sh.erb create mode 100644 modules/role/templates/backup/backup_dirname_part.sh.erb create mode 100644 modules/role/templates/backup/backup_dirname_tail.sh.erb create mode 100644 modules/role/templates/backup/backup_head.sh.erb create mode 100644 modules/role/templates/backup/backup_immae_eu.sh.erb create mode 100644 modules/role/templates/backup/backup_tail.sh.erb create mode 100644 modules/role/templates/backup/ssh_host_changed.info.erb create mode 100644 modules/role/templates/backup/ssh_key_changed.info.erb (limited to 'modules/role') diff --git a/modules/role/manifests/backup.pp b/modules/role/manifests/backup.pp new file mode 100644 index 0000000..edfd5e0 --- /dev/null +++ b/modules/role/manifests/backup.pp @@ -0,0 +1,122 @@ +class role::backup ( + String $user, + String $group, + String $mailto, + Optional[Array] $backups = [], + Optional[String] $mountpoint = "/backup1", + Optional[String] $backup_script = "/usr/local/bin/backup.sh", +) { + include "base_installation" + + include "profile::mail" + include "profile::tools" + include "profile::xmr_stak" + include "profile::known_hosts" + + ssh_keygen { $user: + notify => Notify_refresh["notify-backup-sshkey-change"] + } + + $hosts = $backups.map |$backup| { $backup["host"] } + + notify_refresh { "notify-backup-sshkey-change": + message => template("role/backup/ssh_key_changed.info.erb"), + refreshonly => true + } + + $hosts.each |$host| { + notify_refresh { "notify-backup-sshhost-$host-changed": + message => template("role/backup/ssh_host_changed.info.erb"), + refreshonly => true, + subscribe => Sshkey[$host], + } + } + + concat { $backup_script: + ensure => "present", + ensure_newline => true, + mode => "0755", + } + + cron { "backup": + ensure => present, + command => $backup_script, + user => $user, + minute => 25, + hour => 3, + require => Concat[$backup_script], + } + + concat::fragment { "backup_head": + target => $backup_script, + content => template("role/backup/backup_head.sh.erb"), + order => "01-50", + } + + concat::fragment { "backup_tail": + target => $backup_script, + content => template("role/backup/backup_tail.sh.erb"), + order => "99-50", + } + + $backups.each |$infos| { + $dirname = $infos["name"] + $login = $infos["login"] + $host = $infos["host"] + $dest = "$login@$host" + $base = "$mountpoint/$dirname" + $nbr = $infos["nbr"] + $order_dirname = $infos["order"] + + file { $base: + ensure => "directory", + owner => $user, + group => $group, + require => Mount[$mountpoint], + } -> + file { "$base/older": + ensure => "directory", + owner => $user, + group => $group, + } -> + file { "$base/rsync_output": + ensure => "directory", + owner => $user, + group => $group, + } + + concat::fragment { "backup_${dirname}_head": + target => $backup_script, + content => template("role/backup/backup_dirname_head.sh.erb"), + order => "$order_dirname-01", + } + + concat::fragment { "backup_${dirname}_tail": + target => $backup_script, + content => template("role/backup/backup_dirname_tail.sh.erb"), + order => "$order_dirname-99", + } + + $infos["parts"].each |$part| { + $local_folder = $part["local_folder"] + $remote_folder = $part["remote_folder"] + $exclude_from = $part["exclude_from"] + $files_from = $part["files_from"] + $args = $part["args"] + $order_part = $part["order"] + + file { "$base/$local_folder": + ensure => "directory", + owner => $user, + group => $group, + require => File[$base], + } + + concat::fragment { "backup_${dirname}_${local_folder}": + target => $backup_script, + content => template("role/backup/backup_dirname_part.sh.erb"), + order => "$order_dirname-$order_part", + } + } + } +} diff --git a/modules/role/templates/backup/backup_dirname_head.sh.erb b/modules/role/templates/backup/backup_dirname_head.sh.erb new file mode 100644 index 0000000..e20cfd3 --- /dev/null +++ b/modules/role/templates/backup/backup_dirname_head.sh.erb @@ -0,0 +1,27 @@ +##### <%= @dirname %> ##### +DEST="<%= @dest %>" +BASE="<%= @base %>" +OLD_BAK_BASE=$BASE/older/j +BAK_BASE=${OLD_BAK_BASE}0 +RSYNC_OUTPUT=$BASE/rsync_output +NBR=<%= @nbr %> + +if ! ssh \ + -o PreferredAuthentications=publickey \ + -o StrictHostKeyChecking=yes \ + -o ClearAllForwardings=yes \ + $DEST backup; then + echo "Fichier de verrouillage backup sur $DEST ou impossible de se connecter" >&2 + skip=$DEST +fi + +rm -rf ${OLD_BAK_BASE}${NBR} +for j in `seq -w $(($NBR-1)) -1 0`; do + [ ! -d ${OLD_BAK_BASE}$j ] && continue + mv ${OLD_BAK_BASE}$j ${OLD_BAK_BASE}$(($j+1)) +done +mkdir $BAK_BASE +mv $RSYNC_OUTPUT $BAK_BASE +mkdir $RSYNC_OUTPUT + +if [ "$skip" != "$DEST" ]; then diff --git a/modules/role/templates/backup/backup_dirname_part.sh.erb b/modules/role/templates/backup/backup_dirname_part.sh.erb new file mode 100644 index 0000000..ec662c4 --- /dev/null +++ b/modules/role/templates/backup/backup_dirname_part.sh.erb @@ -0,0 +1,26 @@ +### <%= @dirname %> <%= @local_folder %> ### +LOCAL="<%= @local_folder %>" +REMOTE="<%= @remote_folder %>" + +cd $BASE/$LOCAL +cat > $EXCL_FROM < +EOF +cat > $FILES_FROM < +EOF + +OUT=$RSYNC_OUTPUT/$LOCAL +rsync -XAavbrz --fake-super -e ssh --numeric-ids --delete \ + --backup-dir=$BAK_BASE/$LOCAL \ +<%- unless @args.empty? -%> + <%= @args %>\ +<% end -%> +<%- unless @exclude_from.empty? -%> + --exclude-from=$EXCL_FROM \ +<% end -%> +<%- unless @files_from.empty? -%> + --files-from=$FILES_FROM \ +<% end -%> + $DEST:$REMOTE . > $OUT || true +### End <%= @dirname %> <%= @local_folder %> ### diff --git a/modules/role/templates/backup/backup_dirname_tail.sh.erb b/modules/role/templates/backup/backup_dirname_tail.sh.erb new file mode 100644 index 0000000..6b16c9d --- /dev/null +++ b/modules/role/templates/backup/backup_dirname_tail.sh.erb @@ -0,0 +1,4 @@ + + ssh $DEST sh -c "date > .last_backup" +fi # [ "$skip" != "$DEST" ] +##### End <%= @dirname %> ##### diff --git a/modules/role/templates/backup/backup_head.sh.erb b/modules/role/templates/backup/backup_head.sh.erb new file mode 100644 index 0000000..be9f5bf --- /dev/null +++ b/modules/role/templates/backup/backup_head.sh.erb @@ -0,0 +1,20 @@ +#!/bin/bash +MAILTO="<%= @mailto %>" + +EXCL_FROM=`mktemp` +FILES_FROM=`mktemp` +TMP_STDERR=`mktemp` + +on_exit() { + if [ -s "$TMP_STDERR" ]; then + cat "$TMP_STDERR" | mail -Ssendwait -s "save_distant rsync error" "$MAILTO" + fi + rm -f $TMP_STDERR $EXCL_FROM $FILES_FROM +} + +trap "on_exit" EXIT + +exec 2> "$TMP_STDERR" +exec < /dev/null + +set -e diff --git a/modules/role/templates/backup/backup_immae_eu.sh.erb b/modules/role/templates/backup/backup_immae_eu.sh.erb new file mode 100644 index 0000000..4fab30e --- /dev/null +++ b/modules/role/templates/backup/backup_immae_eu.sh.erb @@ -0,0 +1,79 @@ +#!/bin/bash +DEST="<%= @dest %>" +MAILTO="<%= @mailto %>" +BASE="<%= @base %>" +OLD_BAK_BASE=$BASE/older/j +BAK_BASE=${OLD_BAK_BASE}0 +RSYNC_OUTPUT=$BASE/rsync_output +NBR=7 + +TMP=`mktemp` +TMP_STDERR=`mktemp` + +trap "rm -f $TMP $TMP_STDERR" EXIT + +exec 2> "$TMP_STDERR" + +set -e +if ! `ssh -o ClearAllForwardings=yes $DEST backup`; then + echo "Fichier de verrouillage backup sur $DEST" + exit 1 +fi + +rm -rf ${OLD_BAK_BASE}${NBR} +for j in `seq -w $(($NBR-1)) -1 0`; do + [ ! -d ${OLD_BAK_BASE}$j ] && continue + mv ${OLD_BAK_BASE}$j ${OLD_BAK_BASE}$(($j+1)) +done +mkdir $BAK_BASE +mv $RSYNC_OUTPUT $BAK_BASE +mkdir $RSYNC_OUTPUT + +############## +NAME="home" +FOLDER="/home/immae" + +cd $BASE/$NAME +cat > $TMP < $OUT || true + +############## +NAME="system" +FOLDER="/" + +cd $BASE/$NAME +cat > $TMP < $OUT || true + +############## +ssh $DEST sh -c "date > .last_backup" + +if [ -s "$TMP_STDERR" ]; then + cat "$TMP_STDERR" | mail -Ssendwait -s "save_distant rsync error" "$MAILTO" +fi diff --git a/modules/role/templates/backup/backup_tail.sh.erb b/modules/role/templates/backup/backup_tail.sh.erb new file mode 100644 index 0000000..e69de29 diff --git a/modules/role/templates/backup/ssh_host_changed.info.erb b/modules/role/templates/backup/ssh_host_changed.info.erb new file mode 100644 index 0000000..ebf202e --- /dev/null +++ b/modules/role/templates/backup/ssh_host_changed.info.erb @@ -0,0 +1,4 @@ +Host <%= @host %> added, please send <%= @user %> key if necessary. +<%- if File.exist?("/home/#{@user}/.ssh/id_rsa.pub") %> + <%= File.read("/home/#{@user}/.ssh/id_rsa.pub") %> +<% end -%> diff --git a/modules/role/templates/backup/ssh_key_changed.info.erb b/modules/role/templates/backup/ssh_key_changed.info.erb new file mode 100644 index 0000000..43fd2ec --- /dev/null +++ b/modules/role/templates/backup/ssh_key_changed.info.erb @@ -0,0 +1,5 @@ +ssh key of <%= @user %> changed, +please update hosts: +<%- @hosts.each do |host| %> + - <%= host %> +<% end -%> -- cgit v1.2.3 From f25ad097f24e0d39d5dd9ac2cef39760e671e08f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 19 Mar 2018 17:13:37 +0100 Subject: =?UTF-8?q?Don=E2=80=99t=20mount=20unavailables=20mounts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/role/manifests/backup.pp | 1 + 1 file changed, 1 insertion(+) (limited to 'modules/role') diff --git a/modules/role/manifests/backup.pp b/modules/role/manifests/backup.pp index edfd5e0..d7ba486 100644 --- a/modules/role/manifests/backup.pp +++ b/modules/role/manifests/backup.pp @@ -8,6 +8,7 @@ class role::backup ( ) { include "base_installation" + include "profile::fstab" include "profile::mail" include "profile::tools" include "profile::xmr_stak" -- cgit v1.2.3 From f7d7967b445e59a46b3c2546c10eb9cc48ea42a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 19 Mar 2018 20:32:45 +0100 Subject: Add missing package for backup --- modules/role/manifests/backup.pp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'modules/role') diff --git a/modules/role/manifests/backup.pp b/modules/role/manifests/backup.pp index d7ba486..7a0c275 100644 --- a/modules/role/manifests/backup.pp +++ b/modules/role/manifests/backup.pp @@ -14,6 +14,8 @@ class role::backup ( include "profile::xmr_stak" include "profile::known_hosts" + ensure_packages(["rsync"]) + ssh_keygen { $user: notify => Notify_refresh["notify-backup-sshkey-change"] } -- cgit v1.2.3