From 0a21fb6c2c52ca5cc2dfdfc41ca0a51c0d81296c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Tue, 13 Mar 2018 13:23:17 +0100 Subject: Start to cleanup the files --- environments/global/common.yaml | 4 + environments/global/roles/cryptoportfolio.yaml | 17 +- environments/global/types/s1-2.yaml | 2 +- environments/global/types/vps-ovhssd-1.yaml | 2 +- .../integration/roles/cryptoportfolio.yaml | 6 +- environments/production/roles/cryptoportfolio.yaml | 8 +- manifests/site.pp | 2 +- modules/base_installation/manifests/puppet.pp | 2 +- modules/base_installation/manifests/services.pp | 2 +- modules/base_installation/manifests/users.pp | 20 +- modules/profile/manifests/apache.pp | 15 +- modules/role/manifests/cryptoportfolio.pp | 273 ++++++++++----------- .../templates/cryptoportfolio/api_conf.toml.erb | 12 +- .../templates/cryptoportfolio/bot_config.ini.erb | 10 +- .../cryptoportfolio-app.service.erb | 4 +- .../templates/cryptoportfolio/static_conf.env.erb | 6 +- 16 files changed, 199 insertions(+), 186 deletions(-) diff --git a/environments/global/common.yaml b/environments/global/common.yaml index 5b21dca..05d12ad 100644 --- a/environments/global/common.yaml +++ b/environments/global/common.yaml @@ -6,6 +6,8 @@ lookup_options: merge: deep base_installation::system_users: merge: unique + letsencrypt::hosts: + merge: unique classes: stdlib: ~ @@ -35,3 +37,5 @@ base_installation::system_users: key_type: "ssh-rsa" xmr_stak::mining_pool: "pool.minexmr.com:7777" xmr_stak::wallet: "44CA8TxTFYbQqN2kLyk8AnB6Ghz4mcbGpYC2EyXW7A8H9QspvWnTjDn39XUZDPrFwPa5JNwt4TmAxcooPWv4SaJqL87Bcdo" +letsencrypt::email: "sites+letsencrypt@mail.immae.eu" +letsencrypt::try_for_real_hostname: true diff --git a/environments/global/roles/cryptoportfolio.yaml b/environments/global/roles/cryptoportfolio.yaml index 3d36e71..f875c1b 100644 --- a/environments/global/roles/cryptoportfolio.yaml +++ b/environments/global/roles/cryptoportfolio.yaml @@ -1,4 +1,19 @@ --- classes: role::cryptoportfolio: ~ -cryptoportfolio::slack_webhook: "%{ldapvar.self.vars.cf_slack_webhook.0}" +letsencrypt::hosts: "%{lookup('base_installation::system_hostname')}" +role::cryptoportfolio::user: "cryptoportfolio" +role::cryptoportfolio::group: "cryptoportfolio" +role::cryptoportfolio::home: "/home/cryptoportfolio" +role::cryptoportfolio::env: "prod" +role::cryptoportfolio::webhook_url: "%{ldapvar.self.vars.cf_slack_webhook.0}" +role::cryptoportfolio::pg_db: "cryptoportfolio" +role::cryptoportfolio::pg_user: "cryptoportfolio" +role::cryptoportfolio::pg_user_replication: "cryptoportfolio_replication" +role::cryptoportfolio::web_host: "%{lookup('base_installation::system_hostname')}" +role::cryptoportfolio::web_port: "" +role::cryptoportfolio::web_ssl: true +base_installation::system_users: + - username: "%{lookup('role::cryptoportfolio::user')}" + system: true + password: "!!" diff --git a/environments/global/types/s1-2.yaml b/environments/global/types/s1-2.yaml index 496b741..a7ba753 100644 --- a/environments/global/types/s1-2.yaml +++ b/environments/global/types/s1-2.yaml @@ -6,4 +6,4 @@ classes: base_installation::system_hostname: "%{ldapvar.self.vars.host.0}" base_installation::real_hostname: "%{facts.ec2_metadata.hostname}.v.immae.eu" base_installation::ldap_cert_path: "/etc/ssl/certs/ca-certificates.crt" -ssl::try_letsencrypt_for_real_hostname: true +letsencrypt::try_for_real_hostname: true diff --git a/environments/global/types/vps-ovhssd-1.yaml b/environments/global/types/vps-ovhssd-1.yaml index 73f7a45..68534dc 100644 --- a/environments/global/types/vps-ovhssd-1.yaml +++ b/environments/global/types/vps-ovhssd-1.yaml @@ -7,4 +7,4 @@ base_installation::system_hostname: "%{ldapvar.self.vars.host.0}" base_installation::real_hostname: "%{facts.ec2_metadata.hostname}.ovh.net" base_installation::grub_device: "/dev/sdb" base_installation::ldap_cert_path: "/etc/ssl/certs/ca-certificates.crt" -ssl::try_letsencrypt_for_real_hostname: false +letsencrypt::try_for_real_hostname: false diff --git a/environments/integration/roles/cryptoportfolio.yaml b/environments/integration/roles/cryptoportfolio.yaml index 9825bce..6b8eb92 100644 --- a/environments/integration/roles/cryptoportfolio.yaml +++ b/environments/integration/roles/cryptoportfolio.yaml @@ -1,5 +1,3 @@ --- -cryptoportfolio::front_version: v0.0.2-3-g6200f9a -cryptoportfolio::front_sha256: 69d31251ecd4fcea46d93dfee0184b1171019a765b6744b84f6eec6b10e5818f -cryptoportfolio::bot_version: v0.5-8-g34eb08f -cryptoportfolio::bot_sha256: f5b99c4a1cc4db0228f757705a5a909aa301e42787bc5842f8ba442fec0d3fd1 +role::cryptoportfolio::front_version: v0.0.2-3-g6200f9a +role::cryptoportfolio::front_sha256: 69d31251ecd4fcea46d93dfee0184b1171019a765b6744b84f6eec6b10e5818f diff --git a/environments/production/roles/cryptoportfolio.yaml b/environments/production/roles/cryptoportfolio.yaml index c9328e1..566c7f2 100644 --- a/environments/production/roles/cryptoportfolio.yaml +++ b/environments/production/roles/cryptoportfolio.yaml @@ -1,5 +1,5 @@ --- -cryptoportfolio::front_version: v0.0.2-3-g6200f9a -cryptoportfolio::front_sha256: 69d31251ecd4fcea46d93dfee0184b1171019a765b6744b84f6eec6b10e5818f -cryptoportfolio::bot_version: v0.5.1 -cryptoportfolio::bot_sha256: 733789711365b2397bd996689af616a6789207d26c71a31ad1af68620b267d54 +role::cryptoportfolio::front_version: v0.0.2-3-g6200f9a +role::cryptoportfolio::front_sha256: 69d31251ecd4fcea46d93dfee0184b1171019a765b6744b84f6eec6b10e5818f +role::cryptoportfolio::bot_version: v0.5.1 +role::cryptoportfolio::bot_sha256: 733789711365b2397bd996689af616a6789207d26c71a31ad1af68620b267d54 diff --git a/manifests/site.pp b/manifests/site.pp index f922d2b..3d40ad2 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -1,5 +1,5 @@ node default { - lookup('classes', Hash, 'deep').each |$class_name, $class_hash| { + lookup('classes').each |$class_name, $class_hash| { if empty($class_hash) { include $class_name } else { diff --git a/modules/base_installation/manifests/puppet.pp b/modules/base_installation/manifests/puppet.pp index b3ce492..6f7732d 100644 --- a/modules/base_installation/manifests/puppet.pp +++ b/modules/base_installation/manifests/puppet.pp @@ -67,7 +67,7 @@ class base_installation::puppet ( require => File[$base_installation::puppet_conf_path], } - $ips = lookup("ips") |$key| { {} } + $ips = lookup("ips", { 'default_value' => undef }) file { "$base_installation::puppet_conf_path/host_ldap.info": content => template("base_installation/puppet/host_ldap.info.erb"), require => File[$base_installation::puppet_conf_path], diff --git a/modules/base_installation/manifests/services.pp b/modules/base_installation/manifests/services.pp index c641f4b..d7b4d61 100644 --- a/modules/base_installation/manifests/services.pp +++ b/modules/base_installation/manifests/services.pp @@ -38,7 +38,7 @@ class base_installation::services inherits base_installation { group => "root" } - $ip6 = lookup("ips.v6") |$key| { {} } + $ip6 = lookup("ips.v6", { 'default_value' => undef }) file { '/etc/systemd/network/en-dhcp.network': ensure => "present", path => "/etc/systemd/network/en-dhcp.network", diff --git a/modules/base_installation/manifests/users.pp b/modules/base_installation/manifests/users.pp index 766c0f0..f893c51 100644 --- a/modules/base_installation/manifests/users.pp +++ b/modules/base_installation/manifests/users.pp @@ -26,22 +26,26 @@ class base_installation::users ( ensure => "present", groups => $user[groups], managehome => true, + system => !!$user[system], home => "/home/${user[username]}", - notify => Exec["remove_password"], + notify => Exec["remove_password:${user[username]}:${user[userid]}"], purge_ssh_keys => true } - exec { "remove_password": + exec { "remove_password:${user[username]}:${user[userid]}": command => "/usr/bin/chage -d 0 ${user[username]} && /usr/bin/passwd -d ${user[username]}", + onlyif => "/usr/bin/test -z '${user[password]}'", refreshonly => true } - $user[keys].each |$key| { - ssh_authorized_key { "${user[username]}@${key[host]}": - name => "${user[username]}@${key[host]}", - user => $user[username], - type => $key[key_type], - key => $key[key], + if has_key($user, "keys") { + $user[keys].each |$key| { + ssh_authorized_key { "${user[username]}@${key[host]}": + name => "${user[username]}@${key[host]}", + user => $user[username], + type => $key[key_type], + key => $key[key], + } } } } diff --git a/modules/profile/manifests/apache.pp b/modules/profile/manifests/apache.pp index 8db58da..382633b 100644 --- a/modules/profile/manifests/apache.pp +++ b/modules/profile/manifests/apache.pp @@ -67,13 +67,12 @@ class profile::apache { install_method => "package", package_name => "certbot", package_command => "certbot", - # FIXME - email => 'sites+letsencrypt@mail.immae.eu', + email => lookup('letsencrypt::email'), } - $real_hostname = lookup("base_installation::real_hostname") |$key| { {} } + $real_hostname = lookup("base_installation::real_hostname", { "default_value" => undef }) unless empty($real_hostname) { - if (lookup("ssl::try_letsencrypt_for_real_hostname") |$key| { true }) { + if (lookup("letsencrypt::try_for_real_hostname", { "default_value" => true })) { letsencrypt::certonly { $real_hostname: before => Apache::Vhost["default_ssl"]; default: * => $::profile::apache::letsencrypt_certonly_default; @@ -110,6 +109,14 @@ class profile::apache { } } + lookup("letsencrypt::hosts", { "default_value" => [] }).each |$host| { + if ($host != $real_hostname) { # Done above already + letsencrypt::certonly { $host: ; + default: * => $letsencrypt_certonly_default; + } + } + } + apache::vhost { "redirect_no_ssl": port => '80', error_log => false, diff --git a/modules/role/manifests/cryptoportfolio.pp b/modules/role/manifests/cryptoportfolio.pp index bec247e..8b4a63b 100644 --- a/modules/role/manifests/cryptoportfolio.pp +++ b/modules/role/manifests/cryptoportfolio.pp @@ -1,4 +1,22 @@ -class role::cryptoportfolio { +class role::cryptoportfolio ( + String $user, + String $group, + String $home, + Optional[String] $env = "prod", + Optional[String] $webhook_url = undef, + String $pg_user, + String $pg_user_replication, + String $pg_db, + Optional[String] $pg_hostname = "localhost", + Optional[String] $pg_port = "5432", + Optional[String] $web_host = undef, + Optional[String] $web_port = "", + Optional[Boolean] $web_ssl = true, + Optional[String] $front_version = undef, + Optional[String] $front_sha256 = undef, + Optional[String] $bot_version = undef, + Optional[String] $bot_sha256 = undef, +) { ensure_resource('exec', 'systemctl daemon-reload', { command => '/usr/bin/systemctl daemon-reload', refreshonly => true @@ -11,37 +29,23 @@ class role::cryptoportfolio { include "profile::apache" include "profile::xmr_stak" - $password_seed = lookup("base_installation::puppet_pass_seed") |$key| { {} } - - $cf_pg_user = "cryptoportfolio" - $cf_pg_user_replication = "cryptoportfolio_replication" - $cf_pg_db = "cryptoportfolio" - $cf_pg_password = generate_password(24, $password_seed, "postgres_cryptoportfolio") - $cf_pg_replication_password = generate_password(24, $password_seed, "postgres_cryptoportfolio_replication") - $cf_pg_hostname = "localhost" - $cf_pg_port = "5432" - $cf_pg_host = "${cf_pg_hostname}:${cf_pg_port}" - - $cf_user = "cryptoportfolio" - $cf_group = "cryptoportfolio" - $cf_home = "/opt/cryptoportfolio" - $cf_env = "prod" - $cf_front_app_host = lookup("base_installation::system_hostname") |$key| { "example.com" } - $cf_front_app_port = "" - $cf_front_app_ssl = "true" - $cf_front_app = "${cf_home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio/Front" + $password_seed = lookup("base_installation::puppet_pass_seed") + + $pg_password = generate_password(24, $password_seed, "postgres_cryptoportfolio") + $pg_replication_password = generate_password(24, $password_seed, "postgres_cryptoportfolio_replication") + $pg_host = "${pg_hostname}:${pg_port}" + + $cf_front_app = "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio/Front" $cf_front_app_api_workdir = "${cf_front_app}/cmd/app" $cf_front_app_api_bin = "${cf_front_app_api_workdir}/cryptoportfolio-app" - $cf_front_app_api_conf = "${cf_home}/conf.toml" + $cf_front_app_api_conf = "${home}/conf.toml" $cf_front_app_api_secret = generate_password(24, $password_seed, "cryptoportfolio_api_secret") $cf_front_app_static_conf = "${cf_front_app}/cmd/web/env/prod.env" - $cf_bot_app = "${cf_home}/bot" - $cf_bot_app_conf = "${cf_home}/bot_config.ini" - $cf_bot_app_reports = "${cf_home}/bot_reports" - - $cf_webhook_url = lookup("cryptoportfolio::slack_webhook") |$key| { "" } + $cf_bot_app = "${home}/bot" + $cf_bot_app_conf = "${home}/bot_config.ini" + $cf_bot_app_reports = "${home}/bot_reports" file { "/var/lib/postgres/data/certs": ensure => directory, @@ -52,21 +56,21 @@ class role::cryptoportfolio { } file { "/var/lib/postgres/data/certs/cert.pem": - source => "file:///etc/letsencrypt/live/$cf_front_app_host/cert.pem", + source => "file:///etc/letsencrypt/live/$web_host/cert.pem", mode => "0600", links => "follow", owner => $::profile::postgresql::pg_user, group => $::profile::postgresql::pg_user, - require => [Letsencrypt::Certonly[$cf_front_app_host], File["/var/lib/postgres/data/certs"]] + require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] } file { "/var/lib/postgres/data/certs/privkey.pem": - source => "file:///etc/letsencrypt/live/$cf_front_app_host/privkey.pem", + source => "file:///etc/letsencrypt/live/$web_host/privkey.pem", mode => "0600", links => "follow", owner => $::profile::postgresql::pg_user, group => $::profile::postgresql::pg_user, - require => [Letsencrypt::Certonly[$cf_front_app_host], File["/var/lib/postgres/data/certs"]] + require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] } postgresql::server::config_entry { "wal_level": @@ -75,52 +79,52 @@ class role::cryptoportfolio { postgresql::server::config_entry { "ssl": value => "on", - require => Letsencrypt::Certonly[$cf_front_app_host], + require => Letsencrypt::Certonly[$web_host], } postgresql::server::config_entry { "ssl_cert_file": value => "/var/lib/postgres/data/certs/cert.pem", - require => Letsencrypt::Certonly[$cf_front_app_host], + require => Letsencrypt::Certonly[$web_host], } postgresql::server::config_entry { "ssl_key_file": value => "/var/lib/postgres/data/certs/privkey.pem", - require => Letsencrypt::Certonly[$cf_front_app_host], + require => Letsencrypt::Certonly[$web_host], } - postgresql::server::db { $cf_pg_db: - user => $cf_pg_user, - password => postgresql_password($cf_pg_user, $cf_pg_password), + postgresql::server::db { $pg_db: + user => $pg_user, + password => postgresql_password($pg_user, $pg_password), } -> - postgresql_psql { "CREATE PUBLICATION ${cf_pg_db}_publication FOR ALL TABLES": - db => $cf_pg_db, - unless => "SELECT 1 FROM pg_catalog.pg_publication WHERE pubname = '${cf_pg_db}_publication'", + postgresql_psql { "CREATE PUBLICATION ${pg_db}_publication FOR ALL TABLES": + db => $pg_db, + unless => "SELECT 1 FROM pg_catalog.pg_publication WHERE pubname = '${pg_db}_publication'", } -> - postgresql::server::role { $cf_pg_user_replication: - db => $cf_pg_db, + postgresql::server::role { $pg_user_replication: + db => $pg_db, replication => true, - password_hash => postgresql_password($cf_pg_user_replication, $cf_pg_replication_password), + password_hash => postgresql_password($pg_user_replication, $pg_replication_password), } -> - postgresql::server::database_grant { $cf_pg_user_replication: - db => $cf_pg_db, + postgresql::server::database_grant { $pg_user_replication: + db => $pg_db, privilege => "CONNECT", - role => $cf_pg_user_replication, + role => $pg_user_replication, } -> - postgresql::server::grant { "all tables in schema:public:$cf_pg_user_replication": - db => $cf_pg_db, - role => $cf_pg_user_replication, + postgresql::server::grant { "all tables in schema:public:$pg_user_replication": + db => $pg_db, + role => $pg_user_replication, privilege => "SELECT", object_type => "ALL TABLES IN SCHEMA", object_name => "public", } -> - postgresql::server::grant { "all sequences in schema:public:$cf_pg_user_replication": - db => $cf_pg_db, - role => $cf_pg_user_replication, + postgresql::server::grant { "all sequences in schema:public:$pg_user_replication": + db => $pg_db, + role => $pg_user_replication, privilege => "SELECT", object_type => "ALL SEQUENCES IN SCHEMA", object_name => "public", @@ -128,16 +132,16 @@ class role::cryptoportfolio { postgresql::server::pg_hba_rule { 'allow localhost TCP access to cryptoportfolio user': type => 'host', - database => $cf_pg_db, - user => $cf_pg_user, + database => $pg_db, + user => $pg_user, address => '127.0.0.1/32', auth_method => 'md5', order => "b0", } postgresql::server::pg_hba_rule { 'allow localhost ip6 TCP access to cryptoportfolio user': type => 'host', - database => $cf_pg_db, - user => $cf_pg_user, + database => $pg_db, + user => $pg_user, address => '::1/128', auth_method => 'md5', order => "b0", @@ -145,83 +149,64 @@ class role::cryptoportfolio { postgresql::server::pg_hba_rule { 'allow TCP access to replication user from immae.eu': type => 'hostssl', - database => $cf_pg_db, - user => $cf_pg_user_replication, + database => $pg_db, + user => $pg_user_replication, address => 'immae.eu', auth_method => 'md5', order => "b0", } - letsencrypt::certonly { $cf_front_app_host: ; - default: * => $::profile::apache::letsencrypt_certonly_default; - } - class { 'apache::mod::headers': } - apache::vhost { $cf_front_app_host: + apache::vhost { $web_host: port => '443', docroot => false, manage_docroot => false, proxy_dest => "http://localhost:8000", request_headers => 'set X-Forwarded-Proto "https"', ssl => true, - ssl_cert => "/etc/letsencrypt/live/$cf_front_app_host/cert.pem", - ssl_key => "/etc/letsencrypt/live/$cf_front_app_host/privkey.pem", - ssl_chain => "/etc/letsencrypt/live/$cf_front_app_host/chain.pem", - require => Letsencrypt::Certonly[$cf_front_app_host], + ssl_cert => "/etc/letsencrypt/live/$web_host/cert.pem", + ssl_key => "/etc/letsencrypt/live/$web_host/privkey.pem", + ssl_chain => "/etc/letsencrypt/live/$web_host/chain.pem", + require => Letsencrypt::Certonly[$web_host], proxy_preserve_host => true; default: * => $::profile::apache::apache_vhost_default; } - user { $cf_user: - name => $cf_user, - ensure => "present", - managehome => true, - home => $cf_home, - system => true, - password => '!!', - } - file { "/usr/local/bin/slack-notify": mode => "0755", source => "puppet:///modules/role/cryptoportfolio/slack-notify.py", } - $front_version = lookup("cryptoportfolio::front_version") |$key| { {} } - $front_sha256 = lookup("cryptoportfolio::front_sha256") |$key| { {} } - - $bot_version = lookup("cryptoportfolio::bot_version") |$key| { {} } - $bot_sha256 = lookup("cryptoportfolio::bot_sha256") |$key| { {} } - unless empty($bot_version) { ensure_packages(["python", "python-pip"]) file { $cf_bot_app: ensure => "directory", mode => "0700", - owner => $cf_user, - group => $cf_group, - require => User[$cf_user], + owner => $user, + group => $group, + require => User["$user:"], } - archive { "${cf_home}/trader_${bot_version}.tar.gz": - path => "${cf_home}/trader_${bot_version}.tar.gz", + archive { "${home}/trader_${bot_version}.tar.gz": + path => "${home}/trader_${bot_version}.tar.gz", source => "https://git.immae.eu/releases/cryptoportfolio/trader/trader_${bot_version}.tar.gz", checksum_type => "sha256", checksum => $bot_sha256, cleanup => false, extract => true, - user => $cf_user, + user => $user, username => $facts["ec2_metadata"]["hostname"], password => generate_password(24, $password_seed, "ldap"), extract_path => $cf_bot_app, - require => [User[$cf_user], File[$cf_bot_app]], + require => [User["$user:"], File[$cf_bot_app]], } ~> exec { "py-cryptoportfolio-dependencies": cwd => $cf_bot_app, - user => $cf_user, - environment => ["HOME=${cf_home}"], + user => $user, + environment => ["HOME=${home}"], command => "/usr/bin/make install", - require => User[$cf_user], + require => User["$user:"], refreshonly => true, before => [ File[$cf_bot_app_conf], @@ -231,53 +216,53 @@ class role::cryptoportfolio { } file { $cf_bot_app_conf: - owner => $cf_user, - group => $cf_group, + owner => $user, + group => $group, mode => "0600", content => template("role/cryptoportfolio/bot_config.ini.erb"), require => [ - User[$cf_user], - Archive["${cf_home}/trader_${bot_version}.tar.gz"], + User["$user:"], + Archive["${home}/trader_${bot_version}.tar.gz"], ], } cron { "py-cryptoportfolio-before": ensure => present, command => "cd $cf_bot_app ; python main.py --config $cf_bot_app_conf --before", - user => "cryptoportfolio", + user => $user, weekday => 7, # Sunday hour => 22, minute => 30, - environment => ["HOME=${cf_home}","PATH=/usr/bin/"], + environment => ["HOME=${home}","PATH=/usr/bin/"], require => [ File[$cf_bot_app_conf], - Archive["${cf_home}/trader_${bot_version}.tar.gz"] + Archive["${home}/trader_${bot_version}.tar.gz"] ], } cron { "py-cryptoportfolio-after": ensure => present, command => "cd $cf_bot_app ; python main.py --config $cf_bot_app_conf --after", - user => "cryptoportfolio", + user => $user, weekday => 1, # Monday hour => 1, minute => 0, - environment => ["HOME=${cf_home}","PATH=/usr/bin/"], + environment => ["HOME=${home}","PATH=/usr/bin/"], require => [ File[$cf_bot_app_conf], - Archive["${cf_home}/trader_${bot_version}.tar.gz"] + Archive["${home}/trader_${bot_version}.tar.gz"] ], } - unless empty($cf_webhook_url) { + unless empty($webhook_url) { exec { "bot-slack-notify": refreshonly => true, environment => [ "P_PROJECT=Trader", - "P_WEBHOOK=${cf_webhook_url}", + "P_WEBHOOK=${webhook_url}", "P_VERSION=${bot_version}", - "P_HOST=${cf_front_app_host}", - "P_HTTPS=${cf_front_app_ssl}", + "P_HOST=${web_host}", + "P_HTTPS=${web_ssl}", ], command => "/usr/local/bin/slack-notify", require => File["/usr/local/bin/slack-notify"], @@ -291,22 +276,22 @@ class role::cryptoportfolio { ensure_packages(["go", "npm", "nodejs", "yarn"]) file { [ - "${cf_home}/go/", - "${cf_home}/go/src", - "${cf_home}/go/src/immae.eu", - "${cf_home}/go/src/immae.eu/Immae", - "${cf_home}/go/src/immae.eu/Immae/Projets", - "${cf_home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies", - "${cf_home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio", + "${home}/go/", + "${home}/go/src", + "${home}/go/src/immae.eu", + "${home}/go/src/immae.eu/Immae", + "${home}/go/src/immae.eu/Immae/Projets", + "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies", + "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio", $cf_front_app]: ensure => "directory", mode => "0700", - owner => $cf_user, - group => $cf_group, - require => User[$cf_user], + owner => $user, + group => $group, + require => User["$user:"], } - file { "${cf_home}/front": + file { "${home}/front": ensure => "link", target => $cf_front_app, before => File[$cf_front_app], @@ -326,28 +311,28 @@ class role::cryptoportfolio { subscribe => [Exec["go-cryptoportfolio-app"], Exec["web-cryptoportfolio-build"]], require => [ File["/etc/systemd/system/cryptoportfolio-app.service"], - Postgresql::Server::Db[$cf_pg_db] + Postgresql::Server::Db[$pg_db] ], } ~> - exec { "dump $cf_pg_db structure": + exec { "dump $pg_db structure": refreshonly => true, user => $::profile::postgresql::pg_user, group => $::profile::postgresql::pg_user, - command => "/usr/bin/pg_dump --schema-only --clean --no-publications $cf_pg_db > /var/lib/postgres/${cf_pg_db}.schema", + command => "/usr/bin/pg_dump --schema-only --clean --no-publications $pg_db > /var/lib/postgres/${pg_db}.schema", } - archive { "${cf_home}/front_${front_version}.tar.gz": - path => "${cf_home}/front_${front_version}.tar.gz", + archive { "${home}/front_${front_version}.tar.gz": + path => "${home}/front_${front_version}.tar.gz", source => "https://git.immae.eu/releases/cryptoportfolio/front/front_${front_version}.tar.gz", checksum_type => "sha256", checksum => $front_sha256, cleanup => false, extract => true, - user => $cf_user, + user => $user, username => $facts["ec2_metadata"]["hostname"], password => generate_password(24, $password_seed, "ldap"), extract_path => $cf_front_app, - require => [User[$cf_user], File[$cf_front_app]], + require => [User["$user:"], File[$cf_front_app]], notify => [ Exec["web-cryptoportfolio-dependencies"], Exec["go-get-dep"], @@ -356,39 +341,39 @@ class role::cryptoportfolio { # Api file { $cf_front_app_api_conf: - owner => $cf_user, - group => $cf_group, + owner => $user, + group => $group, mode => "0600", content => template("role/cryptoportfolio/api_conf.toml.erb"), before => Exec["go-cryptoportfolio-app"], } exec { "go-get-dep": - user => $cf_user, - environment => ["HOME=${cf_home}"], - creates => "${cf_home}/go/bin/dep", + user => $user, + environment => ["HOME=${home}"], + creates => "${home}/go/bin/dep", command => "/usr/bin/go get -u github.com/golang/dep/cmd/dep", refreshonly => true, } ~> exec { "go-cryptoportfolio-dependencies": cwd => $cf_front_app, - user => $cf_user, - environment => ["HOME=${cf_home}"], - command => "${cf_home}/go/bin/dep ensure", + user => $user, + environment => ["HOME=${home}"], + command => "${home}/go/bin/dep ensure", refreshonly => true, } ~> exec { "go-cryptoportfolio-app": cwd => $cf_front_app_api_workdir, - user => $cf_user, - environment => ["HOME=${cf_home}"], + user => $user, + environment => ["HOME=${home}"], command => "/usr/bin/make build", refreshonly => true, } # Static pages file { $cf_front_app_static_conf: - owner => $cf_user, - group => $cf_group, + owner => $user, + group => $group, mode => "0600", content => template("role/cryptoportfolio/static_conf.env.erb"), before => Exec["web-cryptoportfolio-build"], @@ -396,30 +381,30 @@ class role::cryptoportfolio { exec { "web-cryptoportfolio-dependencies": cwd => "${cf_front_app}/cmd/web", - user => $cf_user, - environment => ["HOME=${cf_home}"], + user => $user, + environment => ["HOME=${home}"], command => "/usr/bin/make install", refreshonly => true, require => [Package["npm"], Package["nodejs"], Package["yarn"]] } ~> exec { "web-cryptoportfolio-build": cwd => "${cf_front_app}/cmd/web", - user => $cf_user, - environment => ["HOME=${cf_home}"], + user => $user, + environment => ["HOME=${home}"], path => ["${cf_front_app}/cmd/web/node_modules/.bin/", "/usr/bin"], - command => "/usr/bin/make static ENV=${cf_env}", + command => "/usr/bin/make static ENV=${env}", refreshonly => true, } - unless empty($cf_webhook_url) { + unless empty($webhook_url) { exec { "front-slack-notify": refreshonly => true, environment => [ "P_PROJECT=Front", - "P_WEBHOOK=${cf_webhook_url}", + "P_WEBHOOK=${webhook_url}", "P_VERSION=${front_version}", - "P_HOST=${cf_front_app_host}", - "P_HTTPS=${cf_front_app_ssl}", + "P_HOST=${web_host}", + "P_HTTPS=${web_ssl}", ], command => "/usr/local/bin/slack-notify", require => File["/usr/local/bin/slack-notify"], diff --git a/modules/role/templates/cryptoportfolio/api_conf.toml.erb b/modules/role/templates/cryptoportfolio/api_conf.toml.erb index 13550c9..7a4b66d 100644 --- a/modules/role/templates/cryptoportfolio/api_conf.toml.erb +++ b/modules/role/templates/cryptoportfolio/api_conf.toml.erb @@ -1,15 +1,15 @@ log_level="info" -mode="<%= @cf_env %>" +mode="<%= @env %>" log_out="stdout" [db] -user="<%= @cf_pg_user %>" -password="<%= @cf_pg_password %>" -database="<%= @cf_pg_db %>" -address="<%= @cf_pg_host %>" +user="<%= @pg_user %>" +password="<%= @pg_password %>" +database="<%= @pg_db %>" +address="<%= @pg_host %>" [api] -domain="<%= @cf_front_app_host %>" +domain="<%= @web_host %>" jwt_secret="<%= @cf_front_app_api_secret %>" [app] diff --git a/modules/role/templates/cryptoportfolio/bot_config.ini.erb b/modules/role/templates/cryptoportfolio/bot_config.ini.erb index 30298eb..b0211a6 100644 --- a/modules/role/templates/cryptoportfolio/bot_config.ini.erb +++ b/modules/role/templates/cryptoportfolio/bot_config.ini.erb @@ -1,9 +1,9 @@ [postgresql] -host = <%= @cf_pg_hostname %> -port = <%= @cf_pg_port %> -user = <%= @cf_pg_user %> -password = <%= @cf_pg_password %> -database = <%= @cf_pg_db %> +host = <%= @pg_hostname %> +port = <%= @pg_port %> +user = <%= @pg_user %> +password = <%= @pg_password %> +database = <%= @pg_db %> [app] report_path = <%= @cf_bot_app_reports %> diff --git a/modules/role/templates/cryptoportfolio/cryptoportfolio-app.service.erb b/modules/role/templates/cryptoportfolio/cryptoportfolio-app.service.erb index a521c0e..ed2b908 100644 --- a/modules/role/templates/cryptoportfolio/cryptoportfolio-app.service.erb +++ b/modules/role/templates/cryptoportfolio/cryptoportfolio-app.service.erb @@ -5,8 +5,8 @@ Description=Cryptoportfolio app Type=simple WorkingDirectory=<%= @cf_front_app_api_workdir %> -User=<%= @cf_user %> -Group=<%= @cf_group %> +User=<%= @user %> +Group=<%= @group %> UMask=007 ExecStart=<%= @cf_front_app_api_bin %> -conf <%= @cf_front_app_api_conf %> diff --git a/modules/role/templates/cryptoportfolio/static_conf.env.erb b/modules/role/templates/cryptoportfolio/static_conf.env.erb index db9759d..314ee14 100644 --- a/modules/role/templates/cryptoportfolio/static_conf.env.erb +++ b/modules/role/templates/cryptoportfolio/static_conf.env.erb @@ -1,4 +1,4 @@ -API_HOST="<%= @cf_front_app_host %>" -API_PORT="<%= @cf_front_app_port %>" -API_HTTPS="<%= @cf_front_app_ssl %>" +API_HOST="<%= @web_host %>" +API_PORT="<%= @web_port %>" +API_HTTPS="<%= @web_ssl %>" -- cgit v1.2.3 From a045b9dc12f71c286d4afcb196705f430b6731f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Tue, 13 Mar 2018 14:31:07 +0100 Subject: Cleanup xmr_stak profile --- environments/global/common.yaml | 4 ++-- modules/profile/manifests/postgresql.pp | 2 +- modules/profile/manifests/xmr_stak.pp | 11 ++++++----- modules/role/manifests/cryptoportfolio.pp | 4 ++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/environments/global/common.yaml b/environments/global/common.yaml index 05d12ad..4536b83 100644 --- a/environments/global/common.yaml +++ b/environments/global/common.yaml @@ -35,7 +35,7 @@ base_installation::system_users: - host: "immae.eu" key: "AAAAB3NzaC1yc2EAAAADAQABAAABAQDi5PgLBwMRyRwzJPnSgUyRAuB9AAxMijsw1pR/t/wmxQne1O5fIPOleHx+D8dyZbwm+XkzlcJpgT0Qy3qC9J8BPhshJvO/tA/8CI/oS/FE0uWsyACH1DMO2dk4gRRZGSE9IuzDMRPlnfZ3n0tdsPzzv3GH4It/oPIgsvkTowKztGLQ7Xmjr5BxzAhXcIQymqA0U3XWHSdWvnSRDaOFG0PDoVMS85IdwlviVKLnV5Sstb4NC/P28LFfgvW8DO/XrOqujgDomqTmR41dK/AyrGGOb2cQUMO4l8Oa+74aOyKaB61rr/rJkr+wCbEttkTvgFa6zZygSk3edfiWE2rgn4+v" key_type: "ssh-rsa" -xmr_stak::mining_pool: "pool.minexmr.com:7777" -xmr_stak::wallet: "44CA8TxTFYbQqN2kLyk8AnB6Ghz4mcbGpYC2EyXW7A8H9QspvWnTjDn39XUZDPrFwPa5JNwt4TmAxcooPWv4SaJqL87Bcdo" +profile::xmr_stak::mining_pool: "pool.minexmr.com:7777" +profile::xmr_stak::wallet: "44CA8TxTFYbQqN2kLyk8AnB6Ghz4mcbGpYC2EyXW7A8H9QspvWnTjDn39XUZDPrFwPa5JNwt4TmAxcooPWv4SaJqL87Bcdo" letsencrypt::email: "sites+letsencrypt@mail.immae.eu" letsencrypt::try_for_real_hostname: true diff --git a/modules/profile/manifests/postgresql.pp b/modules/profile/manifests/postgresql.pp index 1024c66..9d875c9 100644 --- a/modules/profile/manifests/postgresql.pp +++ b/modules/profile/manifests/postgresql.pp @@ -1,5 +1,5 @@ class profile::postgresql { - $password_seed = lookup("base_installation::puppet_pass_seed") |$key| { {} } + $password_seed = lookup("base_installation::puppet_pass_seed") class { '::postgresql::globals': encoding => 'UTF-8', diff --git a/modules/profile/manifests/xmr_stak.pp b/modules/profile/manifests/xmr_stak.pp index e5582eb..c8bbcdd 100644 --- a/modules/profile/manifests/xmr_stak.pp +++ b/modules/profile/manifests/xmr_stak.pp @@ -1,4 +1,8 @@ -class profile::xmr_stak { +class profile::xmr_stak ( + String $mining_pool, + String $wallet, + Optional[String] $password = "x", +) { ensure_resource('exec', 'systemctl daemon-reload', { command => '/usr/bin/systemctl daemon-reload', refreshonly => true @@ -26,10 +30,7 @@ class profile::xmr_stak { notify => Exec["systemctl daemon-reload"] } - $mining_pool = lookup("xmr_stak::mining_pool") |$key| { {} } - $wallet = lookup("xmr_stak::wallet") |$key| { {} } - $password = lookup("xmr_stak::password") |$key| { "x" } - $instance = regsubst($facts["ec2_metadata"]["hostname"], '\.', "_", "G") + $instance = regsubst(lookup("base_installation::ldap_cn"), '\.', "_", "G") file { "/var/lib/xmr_stak/xmr-stak.conf": mode => "0644", diff --git a/modules/role/manifests/cryptoportfolio.pp b/modules/role/manifests/cryptoportfolio.pp index 8b4a63b..5b64787 100644 --- a/modules/role/manifests/cryptoportfolio.pp +++ b/modules/role/manifests/cryptoportfolio.pp @@ -196,7 +196,7 @@ class role::cryptoportfolio ( cleanup => false, extract => true, user => $user, - username => $facts["ec2_metadata"]["hostname"], + username => lookup("base_installation::ldap_cn"), password => generate_password(24, $password_seed, "ldap"), extract_path => $cf_bot_app, require => [User["$user:"], File[$cf_bot_app]], @@ -329,7 +329,7 @@ class role::cryptoportfolio ( cleanup => false, extract => true, user => $user, - username => $facts["ec2_metadata"]["hostname"], + username => lookup("base_installation::ldap_cn"), password => generate_password(24, $password_seed, "ldap"), extract_path => $cf_front_app, require => [User["$user:"], File[$cf_front_app]], -- cgit v1.2.3 From a386ce060c4c49d772bd4d03d6586012a266317e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Tue, 13 Mar 2018 15:00:26 +0100 Subject: Reorder pg_hba rules --- modules/profile/manifests/postgresql.pp | 12 ++++++------ modules/role/manifests/cryptoportfolio.pp | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/profile/manifests/postgresql.pp b/modules/profile/manifests/postgresql.pp index 9d875c9..2cd1bcc 100644 --- a/modules/profile/manifests/postgresql.pp +++ b/modules/profile/manifests/postgresql.pp @@ -32,7 +32,7 @@ class profile::postgresql { database => 'all', user => $pg_user, auth_method => 'ident', - order => "a1", + order => "00-01", } postgresql::server::pg_hba_rule { 'localhost access as postgres user': description => 'Allow localhost access to postgres user', @@ -41,7 +41,7 @@ class profile::postgresql { user => $pg_user, address => "127.0.0.1/32", auth_method => 'md5', - order => "a2", + order => "00-02", } postgresql::server::pg_hba_rule { 'localhost ip6 access as postgres user': description => 'Allow localhost access to postgres user', @@ -50,7 +50,7 @@ class profile::postgresql { user => $pg_user, address => "::1/128", auth_method => 'md5', - order => "a3", + order => "00-03", } postgresql::server::pg_hba_rule { 'deny access to postgresql user': description => 'Deny remote access to postgres user', @@ -59,7 +59,7 @@ class profile::postgresql { user => $pg_user, address => "0.0.0.0/0", auth_method => 'reject', - order => "a4", + order => "00-04", } postgresql::server::pg_hba_rule { 'local access': @@ -68,7 +68,7 @@ class profile::postgresql { database => 'all', user => 'all', auth_method => 'md5', - order => "b1", + order => "10-01", } postgresql::server::pg_hba_rule { 'local access with same name': @@ -77,7 +77,7 @@ class profile::postgresql { database => 'all', user => 'all', auth_method => 'ident', - order => "b2", + order => "10-02", } } diff --git a/modules/role/manifests/cryptoportfolio.pp b/modules/role/manifests/cryptoportfolio.pp index 5b64787..503620b 100644 --- a/modules/role/manifests/cryptoportfolio.pp +++ b/modules/role/manifests/cryptoportfolio.pp @@ -136,7 +136,7 @@ class role::cryptoportfolio ( user => $pg_user, address => '127.0.0.1/32', auth_method => 'md5', - order => "b0", + order => "05-01", } postgresql::server::pg_hba_rule { 'allow localhost ip6 TCP access to cryptoportfolio user': type => 'host', @@ -144,7 +144,7 @@ class role::cryptoportfolio ( user => $pg_user, address => '::1/128', auth_method => 'md5', - order => "b0", + order => "05-01", } postgresql::server::pg_hba_rule { 'allow TCP access to replication user from immae.eu': @@ -153,7 +153,7 @@ class role::cryptoportfolio ( user => $pg_user_replication, address => 'immae.eu', auth_method => 'md5', - order => "b0", + order => "05-01", } class { 'apache::mod::headers': } -- cgit v1.2.3 From 39e05b4ec79c5dc07417a6cb1f0e1b2dcd111a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Tue, 13 Mar 2018 20:04:52 +0100 Subject: Move cryptoporfolio module to sub-parts --- modules/role/manifests/cryptoportfolio.pp | 385 +-------------------- modules/role/manifests/cryptoportfolio/apache.pp | 17 + modules/role/manifests/cryptoportfolio/bot.pp | 101 ++++++ modules/role/manifests/cryptoportfolio/front.pp | 158 +++++++++ modules/role/manifests/cryptoportfolio/notify.pp | 6 + .../role/manifests/cryptoportfolio/postgresql.pp | 116 +++++++ 6 files changed, 403 insertions(+), 380 deletions(-) create mode 100644 modules/role/manifests/cryptoportfolio/apache.pp create mode 100644 modules/role/manifests/cryptoportfolio/bot.pp create mode 100644 modules/role/manifests/cryptoportfolio/front.pp create mode 100644 modules/role/manifests/cryptoportfolio/notify.pp create mode 100644 modules/role/manifests/cryptoportfolio/postgresql.pp diff --git a/modules/role/manifests/cryptoportfolio.pp b/modules/role/manifests/cryptoportfolio.pp index 503620b..799e297 100644 --- a/modules/role/manifests/cryptoportfolio.pp +++ b/modules/role/manifests/cryptoportfolio.pp @@ -17,11 +17,6 @@ class role::cryptoportfolio ( Optional[String] $bot_version = undef, Optional[String] $bot_sha256 = undef, ) { - ensure_resource('exec', 'systemctl daemon-reload', { - command => '/usr/bin/systemctl daemon-reload', - refreshonly => true - }) - include "base_installation" include "profile::tools" @@ -29,387 +24,17 @@ class role::cryptoportfolio ( include "profile::apache" include "profile::xmr_stak" - $password_seed = lookup("base_installation::puppet_pass_seed") - - $pg_password = generate_password(24, $password_seed, "postgres_cryptoportfolio") - $pg_replication_password = generate_password(24, $password_seed, "postgres_cryptoportfolio_replication") - $pg_host = "${pg_hostname}:${pg_port}" - - $cf_front_app = "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio/Front" - $cf_front_app_api_workdir = "${cf_front_app}/cmd/app" - $cf_front_app_api_bin = "${cf_front_app_api_workdir}/cryptoportfolio-app" - $cf_front_app_api_conf = "${home}/conf.toml" - $cf_front_app_api_secret = generate_password(24, $password_seed, "cryptoportfolio_api_secret") - - $cf_front_app_static_conf = "${cf_front_app}/cmd/web/env/prod.env" - - $cf_bot_app = "${home}/bot" - $cf_bot_app_conf = "${home}/bot_config.ini" - $cf_bot_app_reports = "${home}/bot_reports" - - file { "/var/lib/postgres/data/certs": - ensure => directory, - mode => "0700", - owner => $::profile::postgresql::pg_user, - group => $::profile::postgresql::pg_user, - require => File["/var/lib/postgres"], - } - - file { "/var/lib/postgres/data/certs/cert.pem": - source => "file:///etc/letsencrypt/live/$web_host/cert.pem", - mode => "0600", - links => "follow", - owner => $::profile::postgresql::pg_user, - group => $::profile::postgresql::pg_user, - require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] - } - - file { "/var/lib/postgres/data/certs/privkey.pem": - source => "file:///etc/letsencrypt/live/$web_host/privkey.pem", - mode => "0600", - links => "follow", - owner => $::profile::postgresql::pg_user, - group => $::profile::postgresql::pg_user, - require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] - } - - postgresql::server::config_entry { "wal_level": - value => "logical", - } - - postgresql::server::config_entry { "ssl": - value => "on", - require => Letsencrypt::Certonly[$web_host], - } - - postgresql::server::config_entry { "ssl_cert_file": - value => "/var/lib/postgres/data/certs/cert.pem", - require => Letsencrypt::Certonly[$web_host], - } - - postgresql::server::config_entry { "ssl_key_file": - value => "/var/lib/postgres/data/certs/privkey.pem", - require => Letsencrypt::Certonly[$web_host], - } - - postgresql::server::db { $pg_db: - user => $pg_user, - password => postgresql_password($pg_user, $pg_password), - } - -> - postgresql_psql { "CREATE PUBLICATION ${pg_db}_publication FOR ALL TABLES": - db => $pg_db, - unless => "SELECT 1 FROM pg_catalog.pg_publication WHERE pubname = '${pg_db}_publication'", - } - -> - postgresql::server::role { $pg_user_replication: - db => $pg_db, - replication => true, - password_hash => postgresql_password($pg_user_replication, $pg_replication_password), - } - -> - postgresql::server::database_grant { $pg_user_replication: - db => $pg_db, - privilege => "CONNECT", - role => $pg_user_replication, - } - -> - postgresql::server::grant { "all tables in schema:public:$pg_user_replication": - db => $pg_db, - role => $pg_user_replication, - privilege => "SELECT", - object_type => "ALL TABLES IN SCHEMA", - object_name => "public", - } - -> - postgresql::server::grant { "all sequences in schema:public:$pg_user_replication": - db => $pg_db, - role => $pg_user_replication, - privilege => "SELECT", - object_type => "ALL SEQUENCES IN SCHEMA", - object_name => "public", - } - - postgresql::server::pg_hba_rule { 'allow localhost TCP access to cryptoportfolio user': - type => 'host', - database => $pg_db, - user => $pg_user, - address => '127.0.0.1/32', - auth_method => 'md5', - order => "05-01", - } - postgresql::server::pg_hba_rule { 'allow localhost ip6 TCP access to cryptoportfolio user': - type => 'host', - database => $pg_db, - user => $pg_user, - address => '::1/128', - auth_method => 'md5', - order => "05-01", - } + contain "role::cryptoportfolio::postgresql" + contain "role::cryptoportfolio::apache" - postgresql::server::pg_hba_rule { 'allow TCP access to replication user from immae.eu': - type => 'hostssl', - database => $pg_db, - user => $pg_user_replication, - address => 'immae.eu', - auth_method => 'md5', - order => "05-01", - } - - class { 'apache::mod::headers': } - apache::vhost { $web_host: - port => '443', - docroot => false, - manage_docroot => false, - proxy_dest => "http://localhost:8000", - request_headers => 'set X-Forwarded-Proto "https"', - ssl => true, - ssl_cert => "/etc/letsencrypt/live/$web_host/cert.pem", - ssl_key => "/etc/letsencrypt/live/$web_host/privkey.pem", - ssl_chain => "/etc/letsencrypt/live/$web_host/chain.pem", - require => Letsencrypt::Certonly[$web_host], - proxy_preserve_host => true; - default: * => $::profile::apache::apache_vhost_default; - } - - file { "/usr/local/bin/slack-notify": - mode => "0755", - source => "puppet:///modules/role/cryptoportfolio/slack-notify.py", - } + contain "role::cryptoportfolio::notify" unless empty($bot_version) { - ensure_packages(["python", "python-pip"]) - - file { $cf_bot_app: - ensure => "directory", - mode => "0700", - owner => $user, - group => $group, - require => User["$user:"], - } - - archive { "${home}/trader_${bot_version}.tar.gz": - path => "${home}/trader_${bot_version}.tar.gz", - source => "https://git.immae.eu/releases/cryptoportfolio/trader/trader_${bot_version}.tar.gz", - checksum_type => "sha256", - checksum => $bot_sha256, - cleanup => false, - extract => true, - user => $user, - username => lookup("base_installation::ldap_cn"), - password => generate_password(24, $password_seed, "ldap"), - extract_path => $cf_bot_app, - require => [User["$user:"], File[$cf_bot_app]], - } ~> - exec { "py-cryptoportfolio-dependencies": - cwd => $cf_bot_app, - user => $user, - environment => ["HOME=${home}"], - command => "/usr/bin/make install", - require => User["$user:"], - refreshonly => true, - before => [ - File[$cf_bot_app_conf], - Cron["py-cryptoportfolio-before"], - Cron["py-cryptoportfolio-after"], - ] - } - - file { $cf_bot_app_conf: - owner => $user, - group => $group, - mode => "0600", - content => template("role/cryptoportfolio/bot_config.ini.erb"), - require => [ - User["$user:"], - Archive["${home}/trader_${bot_version}.tar.gz"], - ], - } - - cron { "py-cryptoportfolio-before": - ensure => present, - command => "cd $cf_bot_app ; python main.py --config $cf_bot_app_conf --before", - user => $user, - weekday => 7, # Sunday - hour => 22, - minute => 30, - environment => ["HOME=${home}","PATH=/usr/bin/"], - require => [ - File[$cf_bot_app_conf], - Archive["${home}/trader_${bot_version}.tar.gz"] - ], - } - - cron { "py-cryptoportfolio-after": - ensure => present, - command => "cd $cf_bot_app ; python main.py --config $cf_bot_app_conf --after", - user => $user, - weekday => 1, # Monday - hour => 1, - minute => 0, - environment => ["HOME=${home}","PATH=/usr/bin/"], - require => [ - File[$cf_bot_app_conf], - Archive["${home}/trader_${bot_version}.tar.gz"] - ], - } - - unless empty($webhook_url) { - exec { "bot-slack-notify": - refreshonly => true, - environment => [ - "P_PROJECT=Trader", - "P_WEBHOOK=${webhook_url}", - "P_VERSION=${bot_version}", - "P_HOST=${web_host}", - "P_HTTPS=${web_ssl}", - ], - command => "/usr/local/bin/slack-notify", - require => File["/usr/local/bin/slack-notify"], - subscribe => Exec["py-cryptoportfolio-dependencies"], - } - } + contain "role::cryptoportfolio::bot" } # FIXME: restore backup unless empty($front_version) { - ensure_packages(["go", "npm", "nodejs", "yarn"]) - - file { [ - "${home}/go/", - "${home}/go/src", - "${home}/go/src/immae.eu", - "${home}/go/src/immae.eu/Immae", - "${home}/go/src/immae.eu/Immae/Projets", - "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies", - "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio", - $cf_front_app]: - ensure => "directory", - mode => "0700", - owner => $user, - group => $group, - require => User["$user:"], - } - - file { "${home}/front": - ensure => "link", - target => $cf_front_app, - before => File[$cf_front_app], - } - - file { "/etc/systemd/system/cryptoportfolio-app.service": - mode => "0644", - owner => "root", - group => "root", - content => template("role/cryptoportfolio/cryptoportfolio-app.service.erb"), - notify => Exec["systemctl daemon-reload"], - } - - service { 'cryptoportfolio-app': - enable => true, - ensure => "running", - subscribe => [Exec["go-cryptoportfolio-app"], Exec["web-cryptoportfolio-build"]], - require => [ - File["/etc/systemd/system/cryptoportfolio-app.service"], - Postgresql::Server::Db[$pg_db] - ], - } ~> - exec { "dump $pg_db structure": - refreshonly => true, - user => $::profile::postgresql::pg_user, - group => $::profile::postgresql::pg_user, - command => "/usr/bin/pg_dump --schema-only --clean --no-publications $pg_db > /var/lib/postgres/${pg_db}.schema", - } - - archive { "${home}/front_${front_version}.tar.gz": - path => "${home}/front_${front_version}.tar.gz", - source => "https://git.immae.eu/releases/cryptoportfolio/front/front_${front_version}.tar.gz", - checksum_type => "sha256", - checksum => $front_sha256, - cleanup => false, - extract => true, - user => $user, - username => lookup("base_installation::ldap_cn"), - password => generate_password(24, $password_seed, "ldap"), - extract_path => $cf_front_app, - require => [User["$user:"], File[$cf_front_app]], - notify => [ - Exec["web-cryptoportfolio-dependencies"], - Exec["go-get-dep"], - ] - } - - # Api - file { $cf_front_app_api_conf: - owner => $user, - group => $group, - mode => "0600", - content => template("role/cryptoportfolio/api_conf.toml.erb"), - before => Exec["go-cryptoportfolio-app"], - } - - exec { "go-get-dep": - user => $user, - environment => ["HOME=${home}"], - creates => "${home}/go/bin/dep", - command => "/usr/bin/go get -u github.com/golang/dep/cmd/dep", - refreshonly => true, - } ~> - exec { "go-cryptoportfolio-dependencies": - cwd => $cf_front_app, - user => $user, - environment => ["HOME=${home}"], - command => "${home}/go/bin/dep ensure", - refreshonly => true, - } ~> - exec { "go-cryptoportfolio-app": - cwd => $cf_front_app_api_workdir, - user => $user, - environment => ["HOME=${home}"], - command => "/usr/bin/make build", - refreshonly => true, - } - - # Static pages - file { $cf_front_app_static_conf: - owner => $user, - group => $group, - mode => "0600", - content => template("role/cryptoportfolio/static_conf.env.erb"), - before => Exec["web-cryptoportfolio-build"], - } - - exec { "web-cryptoportfolio-dependencies": - cwd => "${cf_front_app}/cmd/web", - user => $user, - environment => ["HOME=${home}"], - command => "/usr/bin/make install", - refreshonly => true, - require => [Package["npm"], Package["nodejs"], Package["yarn"]] - } ~> - exec { "web-cryptoportfolio-build": - cwd => "${cf_front_app}/cmd/web", - user => $user, - environment => ["HOME=${home}"], - path => ["${cf_front_app}/cmd/web/node_modules/.bin/", "/usr/bin"], - command => "/usr/bin/make static ENV=${env}", - refreshonly => true, - } - - unless empty($webhook_url) { - exec { "front-slack-notify": - refreshonly => true, - environment => [ - "P_PROJECT=Front", - "P_WEBHOOK=${webhook_url}", - "P_VERSION=${front_version}", - "P_HOST=${web_host}", - "P_HTTPS=${web_ssl}", - ], - command => "/usr/local/bin/slack-notify", - require => File["/usr/local/bin/slack-notify"], - subscribe => [Exec["go-cryptoportfolio-app"], Exec["web-cryptoportfolio-build"]], - } - } + contain "role::cryptoportfolio::front" } } diff --git a/modules/role/manifests/cryptoportfolio/apache.pp b/modules/role/manifests/cryptoportfolio/apache.pp new file mode 100644 index 0000000..62d5447 --- /dev/null +++ b/modules/role/manifests/cryptoportfolio/apache.pp @@ -0,0 +1,17 @@ +class role::cryptoportfolio::apache inherits role::cryptoportfolio { + class { 'apache::mod::headers': } + apache::vhost { $web_host: + port => '443', + docroot => false, + manage_docroot => false, + proxy_dest => "http://localhost:8000", + request_headers => 'set X-Forwarded-Proto "https"', + ssl => true, + ssl_cert => "/etc/letsencrypt/live/$web_host/cert.pem", + ssl_key => "/etc/letsencrypt/live/$web_host/privkey.pem", + ssl_chain => "/etc/letsencrypt/live/$web_host/chain.pem", + require => Letsencrypt::Certonly[$web_host], + proxy_preserve_host => true; + default: * => $::profile::apache::apache_vhost_default; + } +} diff --git a/modules/role/manifests/cryptoportfolio/bot.pp b/modules/role/manifests/cryptoportfolio/bot.pp new file mode 100644 index 0000000..a15c779 --- /dev/null +++ b/modules/role/manifests/cryptoportfolio/bot.pp @@ -0,0 +1,101 @@ +class role::cryptoportfolio::bot inherits role::cryptoportfolio { + $password_seed = lookup("base_installation::puppet_pass_seed") + + $cf_bot_app = "${home}/bot" + $cf_bot_app_conf = "${home}/bot_config.ini" + $cf_bot_app_reports = "${home}/bot_reports" + + ensure_packages(["python", "python-pip"]) + + file { $cf_bot_app: + ensure => "directory", + mode => "0700", + owner => $user, + group => $group, + require => User["$user:"], + } + + archive { "${home}/trader_${bot_version}.tar.gz": + path => "${home}/trader_${bot_version}.tar.gz", + source => "https://git.immae.eu/releases/cryptoportfolio/trader/trader_${bot_version}.tar.gz", + checksum_type => "sha256", + checksum => $bot_sha256, + cleanup => false, + extract => true, + user => $user, + username => lookup("base_installation::ldap_cn"), + password => generate_password(24, $password_seed, "ldap"), + extract_path => $cf_bot_app, + require => [User["$user:"], File[$cf_bot_app]], + } ~> + exec { "py-cryptoportfolio-dependencies": + cwd => $cf_bot_app, + user => $user, + environment => ["HOME=${home}"], + command => "/usr/bin/make install", + require => User["$user:"], + refreshonly => true, + before => [ + File[$cf_bot_app_conf], + Cron["py-cryptoportfolio-before"], + Cron["py-cryptoportfolio-after"], + ] + } + + $pg_password = generate_password(24, $password_seed, "postgres_cryptoportfolio") + file { $cf_bot_app_conf: + owner => $user, + group => $group, + mode => "0600", + content => template("role/cryptoportfolio/bot_config.ini.erb"), + require => [ + User["$user:"], + Archive["${home}/trader_${bot_version}.tar.gz"], + ], + } + + cron { "py-cryptoportfolio-before": + ensure => present, + command => "cd $cf_bot_app ; python main.py --config $cf_bot_app_conf --before", + user => $user, + weekday => 7, # Sunday + hour => 22, + minute => 30, + environment => ["HOME=${home}","PATH=/usr/bin/"], + require => [ + File[$cf_bot_app_conf], + Archive["${home}/trader_${bot_version}.tar.gz"] + ], + } + + cron { "py-cryptoportfolio-after": + ensure => present, + command => "cd $cf_bot_app ; python main.py --config $cf_bot_app_conf --after", + user => $user, + weekday => 1, # Monday + hour => 1, + minute => 0, + environment => ["HOME=${home}","PATH=/usr/bin/"], + require => [ + File[$cf_bot_app_conf], + Archive["${home}/trader_${bot_version}.tar.gz"] + ], + } + + unless empty($webhook_url) { + exec { "bot-slack-notify": + refreshonly => true, + environment => [ + "P_PROJECT=Trader", + "P_WEBHOOK=${webhook_url}", + "P_VERSION=${bot_version}", + "P_HOST=${web_host}", + "P_HTTPS=${web_ssl}", + ], + command => "/usr/local/bin/slack-notify", + require => File["/usr/local/bin/slack-notify"], + subscribe => Exec["py-cryptoportfolio-dependencies"], + } + } +} + diff --git a/modules/role/manifests/cryptoportfolio/front.pp b/modules/role/manifests/cryptoportfolio/front.pp new file mode 100644 index 0000000..280ef8b --- /dev/null +++ b/modules/role/manifests/cryptoportfolio/front.pp @@ -0,0 +1,158 @@ +class role::cryptoportfolio::front inherits role::cryptoportfolio { + ensure_resource('exec', 'systemctl daemon-reload', { + command => '/usr/bin/systemctl daemon-reload', + refreshonly => true + }) + + $password_seed = lookup("base_installation::puppet_pass_seed") + + $cf_front_app = "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio/Front" + $cf_front_app_api_workdir = "${cf_front_app}/cmd/app" + $cf_front_app_api_bin = "${cf_front_app_api_workdir}/cryptoportfolio-app" + $cf_front_app_api_conf = "${home}/conf.toml" + $cf_front_app_api_secret = generate_password(24, $password_seed, "cryptoportfolio_api_secret") + + $cf_front_app_static_conf = "${cf_front_app}/cmd/web/env/prod.env" + + ensure_packages(["go", "npm", "nodejs", "yarn"]) + + file { [ + "${home}/go/", + "${home}/go/src", + "${home}/go/src/immae.eu", + "${home}/go/src/immae.eu/Immae", + "${home}/go/src/immae.eu/Immae/Projets", + "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies", + "${home}/go/src/immae.eu/Immae/Projets/Cryptomonnaies/Cryptoportfolio", + $cf_front_app]: + ensure => "directory", + mode => "0700", + owner => $user, + group => $group, + require => User["$user:"], + } + + file { "${home}/front": + ensure => "link", + target => $cf_front_app, + before => File[$cf_front_app], + } + + file { "/etc/systemd/system/cryptoportfolio-app.service": + mode => "0644", + owner => "root", + group => "root", + content => template("role/cryptoportfolio/cryptoportfolio-app.service.erb"), + notify => Exec["systemctl daemon-reload"], + } + + service { 'cryptoportfolio-app': + enable => true, + ensure => "running", + subscribe => [File[$cf_front_app_api_conf], Exec["go-cryptoportfolio-app"], Exec["web-cryptoportfolio-build"]], + require => [ + File["/etc/systemd/system/cryptoportfolio-app.service"], + Postgresql::Server::Db[$pg_db] + ], + } ~> + exec { "dump $pg_db structure": + refreshonly => true, + user => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + command => "/usr/bin/pg_dump --schema-only --clean --no-publications $pg_db > /var/lib/postgres/${pg_db}.schema", + } + + archive { "${home}/front_${front_version}.tar.gz": + path => "${home}/front_${front_version}.tar.gz", + source => "https://git.immae.eu/releases/cryptoportfolio/front/front_${front_version}.tar.gz", + checksum_type => "sha256", + checksum => $front_sha256, + cleanup => false, + extract => true, + user => $user, + username => lookup("base_installation::ldap_cn"), + password => generate_password(24, $password_seed, "ldap"), + extract_path => $cf_front_app, + require => [User["$user:"], File[$cf_front_app]], + notify => [ + Exec["web-cryptoportfolio-dependencies"], + Exec["go-get-dep"], + ] + } + + # Api + $pg_password = generate_password(24, $password_seed, "postgres_cryptoportfolio") + $pg_host = "${pg_hostname}:${pg_port}" + file { $cf_front_app_api_conf: + owner => $user, + group => $group, + mode => "0600", + content => template("role/cryptoportfolio/api_conf.toml.erb"), + before => Exec["go-cryptoportfolio-app"], + } + + exec { "go-get-dep": + user => $user, + environment => ["HOME=${home}"], + creates => "${home}/go/bin/dep", + command => "/usr/bin/go get -u github.com/golang/dep/cmd/dep", + refreshonly => true, + } ~> + exec { "go-cryptoportfolio-dependencies": + cwd => $cf_front_app, + user => $user, + environment => ["HOME=${home}"], + command => "${home}/go/bin/dep ensure", + refreshonly => true, + } ~> + exec { "go-cryptoportfolio-app": + cwd => $cf_front_app_api_workdir, + user => $user, + environment => ["HOME=${home}"], + command => "/usr/bin/make build", + refreshonly => true, + } + + # Static pages + file { $cf_front_app_static_conf: + owner => $user, + group => $group, + mode => "0600", + content => template("role/cryptoportfolio/static_conf.env.erb"), + before => Exec["web-cryptoportfolio-build"], + } + + exec { "web-cryptoportfolio-dependencies": + cwd => "${cf_front_app}/cmd/web", + user => $user, + environment => ["HOME=${home}"], + command => "/usr/bin/make install", + refreshonly => true, + require => [Package["npm"], Package["nodejs"], Package["yarn"]] + } ~> + exec { "web-cryptoportfolio-build": + cwd => "${cf_front_app}/cmd/web", + user => $user, + environment => ["HOME=${home}"], + path => ["${cf_front_app}/cmd/web/node_modules/.bin/", "/usr/bin"], + command => "/usr/bin/make static ENV=${env}", + refreshonly => true, + } + + unless empty($webhook_url) { + exec { "front-slack-notify": + refreshonly => true, + environment => [ + "P_PROJECT=Front", + "P_WEBHOOK=${webhook_url}", + "P_VERSION=${front_version}", + "P_HOST=${web_host}", + "P_HTTPS=${web_ssl}", + ], + command => "/usr/local/bin/slack-notify", + require => File["/usr/local/bin/slack-notify"], + subscribe => [Exec["go-cryptoportfolio-app"], Exec["web-cryptoportfolio-build"]], + } + } + +} diff --git a/modules/role/manifests/cryptoportfolio/notify.pp b/modules/role/manifests/cryptoportfolio/notify.pp new file mode 100644 index 0000000..218312c --- /dev/null +++ b/modules/role/manifests/cryptoportfolio/notify.pp @@ -0,0 +1,6 @@ +class role::cryptoportfolio::notify inherits role::cryptoportfolio { + file { "/usr/local/bin/slack-notify": + mode => "0755", + source => "puppet:///modules/role/cryptoportfolio/slack-notify.py", + } +} diff --git a/modules/role/manifests/cryptoportfolio/postgresql.pp b/modules/role/manifests/cryptoportfolio/postgresql.pp new file mode 100644 index 0000000..cc4d2a9 --- /dev/null +++ b/modules/role/manifests/cryptoportfolio/postgresql.pp @@ -0,0 +1,116 @@ +class role::cryptoportfolio::postgresql inherits role::cryptoportfolio { + $password_seed = lookup("base_installation::puppet_pass_seed") + + $pg_password = generate_password(24, $password_seed, "postgres_cryptoportfolio") + $pg_replication_password = generate_password(24, $password_seed, "postgres_cryptoportfolio_replication") + + file { "/var/lib/postgres/data/certs": + ensure => directory, + mode => "0700", + owner => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + require => File["/var/lib/postgres"], + } + + file { "/var/lib/postgres/data/certs/cert.pem": + source => "file:///etc/letsencrypt/live/$web_host/cert.pem", + mode => "0600", + links => "follow", + owner => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] + } + + file { "/var/lib/postgres/data/certs/privkey.pem": + source => "file:///etc/letsencrypt/live/$web_host/privkey.pem", + mode => "0600", + links => "follow", + owner => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] + } + + postgresql::server::config_entry { "wal_level": + value => "logical", + } + + postgresql::server::config_entry { "ssl": + value => "on", + require => Letsencrypt::Certonly[$web_host], + } + + postgresql::server::config_entry { "ssl_cert_file": + value => "/var/lib/postgres/data/certs/cert.pem", + require => Letsencrypt::Certonly[$web_host], + } + + postgresql::server::config_entry { "ssl_key_file": + value => "/var/lib/postgres/data/certs/privkey.pem", + require => Letsencrypt::Certonly[$web_host], + } + + postgresql::server::db { $pg_db: + user => $pg_user, + password => postgresql_password($pg_user, $pg_password), + } + -> + postgresql_psql { "CREATE PUBLICATION ${pg_db}_publication FOR ALL TABLES": + db => $pg_db, + unless => "SELECT 1 FROM pg_catalog.pg_publication WHERE pubname = '${pg_db}_publication'", + } + -> + postgresql::server::role { $pg_user_replication: + db => $pg_db, + replication => true, + password_hash => postgresql_password($pg_user_replication, $pg_replication_password), + } + -> + postgresql::server::database_grant { $pg_user_replication: + db => $pg_db, + privilege => "CONNECT", + role => $pg_user_replication, + } + -> + postgresql::server::grant { "all tables in schema:public:$pg_user_replication": + db => $pg_db, + role => $pg_user_replication, + privilege => "SELECT", + object_type => "ALL TABLES IN SCHEMA", + object_name => "public", + } + -> + postgresql::server::grant { "all sequences in schema:public:$pg_user_replication": + db => $pg_db, + role => $pg_user_replication, + privilege => "SELECT", + object_type => "ALL SEQUENCES IN SCHEMA", + object_name => "public", + } + + postgresql::server::pg_hba_rule { 'allow localhost TCP access to cryptoportfolio user': + type => 'host', + database => $pg_db, + user => $pg_user, + address => '127.0.0.1/32', + auth_method => 'md5', + order => "05-01", + } + postgresql::server::pg_hba_rule { 'allow localhost ip6 TCP access to cryptoportfolio user': + type => 'host', + database => $pg_db, + user => $pg_user, + address => '::1/128', + auth_method => 'md5', + order => "05-01", + } + + postgresql::server::pg_hba_rule { 'allow TCP access to replication user from immae.eu': + type => 'hostssl', + database => $pg_db, + user => $pg_user_replication, + address => 'immae.eu', + auth_method => 'md5', + order => "05-01", + } + +} -- cgit v1.2.3 From c15f2234474ff8a8266e26856702b3c561050667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Fri, 16 Mar 2018 00:22:38 +0100 Subject: Move scripts to separate chunks --- bin/install_script_ovh_cloud_instance.sh | 88 ++----------- bin/install_script_ovh_vps_ssd.sh | 136 ++------------------- .../templates/puppet/host_ldap.info.erb | 5 +- scripts/arch_install_script.sh | 13 ++ scripts/arch_puppet_configuration_script.sh | 8 ++ .../arch_host_puppet_configuration_script.sh | 11 ++ scripts/ovh_cloud_instance/arch_host_script.sh | 50 ++++++++ scripts/ovh_vps_ssd/arch_chroot_script.sh | 29 +++++ .../arch_host_puppet_configuration_script.sh | 13 ++ scripts/ovh_vps_ssd/arch_host_script.sh | 53 ++++++++ scripts/send_and_run.tcl | 31 +++++ 11 files changed, 232 insertions(+), 205 deletions(-) create mode 100755 scripts/arch_install_script.sh create mode 100755 scripts/arch_puppet_configuration_script.sh create mode 100755 scripts/ovh_cloud_instance/arch_host_puppet_configuration_script.sh create mode 100755 scripts/ovh_cloud_instance/arch_host_script.sh create mode 100755 scripts/ovh_vps_ssd/arch_chroot_script.sh create mode 100755 scripts/ovh_vps_ssd/arch_host_puppet_configuration_script.sh create mode 100755 scripts/ovh_vps_ssd/arch_host_script.sh create mode 100755 scripts/send_and_run.tcl diff --git a/bin/install_script_ovh_cloud_instance.sh b/bin/install_script_ovh_cloud_instance.sh index 26e410e..e1c8a54 100755 --- a/bin/install_script_ovh_cloud_instance.sh +++ b/bin/install_script_ovh_cloud_instance.sh @@ -62,6 +62,7 @@ done DIRECTORY=$(cd `dirname $0` && pwd) PYTHON_DIRECTORY="$DIRECTORY/../python" +SCRIPTS="$DIRECTORY/../scripts" if [ -z "$instance_id" ]; then read -p "Id de l'instance : " instance_id @@ -82,10 +83,10 @@ if [ -z "$no_reboot" -a -z "$no_reboot_start" ]; then fi ARCH_DIR=`mktemp -d` -ARCH_HOST_SCRIPT="$ARCH_DIR/arch_host_script.sh" -ARCH_INSTALL_SCRIPT="$ARCH_DIR/arch_install_script.sh" -ARCH_HOST_PUPPET_CONFIGURATION_SCRIPT="$ARCH_DIR/arch_host_puppet_configuration_script.sh" -ARCH_PUPPET_CONFIGURATION_SCRIPT="$ARCH_DIR/arch_puppet_configuration_script.sh" +ARCH_HOST_SCRIPT="$SCRIPTS/ovh_cloud_instance/arch_host_script.sh" +ARCH_INSTALL_SCRIPT="$SCRIPTS/arch_install_script.sh" +ARCH_HOST_PUPPET_CONFIGURATION_SCRIPT="$SCRIPTS/ovh_cloud_instance/arch_host_puppet_configuration_script.sh" +ARCH_PUPPET_CONFIGURATION_SCRIPT="$SCRIPTS/arch_puppet_configuration_script.sh" ARCH_PUPPET_INITIAL_CONFIGURATION="$ARCH_DIR/puppet_variables.json" trap "rm -rf $ARCH_DIR" EXIT @@ -94,86 +95,17 @@ trap "rm -rf $ARCH_DIR" EXIT python $PYTHON_DIRECTORY/get_initial_configuration_cloud_instance.py $instance_id > $ARCH_PUPPET_INITIAL_CONFIGURATION host_address=$(python $PYTHON_DIRECTORY/get_initial_configuration_cloud_instance.py $instance_id | jq -r '.ips.v4.ipAddress') -cat > $ARCH_HOST_SCRIPT < $ARCH_INSTALL_SCRIPT < $ARCH_PUPPET_CONFIGURATION_SCRIPT < $ARCH_HOST_PUPPET_CONFIGURATION_SCRIPT < $ARCH_PUPPET_INITIAL_CONFIGURATION -cat > $ARCH_HOST_SCRIPT < /tmp/root.x86_64/etc/pacman.d/mirrorlist - -DEVICE_STR=\$(cat /proc/mounts | grep "/dev/[sv]d.. /mnt/") -DEVICE=\$(echo "\$DEVICE_STR" | cut -d' ' -f1) -MOUNTPOINT=\$(echo "\$DEVICE_STR" | cut -d' ' -f2) - -umount "\$DEVICE" -UUID=\$(lsblk -rno UUID "\$DEVICE") - -echo "\$UUID" > /tmp/root.x86_64/device_uuid - -cp /tmp/arch_chroot_script.sh /tmp/root.x86_64/ - -/tmp/root.x86_64/bin/arch-chroot /tmp/root.x86_64/ /arch_chroot_script.sh - -mount "\$DEVICE" - -cp /tmp/arch_install_script.sh "\$MOUNTPOINT/root/" -cp /tmp/puppet_variables.json "\$MOUNTPOINT/root/" - -/tmp/root.x86_64/bin/arch-chroot "\$MOUNTPOINT" /root/arch_install_script.sh -EOF - - -cat > $ARCH_CHROOT_SCRIPT < /mnt/etc/fstab - -umount /mnt -EOF - -cat > $ARCH_INSTALL_SCRIPT < $ARCH_PUPPET_CONFIGURATION_SCRIPT < $ARCH_HOST_PUPPET_CONFIGURATION_SCRIPT <" -W << 'EOF' dn: <%= @ldap_dn %> cn: <%= @ldap_cn %> @@ -14,12 +14,13 @@ objectclass: ipHost environment: <%= @environment %> userpassword: {SSHA}<%= Base64.encode64(Digest::SHA1.digest(@ldap_password+@ssha_ldap_seed)+@ssha_ldap_seed).chomp! %> EOF -#### Or modify an existing entry: +#### Or modify an existing entry: ldapmodify -D "cn=root,<%= @ldap_base %>" -W << 'EOF' dn: <%= @ldap_dn %> changetype: modify replace: userPassword userpassword: {SSHA}<%= Base64.encode64(Digest::SHA1.digest(@ldap_password+@ssha_ldap_seed)+@ssha_ldap_seed).chomp! %> +- replace: environment environment: <%= @environment %> <%- unless @ips.empty? -%> diff --git a/scripts/arch_install_script.sh b/scripts/arch_install_script.sh new file mode 100755 index 0000000..21e202f --- /dev/null +++ b/scripts/arch_install_script.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +git_branch="$1" +environment="$2" +CODE_PATH="/etc/puppetlabs/code" + +rm -rf $CODE_PATH + +git clone -b $git_branch --recursive https://git.immae.eu/perso/Immae/Projets/Puppet.git $CODE_PATH +puppet apply --environment $environment --tags base_installation --test $CODE_PATH/manifests/site.pp +# The password seed requires puppet to be run twice +puppet apply --environment $environment --tags base_installation --test $CODE_PATH/manifests/site.pp + diff --git a/scripts/arch_puppet_configuration_script.sh b/scripts/arch_puppet_configuration_script.sh new file mode 100755 index 0000000..caf8987 --- /dev/null +++ b/scripts/arch_puppet_configuration_script.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +git_branch="$1" +environment="$2" +CODE_PATH="/etc/puppetlabs/code" + +puppet apply --environment $environment --tags base_installation --test $CODE_PATH/manifests/site.pp + diff --git a/scripts/ovh_cloud_instance/arch_host_puppet_configuration_script.sh b/scripts/ovh_cloud_instance/arch_host_puppet_configuration_script.sh new file mode 100755 index 0000000..b58e255 --- /dev/null +++ b/scripts/ovh_cloud_instance/arch_host_puppet_configuration_script.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +git_branch="$1" +environment="$2" + +MOUNTPOINT=/mnt + +sudo cp /tmp/arch_puppet_configuration_script.sh "$MOUNTPOINT/root/" + +sudo arch-chroot "$MOUNTPOINT" /root/arch_puppet_configuration_script.sh "$git_branch" "$environment" + diff --git a/scripts/ovh_cloud_instance/arch_host_script.sh b/scripts/ovh_cloud_instance/arch_host_script.sh new file mode 100755 index 0000000..c188514 --- /dev/null +++ b/scripts/ovh_cloud_instance/arch_host_script.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -e + +git_branch="$1" +environment="$2" + +# Randomizer +sudo haveged & +# /Randomizer + +# Prepare an arch chroot +sudo pacman -Sy --noconfirm arch-install-scripts +# /Prepare an arch chroot + +# Prepare device information +DEVICE=/dev/vdb1 +MOUNTPOINT=/mnt + +UUID=$(lsblk -rno UUID "$DEVICE") +PART="/dev/disk/by-uuid/$UUID" +# /Prepare device information + +# Install very basic system (base git puppet) +# mkfs.ext4 -F -U "$UUID" "$DEVICE" +sudo mount "$DEVICE" /mnt + +##### FIXME: mkfs.ext4 would be better #### +for i in /mnt/*; do + if [ "$i" = "/mnt/boot" ]; then + # keep /boot/grub + sudo rm -f $i/* + else + sudo rm -rf $i + fi +done +##### /FIXME #### + +sudo pacstrap /mnt base git puppet + +echo "$PART / auto defaults 0 1" | sudo tee /mnt/etc/fstab +# /Install very basic system + +# Install rest of system (via puppet) +sudo cp /tmp/arch_install_script.sh "$MOUNTPOINT/root/" +sudo cp /tmp/puppet_variables.json "$MOUNTPOINT/root/" + +sudo arch-chroot "$MOUNTPOINT" /root/arch_install_script.sh "$git_branch" "$environment" +# /Install rest of system + diff --git a/scripts/ovh_vps_ssd/arch_chroot_script.sh b/scripts/ovh_vps_ssd/arch_chroot_script.sh new file mode 100755 index 0000000..9dc5c46 --- /dev/null +++ b/scripts/ovh_vps_ssd/arch_chroot_script.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +pacman-key --init +pacman-key --populate archlinux + +UUID=$(cat /device_uuid) +PART="/dev/disk/by-uuid/$UUID" +DEVICE=$(realpath "$PART") + +# mkfs.ext4 -F -U "$UUID" "$DEVICE" +mount "$DEVICE" /mnt + +##### FIXME: mkfs.ext4 would be better #### +for i in /mnt/*; do + if [ "$i" = "/mnt/boot" ]; then + # keep /boot/grub + rm -f $i/* + else + rm -rf $i + fi +done +##### /FIXME #### + +pacstrap /mnt base git puppet + +echo "$PART / auto defaults 0 1" > /mnt/etc/fstab + +umount /mnt + diff --git a/scripts/ovh_vps_ssd/arch_host_puppet_configuration_script.sh b/scripts/ovh_vps_ssd/arch_host_puppet_configuration_script.sh new file mode 100755 index 0000000..dab8fac --- /dev/null +++ b/scripts/ovh_vps_ssd/arch_host_puppet_configuration_script.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +git_branch="$1" +environment="$2" + +DEVICE_STR=$(cat /proc/mounts | grep "/dev/[sv]d.. /mnt/") +DEVICE=$(echo "$DEVICE_STR" | cut -d' ' -f1) +MOUNTPOINT=$(echo "$DEVICE_STR" | cut -d' ' -f2) + +cp /tmp/arch_puppet_configuration_script.sh "$MOUNTPOINT/root/" + +/tmp/root.x86_64/bin/arch-chroot "$MOUNTPOINT" /root/arch_puppet_configuration_script.sh "$git_branch" "$environment" + diff --git a/scripts/ovh_vps_ssd/arch_host_script.sh b/scripts/ovh_vps_ssd/arch_host_script.sh new file mode 100755 index 0000000..8eecae4 --- /dev/null +++ b/scripts/ovh_vps_ssd/arch_host_script.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -e + +git_branch="$1" +environment="$2" + +# Randomizer +apt-get update +apt-get install -y haveged +haveged & +# /Randomizer + +# Prepare an arch chroot +cd /tmp + +LATEST=$(curl -L https://mirrors.kernel.org/archlinux/iso/latest/sha1sums.txt | grep "bootstrap" | head -n1) +SHA1=$(echo "$LATEST" | cut -d' ' -f1) +NAME=$(echo "$LATEST" | cut -d' ' -f3) + +curl -L -O "https://mirrors.kernel.org/archlinux/iso/latest/$NAME" + +tar -xzf "$NAME" + +echo 'Server = http://archlinux.mirrors.ovh.net/archlinux/$repo/os/$arch' > /tmp/root.x86_64/etc/pacman.d/mirrorlist +# /Prepare an arch chroot + +# Prepare device information (not available in chroot) +DEVICE_STR=$(cat /proc/mounts | grep "/dev/[sv]d.. /mnt/") +DEVICE=$(echo "$DEVICE_STR" | cut -d' ' -f1) +MOUNTPOINT=$(echo "$DEVICE_STR" | cut -d' ' -f2) + +umount "$DEVICE" +UUID=$(lsblk -rno UUID "$DEVICE") + +echo "$UUID" > /tmp/root.x86_64/device_uuid +# /Prepare device information + +# Install very basic system via chroot (base git puppet) +cp /tmp/arch_chroot_script.sh /tmp/root.x86_64/ + +/tmp/root.x86_64/bin/arch-chroot /tmp/root.x86_64/ /arch_chroot_script.sh +# /Install very basic system via chroot + +# Mount and install rest of system (via puppet) +mount "$DEVICE" + +cp /tmp/arch_install_script.sh "$MOUNTPOINT/root/" +cp /tmp/puppet_variables.json "$MOUNTPOINT/root/" + +/tmp/root.x86_64/bin/arch-chroot "$MOUNTPOINT" /root/arch_install_script.sh "$git_branch" "$environment" +# /Mount and install rest of system + diff --git a/scripts/send_and_run.tcl b/scripts/send_and_run.tcl new file mode 100755 index 0000000..a087a2d --- /dev/null +++ b/scripts/send_and_run.tcl @@ -0,0 +1,31 @@ +#!/usr/bin/expect -f +set dest [lindex $argv 0] +set password [lindex $argv 1] +set git_branch [lindex $argv 2] +set environment [lindex $argv 3] +set script [lindex $argv 4] +set files [lrange $argv 4 end] + +if {$password == ""} { + set ask_password 0 +} else { + set ask_password 1 +} + +set scriptname [file tail $script] + +set sshopts [split "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no"] + +set timeout -1 +spawn scp {*}$sshopts {*}$files $dest:/tmp +if {$ask_password} { + expect "assword:" + send "$password\n" +} +expect eof +spawn ssh {*}$sshopts $dest /tmp/$scriptname $git_branch $environment +if {$ask_password} { + expect "assword:" + send "$password\n" +} +expect eof -- cgit v1.2.3 From 69da835d04e741f4e85da3c473ba86c8801931fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Fri, 16 Mar 2018 19:14:29 +0100 Subject: Merge install scripts --- bin/install_script.sh | 155 +++++++++++++++++++++ bin/install_script_ovh_cloud_instance.sh | 114 --------------- bin/install_script_ovh_vps_ssd.sh | 123 ---------------- python/buy_ovh_vps_ssd.py | 63 +++++++++ python/buy_vps_server.py | 63 --------- python/get_initial_configuration.py | 37 ----- python/get_initial_configuration_cloud_instance.py | 41 ------ ...get_initial_configuration_ovh_cloud_instance.py | 30 ++++ python/get_initial_configuration_ovh_vps_ssd.py | 37 +++++ python/list_servers.py | 13 +- python/ovh_helper.py | 17 +++ python/reboot_cloud_instance.py | 39 ------ python/reboot_ovh_cloud_instance.py | 28 ++++ python/reboot_ovh_vps_ssd.py | 40 ++++++ python/reboot_vps_server.py | 40 ------ python/reinstall_cloud_instance.py | 57 -------- python/reinstall_ovh_cloud_instance.py | 46 ++++++ python/reinstall_ovh_vps_ssd.py | 58 ++++++++ python/reinstall_vps_server.py | 58 -------- scripts/ovh_cloud_instance/arch_host_script.sh | 2 +- scripts/send_and_run.tcl | 2 +- 21 files changed, 488 insertions(+), 575 deletions(-) create mode 100755 bin/install_script.sh delete mode 100755 bin/install_script_ovh_cloud_instance.sh delete mode 100755 bin/install_script_ovh_vps_ssd.sh create mode 100644 python/buy_ovh_vps_ssd.py delete mode 100644 python/buy_vps_server.py delete mode 100644 python/get_initial_configuration.py delete mode 100644 python/get_initial_configuration_cloud_instance.py create mode 100644 python/get_initial_configuration_ovh_cloud_instance.py create mode 100644 python/get_initial_configuration_ovh_vps_ssd.py delete mode 100644 python/reboot_cloud_instance.py create mode 100644 python/reboot_ovh_cloud_instance.py create mode 100644 python/reboot_ovh_vps_ssd.py delete mode 100644 python/reboot_vps_server.py delete mode 100644 python/reinstall_cloud_instance.py create mode 100644 python/reinstall_ovh_cloud_instance.py create mode 100644 python/reinstall_ovh_vps_ssd.py delete mode 100644 python/reinstall_vps_server.py diff --git a/bin/install_script.sh b/bin/install_script.sh new file mode 100755 index 0000000..bd7f38b --- /dev/null +++ b/bin/install_script.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +usage() { +cat < $ARCH_PUPPET_INITIAL_CONFIGURATION +host_address=$(python $PYTHON_DIRECTORY/get_initial_configuration_$T.py $host_id | jq -r '.ips.v4.ipAddress') + +dest="$host_user@$host_address" +files="$ARCH_HOST_SCRIPT $ARCH_CHROOT_SCRIPT $ARCH_PUPPET_INITIAL_CONFIGURATION $ARCH_INSTALL_SCRIPT" + +$SCRIPTS/send_and_run.tcl "$dest" "$password" "$git_branch" "$environment" $files + +### Role specific stage +read -p "Press key when LDAP is configured" i + +files="$ARCH_HOST_PUPPET_CONFIGURATION_SCRIPT $ARCH_PUPPET_CONFIGURATION_SCRIPT" + +$SCRIPTS/send_and_run.tcl "$dest" "$password" "$git_branch" "$environment" $files + +### Installation finished +if [ -z "$no_reboot" -a -z "$no_reboot_end" ]; then + echo "Rebooting" + python $PYTHON_DIRECTORY/reboot_$T.py --local "$host_id" +fi diff --git a/bin/install_script_ovh_cloud_instance.sh b/bin/install_script_ovh_cloud_instance.sh deleted file mode 100755 index e1c8a54..0000000 --- a/bin/install_script_ovh_cloud_instance.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash - -usage() { -cat < $ARCH_PUPPET_INITIAL_CONFIGURATION -host_address=$(python $PYTHON_DIRECTORY/get_initial_configuration_cloud_instance.py $instance_id | jq -r '.ips.v4.ipAddress') - -dest="$host_user@$host_address" -files="$ARCH_HOST_SCRIPT $ARCH_PUPPET_INITIAL_CONFIGURATION $ARCH_INSTALL_SCRIPT" - -$SCRIPTS/send_and_run.tcl "$dest" "" "$git_branch" "$environment" $files - -### Role specific stage -read -p "Press key when LDAP is configured" i - -files="$ARCH_HOST_PUPPET_CONFIGURATION_SCRIPT $ARCH_PUPPET_CONFIGURATION_SCRIPT" - -$SCRIPTS/send_and_run.tcl "$dest" "" "$git_branch" "$environment" $files - -### Installation finished -if [ -z "$no_reboot" -a -z "$no_reboot_end" ]; then - echo "Rebooting" - python $PYTHON_DIRECTORY/reboot_cloud_instance.py --local "$instance_id" -fi diff --git a/bin/install_script_ovh_vps_ssd.sh b/bin/install_script_ovh_vps_ssd.sh deleted file mode 100755 index b5b0efb..0000000 --- a/bin/install_script_ovh_vps_ssd.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash - -usage() { -cat < $ARCH_PUPPET_INITIAL_CONFIGURATION - -dest="$host_user@$vps_name" -files="$ARCH_HOST_SCRIPT $ARCH_CHROOT_SCRIPT $ARCH_PUPPET_INITIAL_CONFIGURATION $ARCH_INSTALL_SCRIPT" - -$SCRIPTS/send_and_run.tcl "$dest" "$password" "$git_branch" "$environment" $files - -### Role specific stage -read -p "Press key when LDAP is configured" i - -files="$ARCH_HOST_PUPPET_CONFIGURATION_SCRIPT $ARCH_PUPPET_CONFIGURATION_SCRIPT" - -$SCRIPTS/send_and_run.tcl "$dest" "$password" "$git_branch" "$environment" $files - -### Installation finished -if [ -z "$no_reboot" -a -z "$no_reboot_end" ]; then - echo "Rebooting" - python $PYTHON_DIRECTORY/reboot_vps_server.py --local "$vps_name" -fi diff --git a/python/buy_ovh_vps_ssd.py b/python/buy_ovh_vps_ssd.py new file mode 100644 index 0000000..44ae786 --- /dev/null +++ b/python/buy_ovh_vps_ssd.py @@ -0,0 +1,63 @@ +# -*- encoding: utf-8 -*- +import json +try: + from ovh import ovh +except ImportError: + # In case it's installed globally + import ovh +import sys +import ovh_helper + +# Credentials are stored in ~/.ovh.conf +# See ovh/README.rst +client = ovh.Client() + +# Create a cart +cart = client.post('/order/cart', ovhSubsidiary="FR") +cart_id = cart["cartId"] + +# Assign it to my user +client.post('/order/cart/{}/assign'.format(cart_id)) + +# list of vps: +# client.get('/order/cart/{}/vps'.format(cart_id)) +item = client.post('/order/cart/{}/vps'.format(cart_id), + duration="P1M", + planCode="vps_ssd_model1", + pricingMode="default", + quantity=1) +item_id = item["itemId"] + +# Datacenter, OS, auto-renew +client.post('/order/cart/{}/item/{}/configuration'.format(cart_id, item_id), + label="vps_ssd_datacenter", + value="gra") +client.post('/order/cart/{}/item/{}/configuration'.format(cart_id, item_id), + label="vps_ssd_os", + value="linux--archlinux--64--en") +client.post('/order/cart/{}/item/{}/configuration'.format(cart_id, item_id), + label="AUTO_RENEW_VPS", + value=False) + +#summary = client.get('/order/cart/{}/summary'.format(cart_id)) +#checkout = client.get('/order/cart/{}/checkout'.format(cart_id)) + +# Checkout +order = client.post('/order/cart/{}/checkout'.format(cart_id), + waiveRetractationPeriod=True) +order_id = order["orderId"] +print(order_id) + +payment_mean = client.get('/me/order/{}/availableRegisteredPaymentMean'.format(order_id))[0]['paymentMean'] +payment_mean_id = client.get('/me/paymentMean/{}'.format(payment_mean))[0] + +payment_mean_status = client.get('/me/paymentMean/{}/{}'.format(payment_mean, payment_mean_id)) + +if payment_mean_status["state"] != "valid": + raise "Bouh" + +paid_order = client.post('/me/order/{}/payWithRegisteredPaymentMean'.format(order_id), + paymentMean=payment_mean, paymentMeanId=payment_mean_id) + +if 'paymentDate' in paid_order: + print("successful") diff --git a/python/buy_vps_server.py b/python/buy_vps_server.py deleted file mode 100644 index 44ae786..0000000 --- a/python/buy_vps_server.py +++ /dev/null @@ -1,63 +0,0 @@ -# -*- encoding: utf-8 -*- -import json -try: - from ovh import ovh -except ImportError: - # In case it's installed globally - import ovh -import sys -import ovh_helper - -# Credentials are stored in ~/.ovh.conf -# See ovh/README.rst -client = ovh.Client() - -# Create a cart -cart = client.post('/order/cart', ovhSubsidiary="FR") -cart_id = cart["cartId"] - -# Assign it to my user -client.post('/order/cart/{}/assign'.format(cart_id)) - -# list of vps: -# client.get('/order/cart/{}/vps'.format(cart_id)) -item = client.post('/order/cart/{}/vps'.format(cart_id), - duration="P1M", - planCode="vps_ssd_model1", - pricingMode="default", - quantity=1) -item_id = item["itemId"] - -# Datacenter, OS, auto-renew -client.post('/order/cart/{}/item/{}/configuration'.format(cart_id, item_id), - label="vps_ssd_datacenter", - value="gra") -client.post('/order/cart/{}/item/{}/configuration'.format(cart_id, item_id), - label="vps_ssd_os", - value="linux--archlinux--64--en") -client.post('/order/cart/{}/item/{}/configuration'.format(cart_id, item_id), - label="AUTO_RENEW_VPS", - value=False) - -#summary = client.get('/order/cart/{}/summary'.format(cart_id)) -#checkout = client.get('/order/cart/{}/checkout'.format(cart_id)) - -# Checkout -order = client.post('/order/cart/{}/checkout'.format(cart_id), - waiveRetractationPeriod=True) -order_id = order["orderId"] -print(order_id) - -payment_mean = client.get('/me/order/{}/availableRegisteredPaymentMean'.format(order_id))[0]['paymentMean'] -payment_mean_id = client.get('/me/paymentMean/{}'.format(payment_mean))[0] - -payment_mean_status = client.get('/me/paymentMean/{}/{}'.format(payment_mean, payment_mean_id)) - -if payment_mean_status["state"] != "valid": - raise "Bouh" - -paid_order = client.post('/me/order/{}/payWithRegisteredPaymentMean'.format(order_id), - paymentMean=payment_mean, paymentMeanId=payment_mean_id) - -if 'paymentDate' in paid_order: - print("successful") diff --git a/python/get_initial_configuration.py b/python/get_initial_configuration.py deleted file mode 100644 index 0c6f698..0000000 --- a/python/get_initial_configuration.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- encoding: utf-8 -*- -import json -try: - from ovh import ovh -except ImportError: - # In case it's installed globally - import ovh -import sys - -infos = {} - -# Credentials are stored in ~/.ovh.conf -# See ovh/README.rst -client = ovh.Client() - -vps_list = client.get('/vps/') -if sys.argv[-1] in vps_list: - vps = sys.argv[-1] -else: - print("VPS not in list:") - for vps in vps_list: - print(vps) - sys.exit(1) - -ips = client.get('/vps/{}/ips'.format(vps)) - -infos["ips"] = {} -for ip in ips: - ip_infos = client.get('/vps/{}/ips/{}'.format(vps, ip)) - - if ip_infos["version"] == "v4": - infos["ips"]["v4"] = ip_infos - else: - infos["ips"]["v6"] = ip_infos - infos["ips"]["v6"]["mask"] = 128 - -print(json.dumps(infos)) diff --git a/python/get_initial_configuration_cloud_instance.py b/python/get_initial_configuration_cloud_instance.py deleted file mode 100644 index 4157716..0000000 --- a/python/get_initial_configuration_cloud_instance.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- encoding: utf-8 -*- -import json -try: - from ovh import ovh -except ImportError: - # In case it's installed globally - import ovh -import sys - -infos = {} - -# Credentials are stored in ~/.ovh.conf -# See ovh/README.rst -client = ovh.Client() - -projects_list = client.get('/cloud/project/') -if len(projects_list) > 1: - print("More than one project is not supported, taking the first one") -project = projects_list[0] -instances_list = client.get('/cloud/project/{}/instance'.format(project)) -instances = dict(map(lambda x: (x["id"], x), instances_list)) -if sys.argv[-1] in instances: - instance = instances[sys.argv[-1]] -else: - print("Instance not in list:") - for instance in instances_list: - print("{}: {}".format(instance["name"], instance["id"])) - sys.exit(1) - -infos["ips"] = {} -for ip_infos in instance["ipAddresses"]: - ip_infos["ipAddress"] = ip_infos.pop("ip") - ip_infos["gateway"] = ip_infos.pop("gatewayIp") - - if ip_infos["version"] == 4: - infos["ips"]["v4"] = ip_infos - else: - infos["ips"]["v6"] = ip_infos - infos["ips"]["v6"]["mask"] = 128 - -print(json.dumps(infos)) diff --git a/python/get_initial_configuration_ovh_cloud_instance.py b/python/get_initial_configuration_ovh_cloud_instance.py new file mode 100644 index 0000000..844373c --- /dev/null +++ b/python/get_initial_configuration_ovh_cloud_instance.py @@ -0,0 +1,30 @@ +# -*- encoding: utf-8 -*- +import json +try: + from ovh import ovh +except ImportError: + # In case it's installed globally + import ovh +import sys +from ovh_helper import find_cloud_instance + +infos = {} + +# Credentials are stored in ~/.ovh.conf +# See ovh/README.rst +client = ovh.Client() + +_, instance = find_cloud_instance(client, sys.argv[-1]) + +infos["ips"] = {} +for ip_infos in instance["ipAddresses"]: + ip_infos["ipAddress"] = ip_infos.pop("ip") + ip_infos["gateway"] = ip_infos.pop("gatewayIp") + + if ip_infos["version"] == 4: + infos["ips"]["v4"] = ip_infos + else: + infos["ips"]["v6"] = ip_infos + infos["ips"]["v6"]["mask"] = 128 + +print(json.dumps(infos)) diff --git a/python/get_initial_configuration_ovh_vps_ssd.py b/python/get_initial_configuration_ovh_vps_ssd.py new file mode 100644 index 0000000..0c6f698 --- /dev/null +++ b/python/get_initial_configuration_ovh_vps_ssd.py @@ -0,0 +1,37 @@ +# -*- encoding: utf-8 -*- +import json +try: + from ovh import ovh +except ImportError: + # In case it's installed globally + import ovh +import sys + +infos = {} + +# Credentials are stored in ~/.ovh.conf +# See ovh/README.rst +client = ovh.Client() + +vps_list = client.get('/vps/') +if sys.argv[-1] in vps_list: + vps = sys.argv[-1] +else: + print("VPS not in list:") + for vps in vps_list: + print(vps) + sys.exit(1) + +ips = client.get('/vps/{}/ips'.format(vps)) + +infos["ips"] = {} +for ip in ips: + ip_infos = client.get('/vps/{}/ips/{}'.format(vps, ip)) + + if ip_infos["version"] == "v4": + infos["ips"]["v4"] = ip_infos + else: + infos["ips"]["v6"] = ip_infos + infos["ips"]["v6"]["mask"] = 128 + +print(json.dumps(infos)) diff --git a/python/list_servers.py b/python/list_servers.py index 9b8bc64..e7bd2af 100644 --- a/python/list_servers.py +++ b/python/list_servers.py @@ -6,7 +6,18 @@ except ImportError: client = ovh.Client() +print("OVH cloud instances:") +projects_list = client.get('/cloud/project/') +for project_id in projects_list: + project = client.get('/cloud/project/{}'.format(project_id)) + print("\t{}:".format(project["description"])) + instances_list = client.get('/cloud/project/{}/instance'.format(project_id)) + for instance in instances_list: + print("\t\t{}: {}".format(instance["name"], instance["id"])) + vps_list = client.get('/vps/') +print("OVH VPS SSD servers:") for vps in vps_list: - print(vps) + print("\t{}".format(vps)) + diff --git a/python/ovh_helper.py b/python/ovh_helper.py index a49a245..19834ae 100644 --- a/python/ovh_helper.py +++ b/python/ovh_helper.py @@ -1,4 +1,5 @@ import time +import sys def show_progress(client, vps, task_type): running_task_id = client.get("/vps/{}/tasks?type={}".format(vps, task_type))[0] @@ -17,3 +18,19 @@ def show_progress(client, vps, task_type): time.sleep(3) print("\rFinished") + +def find_cloud_instance(client, instance_id): + projects_list = client.get('/cloud/project/') + instances_list = [] + for project in projects_list: + instances_list += list(map(lambda x: [project, x], + client.get('/cloud/project/{}/instance'.format(project)))) + instances = dict(map(lambda x: (x[1]["id"], x), instances_list)) + if instance_id in instances: + project_instance = instances[instance_id] + else: + print("Instance not in list:") + for instance in instances_list: + print("{}: {}".format(instance[1]["name"], instance[1]["id"])) + sys.exit(1) + return project_instance diff --git a/python/reboot_cloud_instance.py b/python/reboot_cloud_instance.py deleted file mode 100644 index b90f488..0000000 --- a/python/reboot_cloud_instance.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- encoding: utf-8 -*- -import json -try: - from ovh import ovh -except ImportError: - # In case it's installed globally - import ovh -import sys - -# Credentials are stored in ~/.ovh.conf -# See ovh/README.rst -client = ovh.Client() - -projects_list = client.get('/cloud/project/') -if len(projects_list) > 1: - print("More than one project is not supported, taking the first one") -project = projects_list[0] -instances_list = client.get('/cloud/project/{}/instance'.format(project)) -instances = dict(map(lambda x: (x["id"], x), instances_list)) -if sys.argv[-1] in instances: - instance = instances[sys.argv[-1]] -else: - print("Instance not in list:") - for instance in instances_list: - print("{}: {}".format(instance["name"], instance["id"])) - sys.exit(1) - -if "--rescue" in sys.argv: - netboot_mode="rescue" -elif "--local" in sys.argv: - netboot_mode="local" -else: - netboot_mode=None - -if netboot_mode is not None: - result = client.post("/cloud/project/{}/instance/{}/rescueMode".format(project, - instance["id"]), imageId=instance["imageId"], rescue=(netboot_mode == "rescue")) - print(result) - diff --git a/python/reboot_ovh_cloud_instance.py b/python/reboot_ovh_cloud_instance.py new file mode 100644 index 0000000..de20c07 --- /dev/null +++ b/python/reboot_ovh_cloud_instance.py @@ -0,0 +1,28 @@ +# -*- encoding: utf-8 -*- +import json +try: + from ovh import ovh +except ImportError: + # In case it's installed globally + import ovh +import sys +from ovh_helper import find_cloud_instance + +# Credentials are stored in ~/.ovh.conf +# See ovh/README.rst +client = ovh.Client() + +project, instance = find_cloud_instance(client, sys.argv[-1]) + +if "--rescue" in sys.argv: + netboot_mode="rescue" +elif "--local" in sys.argv: + netboot_mode="local" +else: + netboot_mode=None + +if netboot_mode is not None: + result = client.post("/cloud/project/{}/instance/{}/rescueMode".format(project, + instance["id"]), imageId=instance["imageId"], rescue=(netboot_mode == "rescue")) + print(result) + diff --git a/python/reboot_ovh_vps_ssd.py b/python/reboot_ovh_vps_ssd.py new file mode 100644 index 0000000..71c5227 --- /dev/null +++ b/python/reboot_ovh_vps_ssd.py @@ -0,0 +1,40 @@ +# -*- encoding: utf-8 -*- +import json +try: + from ovh import ovh +except ImportError: + # In case it's installed globally + import ovh +import sys +import ovh_helper + +# Credentials are stored in ~/.ovh.conf +# See ovh/README.rst +client = ovh.Client() + +vps_list = client.get('/vps/') +if sys.argv[-1] in vps_list: + vps = sys.argv[-1] +else: + print("VPS not in list:") + for vps in vps_list: + print(vps) + sys.exit(1) + +if "--rescue" in sys.argv: + netboot_mode="rescue" +elif "--local" in sys.argv: + netboot_mode="local" +else: + netboot_mode=None + +current_state=client.get("/vps/{}".format(vps))["netbootMode"] + +if netboot_mode is None or current_state == netboot_mode: + client.post("/vps/{}/reboot".format(vps)) + task_type="rebootVm" +else: + client.put("/vps/{}".format(vps), netbootMode=netboot_mode) + task_type="setNetboot" + +ovh_helper.show_progress(client, vps, task_type) diff --git a/python/reboot_vps_server.py b/python/reboot_vps_server.py deleted file mode 100644 index 71c5227..0000000 --- a/python/reboot_vps_server.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- encoding: utf-8 -*- -import json -try: - from ovh import ovh -except ImportError: - # In case it's installed globally - import ovh -import sys -import ovh_helper - -# Credentials are stored in ~/.ovh.conf -# See ovh/README.rst -client = ovh.Client() - -vps_list = client.get('/vps/') -if sys.argv[-1] in vps_list: - vps = sys.argv[-1] -else: - print("VPS not in list:") - for vps in vps_list: - print(vps) - sys.exit(1) - -if "--rescue" in sys.argv: - netboot_mode="rescue" -elif "--local" in sys.argv: - netboot_mode="local" -else: - netboot_mode=None - -current_state=client.get("/vps/{}".format(vps))["netbootMode"] - -if netboot_mode is None or current_state == netboot_mode: - client.post("/vps/{}/reboot".format(vps)) - task_type="rebootVm" -else: - client.put("/vps/{}".format(vps), netbootMode=netboot_mode) - task_type="setNetboot" - -ovh_helper.show_progress(client, vps, task_type) diff --git a/python/reinstall_cloud_instance.py b/python/reinstall_cloud_instance.py deleted file mode 100644 index c488fda..0000000 --- a/python/reinstall_cloud_instance.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- encoding: utf-8 -*- -import json -try: - from ovh import ovh -except ImportError: - # In case it's installed globally - import ovh -import sys - -# Credentials are stored in ~/.ovh.conf -# See ovh/README.rst -client = ovh.Client() - -projects_list = client.get('/cloud/project/') -if len(projects_list) > 1: - print("More than one project is not supported, taking the first one") -project = projects_list[0] -instances_list = client.get('/cloud/project/{}/instance'.format(project)) -instances = dict(map(lambda x: (x["id"], x), instances_list)) -if sys.argv[-1] in instances: - instance = instances[sys.argv[-1]] -else: - print("Instance not in list:") - for instance in instances_list: - print("{}: {}".format(instance["name"], instance["id"])) - sys.exit(1) - -current_image = instance["imageId"] -available_images = client.get('/cloud/project/{}/image'.format(project), - osType="linux", - region=instance["region"]) -available_images_ids = list(map(lambda x: x["id"], available_images)) - -def print_images(available_images): - for image in available_images: - print("{}: {}".format(image["name"], image["id"])) - -def reinstall(image_id): - return client.post('/cloud/project/{}/instance/{}/reinstall'.format(project, instance["id"]), - imageId=image_id) - -if "--get-state" in sys.argv: - print(instance["status"]) -elif "--use-current" in sys.argv: - if current_image in available_images_ids: - print("Current image still available, using it") - print(reinstall(current_image)) - else: - print("Current image no more available. Chose among:") - print_images(available_images) -elif sys.argv[-1] in available_templates: - print("Chosen image available, using it") - print(reinstall(current_image)) -else: - print("Chosen image not available. Chose among:") - print_images(available_images) - diff --git a/python/reinstall_ovh_cloud_instance.py b/python/reinstall_ovh_cloud_instance.py new file mode 100644 index 0000000..c0d2617 --- /dev/null +++ b/python/reinstall_ovh_cloud_instance.py @@ -0,0 +1,46 @@ +# -*- encoding: utf-8 -*- +import json +try: + from ovh import ovh +except ImportError: + # In case it's installed globally + import ovh +import sys +from ovh_helper import find_cloud_instance + +# Credentials are stored in ~/.ovh.conf +# See ovh/README.rst +client = ovh.Client() + +project, instance = find_cloud_instance(client, sys.argv[-1]) + +current_image = instance["imageId"] +available_images = client.get('/cloud/project/{}/image'.format(project), + osType="linux", + region=instance["region"]) +available_images_ids = list(map(lambda x: x["id"], available_images)) + +def print_images(available_images): + for image in available_images: + print("{}: {}".format(image["name"], image["id"])) + +def reinstall(image_id): + return client.post('/cloud/project/{}/instance/{}/reinstall'.format(project, instance["id"]), + imageId=image_id) + +if "--get-state" in sys.argv: + print(instance["status"]) +elif "--use-current" in sys.argv: + if current_image in available_images_ids: + print("Current image still available, using it") + print(reinstall(current_image)) + else: + print("Current image no more available. Chose among:") + print_images(available_images) +elif sys.argv[-1] in available_templates: + print("Chosen image available, using it") + print(reinstall(current_image)) +else: + print("Chosen image not available. Chose among:") + print_images(available_images) + diff --git a/python/reinstall_ovh_vps_ssd.py b/python/reinstall_ovh_vps_ssd.py new file mode 100644 index 0000000..9f535cc --- /dev/null +++ b/python/reinstall_ovh_vps_ssd.py @@ -0,0 +1,58 @@ +# -*- encoding: utf-8 -*- +import json +try: + from ovh import ovh +except ImportError: + # In case it's installed globally + import ovh +import sys +import ovh_helper + +# Credentials are stored in ~/.ovh.conf +# See ovh/README.rst +client = ovh.Client() + +vps_list = client.get('/vps/') +if sys.argv[-1] in vps_list: + vps = sys.argv[-1] +else: + print("VPS not in list:") + for vps in vps_list: + print(vps) + sys.exit(1) + +current_distribution = client.get('/vps/{}/distribution'.format(vps)) + +available_templates = client.get('/vps/{}/templates'.format(vps)) + +def print_templates(client, vps, available_templates): + for tid in available_templates: + template = client.get('/vps/{}/templates/{}'.format(vps, tid)) + print("{}: {}".format(template["id"], template["distribution"])) + + +if "--get-state" in sys.argv: + print(client.get('/vps/{}'.format(vps))["state"]) +elif "--use-current" in sys.argv: + if current_distribution['id'] in available_templates: + print("Current template still available, using it") + result = client.post('/vps/{}/reinstall'.format(vps), templateId=current_distribution['id']) + print(result) + ovh_helper.show_progress(client, vps, "reinstallVm") + else: + print("Current template no more available. Chose among:") + print_templates(client, vps, available_templates) +elif sys.argv[-1] in available_templates: + print("Chosen template available, using it") + result = client.post('/vps/{}/reinstall'.format(vps), templateId=int(sys.argv[-1])) + print(result) + ovh_helper.show_progress(client, vps, "reinstallVm") +else: + print("Chosen template not available. Chose among:") + print_templates(client, vps, available_templates) + +# Reboot in rescue: +# PUT /vps/{serviceName} +# netbootMode "rescue" / "local" +# changer son nom: +# displayName: "..." diff --git a/python/reinstall_vps_server.py b/python/reinstall_vps_server.py deleted file mode 100644 index 9f535cc..0000000 --- a/python/reinstall_vps_server.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- encoding: utf-8 -*- -import json -try: - from ovh import ovh -except ImportError: - # In case it's installed globally - import ovh -import sys -import ovh_helper - -# Credentials are stored in ~/.ovh.conf -# See ovh/README.rst -client = ovh.Client() - -vps_list = client.get('/vps/') -if sys.argv[-1] in vps_list: - vps = sys.argv[-1] -else: - print("VPS not in list:") - for vps in vps_list: - print(vps) - sys.exit(1) - -current_distribution = client.get('/vps/{}/distribution'.format(vps)) - -available_templates = client.get('/vps/{}/templates'.format(vps)) - -def print_templates(client, vps, available_templates): - for tid in available_templates: - template = client.get('/vps/{}/templates/{}'.format(vps, tid)) - print("{}: {}".format(template["id"], template["distribution"])) - - -if "--get-state" in sys.argv: - print(client.get('/vps/{}'.format(vps))["state"]) -elif "--use-current" in sys.argv: - if current_distribution['id'] in available_templates: - print("Current template still available, using it") - result = client.post('/vps/{}/reinstall'.format(vps), templateId=current_distribution['id']) - print(result) - ovh_helper.show_progress(client, vps, "reinstallVm") - else: - print("Current template no more available. Chose among:") - print_templates(client, vps, available_templates) -elif sys.argv[-1] in available_templates: - print("Chosen template available, using it") - result = client.post('/vps/{}/reinstall'.format(vps), templateId=int(sys.argv[-1])) - print(result) - ovh_helper.show_progress(client, vps, "reinstallVm") -else: - print("Chosen template not available. Chose among:") - print_templates(client, vps, available_templates) - -# Reboot in rescue: -# PUT /vps/{serviceName} -# netbootMode "rescue" / "local" -# changer son nom: -# displayName: "..." diff --git a/scripts/ovh_cloud_instance/arch_host_script.sh b/scripts/ovh_cloud_instance/arch_host_script.sh index c188514..3408563 100755 --- a/scripts/ovh_cloud_instance/arch_host_script.sh +++ b/scripts/ovh_cloud_instance/arch_host_script.sh @@ -29,7 +29,7 @@ sudo mount "$DEVICE" /mnt for i in /mnt/*; do if [ "$i" = "/mnt/boot" ]; then # keep /boot/grub - sudo rm -f $i/* + sudo rm -f $i/* || true else sudo rm -rf $i fi diff --git a/scripts/send_and_run.tcl b/scripts/send_and_run.tcl index a087a2d..42fae62 100755 --- a/scripts/send_and_run.tcl +++ b/scripts/send_and_run.tcl @@ -6,7 +6,7 @@ set environment [lindex $argv 3] set script [lindex $argv 4] set files [lrange $argv 4 end] -if {$password == ""} { +if {$password == "x"} { set ask_password 0 } else { set ask_password 1 -- cgit v1.2.3 From 62fe8998b5497864d51ed0fe909a0ff60cc1520b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Fri, 16 Mar 2018 20:37:40 +0100 Subject: Make cpu limit for xmr stak parameter --- environments/global/types/s1-2.yaml | 1 + environments/global/types/vps-ovhssd-1.yaml | 1 + modules/profile/files/xmr_stak/xmr-stak.service | 15 --------------- modules/profile/manifests/xmr_stak.pp | 3 ++- modules/profile/templates/xmr_stak/xmr-stak.service.erb | 16 ++++++++++++++++ 5 files changed, 20 insertions(+), 16 deletions(-) delete mode 100644 modules/profile/files/xmr_stak/xmr-stak.service create mode 100644 modules/profile/templates/xmr_stak/xmr-stak.service.erb diff --git a/environments/global/types/s1-2.yaml b/environments/global/types/s1-2.yaml index a7ba753..5bfdf9a 100644 --- a/environments/global/types/s1-2.yaml +++ b/environments/global/types/s1-2.yaml @@ -7,3 +7,4 @@ base_installation::system_hostname: "%{ldapvar.self.vars.host.0}" base_installation::real_hostname: "%{facts.ec2_metadata.hostname}.v.immae.eu" base_installation::ldap_cert_path: "/etc/ssl/certs/ca-certificates.crt" letsencrypt::try_for_real_hostname: true +profile::xmr_stak::cpulimit: "30" diff --git a/environments/global/types/vps-ovhssd-1.yaml b/environments/global/types/vps-ovhssd-1.yaml index 68534dc..8dd512c 100644 --- a/environments/global/types/vps-ovhssd-1.yaml +++ b/environments/global/types/vps-ovhssd-1.yaml @@ -8,3 +8,4 @@ base_installation::real_hostname: "%{facts.ec2_metadata.hostname}.ovh.net" base_installation::grub_device: "/dev/sdb" base_installation::ldap_cert_path: "/etc/ssl/certs/ca-certificates.crt" letsencrypt::try_for_real_hostname: false +profile::xmr_stak::cpulimit: "90" diff --git a/modules/profile/files/xmr_stak/xmr-stak.service b/modules/profile/files/xmr_stak/xmr-stak.service deleted file mode 100644 index 93ee383..0000000 --- a/modules/profile/files/xmr_stak/xmr-stak.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=XMR Miner -After=network.target - -[Service] -WorkingDirectory=/var/lib/xmr_stak - -Type=simple -User=xmr_stak -Group=xmr_stak -ExecStart=/usr/bin/cpulimit --limit 90 /usr/bin/xmr-stak -c /var/lib/xmr_stak/xmr-stak.conf -Nice=19 - -[Install] -WantedBy=multi-user.target diff --git a/modules/profile/manifests/xmr_stak.pp b/modules/profile/manifests/xmr_stak.pp index c8bbcdd..ccb6baa 100644 --- a/modules/profile/manifests/xmr_stak.pp +++ b/modules/profile/manifests/xmr_stak.pp @@ -1,6 +1,7 @@ class profile::xmr_stak ( String $mining_pool, String $wallet, + Optional[String] $cpulimit = "50", Optional[String] $password = "x", ) { ensure_resource('exec', 'systemctl daemon-reload', { @@ -25,7 +26,7 @@ class profile::xmr_stak ( mode => "0644", owner => "root", group => "root", - source => "puppet:///modules/profile/xmr_stak/xmr-stak.service", + content => template("profile/xmr_stak/xmr-stak.service.erb"), require => User["xmr_stak"], notify => Exec["systemctl daemon-reload"] } diff --git a/modules/profile/templates/xmr_stak/xmr-stak.service.erb b/modules/profile/templates/xmr_stak/xmr-stak.service.erb new file mode 100644 index 0000000..d63103b --- /dev/null +++ b/modules/profile/templates/xmr_stak/xmr-stak.service.erb @@ -0,0 +1,16 @@ +[Unit] +Description=XMR Miner +After=network.target + +[Service] +WorkingDirectory=/var/lib/xmr_stak + +Type=simple +User=xmr_stak +Group=xmr_stak +ExecStart=/usr/bin/cpulimit --limit <%= @cpulimit %> /usr/bin/xmr-stak -c /var/lib/xmr_stak/xmr-stak.conf +Nice=19 + +[Install] +WantedBy=multi-user.target + -- cgit v1.2.3 From c443842eae41fb28d5608acfd303106e5f24798a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Sat, 17 Mar 2018 10:48:34 +0100 Subject: Add cron to update pacman keys --- modules/base_installation/manifests/cronie.pp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/base_installation/manifests/cronie.pp b/modules/base_installation/manifests/cronie.pp index 72f2d8f..e8d5dfd 100644 --- a/modules/base_installation/manifests/cronie.pp +++ b/modules/base_installation/manifests/cronie.pp @@ -29,5 +29,12 @@ class base_installation::cronie inherits base_installation { user => root, special => "reboot" } + cron { 'pacman_keys': + ensure => present, + command => "pacman-key --populate archlinux", + user => root, + minute => "0", + hour => "1", + } } } -- cgit v1.2.3 From e8493916ff6e957c752df1cfc1789844c426d987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 19 Mar 2018 16:02:30 +0100 Subject: Add ldap backend for hiera lookup --- environments/hiera.yaml | 3 ++ .../lib/puppet/functions/ldap_data.rb | 46 ++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 modules/base_installation/lib/puppet/functions/ldap_data.rb diff --git a/environments/hiera.yaml b/environments/hiera.yaml index 5a9a6d6..eda5eb3 100644 --- a/environments/hiera.yaml +++ b/environments/hiera.yaml @@ -9,6 +9,9 @@ hierarchy: - name: "Initialization variables" path: "/root/puppet_variables.json" + - name: "Puppet ldap variables" + data_hash: ldap_data + - name: "Per-role environment data" mapped_paths: [ldapvar.self.vars.roles, role, "roles/%{role}.yaml"] diff --git a/modules/base_installation/lib/puppet/functions/ldap_data.rb b/modules/base_installation/lib/puppet/functions/ldap_data.rb new file mode 100644 index 0000000..ff8d779 --- /dev/null +++ b/modules/base_installation/lib/puppet/functions/ldap_data.rb @@ -0,0 +1,46 @@ +require 'json' + +Puppet::Functions.create_function(:ldap_data) do + dispatch :ldap_data do + param 'Hash', :options + param 'Puppet::LookupContext', :context + end + + def ldap_data(options, context) + begin + require 'ldap' + require 'puppet/util/ldap/connection' + rescue + context.not_found + return + end + + if !context.cache_has_key("ldap_lookup") + begin + conn = Puppet::Util::Ldap::Connection.instance + conn.start + connection = conn.connection + rescue ::LDAP::ResultError => e + raise Puppet::ParseError, ("ldapquery(): LDAP ResultError - #{e.message}") + end + + host = Facter.value('ec2_metadata')["hostname"] + base = Puppet[:ldapbase] + scope = ::LDAP::LDAP_SCOPE_SUBTREE + filter = "(objectclass=*)" + + data = {} + connection.search(base, scope, filter) do |entry| + data_ = entry.to_hash + jsons = data_["immaePuppetJson"] || [] + jsons.each do |json| + data.merge!(JSON.parse(json)) + end + end + + context.cache("ldap_lookup", data) + end + + context.cached_value("ldap_lookup") + end +end -- cgit v1.2.3 From 284fd97e815e9dddf6640fcc4fc5e996687edc8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 19 Mar 2018 16:07:56 +0100 Subject: Migrate hiera information to ldap --- environments/global/common.yaml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/environments/global/common.yaml b/environments/global/common.yaml index 4536b83..5911194 100644 --- a/environments/global/common.yaml +++ b/environments/global/common.yaml @@ -27,15 +27,8 @@ base_installation::puppet_pass_seed: "/etc/puppetlabs/puppet/password_seed" base_installation::puppet_ssl_path: "/etc/puppetlabs/ssl" base_installation::system_locales: ["fr_FR.UTF-8", "en_US.UTF-8"] base_installation::system_timezone: "Europe/Paris" -base_installation::system_users: - - userid: 1000 - username: "immae" - groups: ["wheel"] - keys: - - host: "immae.eu" - key: "AAAAB3NzaC1yc2EAAAADAQABAAABAQDi5PgLBwMRyRwzJPnSgUyRAuB9AAxMijsw1pR/t/wmxQne1O5fIPOleHx+D8dyZbwm+XkzlcJpgT0Qy3qC9J8BPhshJvO/tA/8CI/oS/FE0uWsyACH1DMO2dk4gRRZGSE9IuzDMRPlnfZ3n0tdsPzzv3GH4It/oPIgsvkTowKztGLQ7Xmjr5BxzAhXcIQymqA0U3XWHSdWvnSRDaOFG0PDoVMS85IdwlviVKLnV5Sstb4NC/P28LFfgvW8DO/XrOqujgDomqTmR41dK/AyrGGOb2cQUMO4l8Oa+74aOyKaB61rr/rJkr+wCbEttkTvgFa6zZygSk3edfiWE2rgn4+v" - key_type: "ssh-rsa" -profile::xmr_stak::mining_pool: "pool.minexmr.com:7777" -profile::xmr_stak::wallet: "44CA8TxTFYbQqN2kLyk8AnB6Ghz4mcbGpYC2EyXW7A8H9QspvWnTjDn39XUZDPrFwPa5JNwt4TmAxcooPWv4SaJqL87Bcdo" -letsencrypt::email: "sites+letsencrypt@mail.immae.eu" +base_installation::system_users: [] # Fetched via ldap +profile::xmr_stak::mining_pool: "" # Fetched via ldap +profile::xmr_stak::wallet: "" # Fetched via ldap +letsencrypt::email: ~ # Fetched via ldap letsencrypt::try_for_real_hostname: true -- cgit v1.2.3 From 82caf31189f11a9bca4836260e8ee8ba954690f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Sat, 17 Mar 2018 11:00:48 +0100 Subject: Add mail profile --- environments/global/common.yaml | 1 + modules/profile/manifests/mail.pp | 14 ++++++++++++++ modules/profile/templates/mail/ssmtp.conf.erb | 14 ++++++++++++++ 3 files changed, 29 insertions(+) create mode 100644 modules/profile/manifests/mail.pp create mode 100644 modules/profile/templates/mail/ssmtp.conf.erb diff --git a/environments/global/common.yaml b/environments/global/common.yaml index 5911194..e7c70d4 100644 --- a/environments/global/common.yaml +++ b/environments/global/common.yaml @@ -30,5 +30,6 @@ base_installation::system_timezone: "Europe/Paris" base_installation::system_users: [] # Fetched via ldap profile::xmr_stak::mining_pool: "" # Fetched via ldap profile::xmr_stak::wallet: "" # Fetched via ldap +profile::mail::mailhub: "" # Fetched via ldap letsencrypt::email: ~ # Fetched via ldap letsencrypt::try_for_real_hostname: true diff --git a/modules/profile/manifests/mail.pp b/modules/profile/manifests/mail.pp new file mode 100644 index 0000000..cc47b77 --- /dev/null +++ b/modules/profile/manifests/mail.pp @@ -0,0 +1,14 @@ +class profile::mail ( + String $mailhub, + Optional[Integer] $mailhub_port = 25, +) { + ensure_packages(["s-nail", "ssmtp"]) + + $hostname = lookup("base_installation::real_hostname") + + file { "/etc/ssmtp/ssmtp.conf": + ensure => "present", + content => template("profile/mail/ssmtp.conf.erb"), + } +} + diff --git a/modules/profile/templates/mail/ssmtp.conf.erb b/modules/profile/templates/mail/ssmtp.conf.erb new file mode 100644 index 0000000..e7a0410 --- /dev/null +++ b/modules/profile/templates/mail/ssmtp.conf.erb @@ -0,0 +1,14 @@ +# +# /etc/ssmtp.conf -- a config file for sSMTP sendmail. +# +# The person who gets all mail for userids < 1000 +# Make this empty to disable rewriting. +root=postmaster +# The place where the mail goes. The actual machine name is required +# no MX records are consulted. Commonly mailhosts are named mail.domain.com +# The example will fit if you are in domain.com and you mailhub is so named. +mailhub=<%= @mailhub %>:<%= @mailhub_port %> +# Where will the mail seem to come from? +#rewriteDomain=y +# The full hostname +hostname=<%= @hostname %> -- cgit v1.2.3 From cfa8eb925ff0115d5d4b2298f7d9a054b869fa75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Sat, 17 Mar 2018 12:46:37 +0100 Subject: Add puppet_apply script with lockfile --- .../files/cronie/puppet-post-merge | 4 ++-- .../base_installation/files/scripts/puppet_apply | 23 ++++++++++++++++++++++ .../files/scripts/puppet_reset_and_apply | 2 +- modules/base_installation/manifests/cronie.pp | 4 ++-- modules/base_installation/manifests/puppet.pp | 6 ++++++ 5 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 modules/base_installation/files/scripts/puppet_apply diff --git a/modules/base_installation/files/cronie/puppet-post-merge b/modules/base_installation/files/cronie/puppet-post-merge index ac5e3ff..f5c21a7 100644 --- a/modules/base_installation/files/cronie/puppet-post-merge +++ b/modules/base_installation/files/cronie/puppet-post-merge @@ -1,7 +1,7 @@ #!/bin/bash -## Run Puppet locally using puppet apply +## Run Puppet locally using puppet_apply git submodule update --init -/usr/bin/puppet apply `pwd`/manifests/site.pp +/usr/local/sbin/puppet_apply `pwd`/manifests/site.pp ## Log status of the Puppet run if [ $? -eq 0 ] diff --git a/modules/base_installation/files/scripts/puppet_apply b/modules/base_installation/files/scripts/puppet_apply new file mode 100644 index 0000000..69673cc --- /dev/null +++ b/modules/base_installation/files/scripts/puppet_apply @@ -0,0 +1,23 @@ +#!/bin/bash + +lockfile=/var/run/puppet-apply.lock +path=`dirname $0` +path=`cd $path/..; pwd` + +if [ $(id -u) -gt 0 ]; then + echo "You must be root to run this script." >&2 + exit 2 +fi + +if (set -o noclobber; echo "$$" > "$lockfile") 2> /dev/null; then + trap 'rm -f "$lockfile"; exit $?' INT TERM EXIT + + puppet apply "$@" + + rm -f "$lockfile" + trap - INT TERM EXIT +else + echo "Failed to acquire lockfile: $lockfile." >&2 + echo "Held by $(cat $lockfile 2>/dev/null)" >&2 + exit 1 +fi diff --git a/modules/base_installation/files/scripts/puppet_reset_and_apply b/modules/base_installation/files/scripts/puppet_reset_and_apply index 6743044..0350e6e 100644 --- a/modules/base_installation/files/scripts/puppet_reset_and_apply +++ b/modules/base_installation/files/scripts/puppet_reset_and_apply @@ -11,4 +11,4 @@ fi git reset --hard origin/$branch git submodule update --init -puppet apply --test manifests/site.pp +puppet_apply --test manifests/site.pp diff --git a/modules/base_installation/manifests/cronie.pp b/modules/base_installation/manifests/cronie.pp index e8d5dfd..e8f3e20 100644 --- a/modules/base_installation/manifests/cronie.pp +++ b/modules/base_installation/manifests/cronie.pp @@ -19,13 +19,13 @@ class base_installation::cronie inherits base_installation { } cron { 'puppet-apply': ensure => present, - command => "cd $base_installation::puppet_code_path ; puppet apply $base_installation::puppet_code_path/manifests/site.pp", + command => "cd $base_installation::puppet_code_path ; puppet_apply $base_installation::puppet_code_path/manifests/site.pp", user => root, minute => '*/20' } cron { 'puppet-apply-reboot': ensure => present, - command => "cd $base_installation::puppet_code_path ; puppet apply $base_installation::puppet_code_path/manifests/site.pp", + command => "cd $base_installation::puppet_code_path ; puppet_apply $base_installation::puppet_code_path/manifests/site.pp", user => root, special => "reboot" } diff --git a/modules/base_installation/manifests/puppet.pp b/modules/base_installation/manifests/puppet.pp index 6f7732d..0cb43bc 100644 --- a/modules/base_installation/manifests/puppet.pp +++ b/modules/base_installation/manifests/puppet.pp @@ -39,6 +39,12 @@ class base_installation::puppet ( source => "puppet:///modules/base_installation/scripts/report_print.rb" } + file { '/usr/local/sbin/puppet_apply': + mode => "0755", + ensure => present, + source => "puppet:///modules/base_installation/scripts/puppet_apply", + } + unless empty(find_file($password_seed)) { $ldap_password = generate_password(24, $password_seed, "ldap") $ssha_ldap_seed = generate_password(5, $password_seed, "ldap_seed") -- cgit v1.2.3 From 97103ded8390d10ea3468615c654e89d07c0a921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 19 Mar 2018 16:00:01 +0100 Subject: Filter ldapvar to contain only puppetClient information --- modules/base_installation/lib/facter/ldapvar.rb | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/modules/base_installation/lib/facter/ldapvar.rb b/modules/base_installation/lib/facter/ldapvar.rb index 3ee6623..08d58e4 100644 --- a/modules/base_installation/lib/facter/ldapvar.rb +++ b/modules/base_installation/lib/facter/ldapvar.rb @@ -27,16 +27,18 @@ begin connection.search(base, scope, filter) do |entry| data_ = entry.to_hash - data_['vars'] = (data_[Puppet[:ldapstackedattrs]] || []) - .map { |var| var.split("=", 2) } - .group_by { |(key, value)| key } - .map { |key, value| [key, value.map(&:last)] } - .to_h + if data_["objectClass"].any? { |class_| class_ == "puppetClient" } + data_['vars'] = (data_[Puppet[:ldapstackedattrs]] || []) + .map { |var| var.split("=", 2) } + .group_by { |(key, value)| key } + .map { |key, value| [key, value.map(&:last)] } + .to_h - data[:other] << data_ + data[:other] << data_ - if data_["cn"].any? { |cn| cn == host } - data[:self] = data_ + if data_["cn"].any? { |cn| cn == host } + data[:self] = data_ + end end end -- cgit v1.2.3 From d87a489f9585d10f0a185beb59ae16a10f27a7bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Fri, 16 Mar 2018 20:40:27 +0100 Subject: Add backup role --- .gitmodules | 3 + environments/global/common.yaml | 7 ++ environments/global/roles/backup.yaml | 11 ++ environments/hiera.yaml | 1 + modules/profile/manifests/known_hosts.pp | 11 ++ modules/role/manifests/backup.pp | 122 +++++++++++++++++++++ .../templates/backup/backup_dirname_head.sh.erb | 27 +++++ .../templates/backup/backup_dirname_part.sh.erb | 26 +++++ .../templates/backup/backup_dirname_tail.sh.erb | 4 + modules/role/templates/backup/backup_head.sh.erb | 20 ++++ .../role/templates/backup/backup_immae_eu.sh.erb | 79 +++++++++++++ modules/role/templates/backup/backup_tail.sh.erb | 0 .../templates/backup/ssh_host_changed.info.erb | 4 + .../role/templates/backup/ssh_key_changed.info.erb | 5 + modules/ssh_keygen | 1 + 15 files changed, 321 insertions(+) create mode 100644 environments/global/roles/backup.yaml create mode 100644 modules/profile/manifests/known_hosts.pp create mode 100644 modules/role/manifests/backup.pp create mode 100644 modules/role/templates/backup/backup_dirname_head.sh.erb create mode 100644 modules/role/templates/backup/backup_dirname_part.sh.erb create mode 100644 modules/role/templates/backup/backup_dirname_tail.sh.erb create mode 100644 modules/role/templates/backup/backup_head.sh.erb create mode 100644 modules/role/templates/backup/backup_immae_eu.sh.erb create mode 100644 modules/role/templates/backup/backup_tail.sh.erb create mode 100644 modules/role/templates/backup/ssh_host_changed.info.erb create mode 100644 modules/role/templates/backup/ssh_key_changed.info.erb create mode 160000 modules/ssh_keygen diff --git a/.gitmodules b/.gitmodules index fa5163a..735ca8c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -46,6 +46,9 @@ [submodule "python/ovh"] path = python/ovh url = git://git.immae.eu/github/ovh/python-ovh +[submodule "modules/ssh_keygen"] + path = modules/ssh_keygen + url = git://git.immae.eu/github/voxpupuli/puppet-ssh_keygen [submodule "modules/ssl"] path = modules/ssl url = git://git.immae.eu/github/fnerdwq/puppet-ssl diff --git a/environments/global/common.yaml b/environments/global/common.yaml index e7c70d4..4836f6e 100644 --- a/environments/global/common.yaml +++ b/environments/global/common.yaml @@ -8,6 +8,10 @@ lookup_options: merge: unique letsencrypt::hosts: merge: unique + role::backup::backups: + merge: unique + profile::known_hosts::hosts: + merge: unique classes: stdlib: ~ @@ -31,5 +35,8 @@ base_installation::system_users: [] # Fetched via ldap profile::xmr_stak::mining_pool: "" # Fetched via ldap profile::xmr_stak::wallet: "" # Fetched via ldap profile::mail::mailhub: "" # Fetched via ldap +role::backup::mailto: "" # Fetched via ldap +role::backup::backups: [] # Fetched via ldap +profile::known_hosts::hosts: [] # Fetched via ldap letsencrypt::email: ~ # Fetched via ldap letsencrypt::try_for_real_hostname: true diff --git a/environments/global/roles/backup.yaml b/environments/global/roles/backup.yaml new file mode 100644 index 0000000..52befe2 --- /dev/null +++ b/environments/global/roles/backup.yaml @@ -0,0 +1,11 @@ +--- +classes: + role::backup: ~ +role::backup::user: "backup" +role::backup::group: "backup" +base_installation::system_users: + - username: "%{lookup('role::backup::user')}" + userid: 976 + system: true + password: "!!" + diff --git a/environments/hiera.yaml b/environments/hiera.yaml index eda5eb3..61d40d8 100644 --- a/environments/hiera.yaml +++ b/environments/hiera.yaml @@ -8,6 +8,7 @@ defaults: hierarchy: - name: "Initialization variables" path: "/root/puppet_variables.json" + data_hash: json_data - name: "Puppet ldap variables" data_hash: ldap_data diff --git a/modules/profile/manifests/known_hosts.pp b/modules/profile/manifests/known_hosts.pp new file mode 100644 index 0000000..ed9ec8e --- /dev/null +++ b/modules/profile/manifests/known_hosts.pp @@ -0,0 +1,11 @@ +class profile::known_hosts ( + Optional[Array] $hosts = [] +) { + $hosts.each |$host| { + sshkey { $host["name"]: + ensure => "present", + key => $host["key"], + type => $host["type"], + } + } +} diff --git a/modules/role/manifests/backup.pp b/modules/role/manifests/backup.pp new file mode 100644 index 0000000..edfd5e0 --- /dev/null +++ b/modules/role/manifests/backup.pp @@ -0,0 +1,122 @@ +class role::backup ( + String $user, + String $group, + String $mailto, + Optional[Array] $backups = [], + Optional[String] $mountpoint = "/backup1", + Optional[String] $backup_script = "/usr/local/bin/backup.sh", +) { + include "base_installation" + + include "profile::mail" + include "profile::tools" + include "profile::xmr_stak" + include "profile::known_hosts" + + ssh_keygen { $user: + notify => Notify_refresh["notify-backup-sshkey-change"] + } + + $hosts = $backups.map |$backup| { $backup["host"] } + + notify_refresh { "notify-backup-sshkey-change": + message => template("role/backup/ssh_key_changed.info.erb"), + refreshonly => true + } + + $hosts.each |$host| { + notify_refresh { "notify-backup-sshhost-$host-changed": + message => template("role/backup/ssh_host_changed.info.erb"), + refreshonly => true, + subscribe => Sshkey[$host], + } + } + + concat { $backup_script: + ensure => "present", + ensure_newline => true, + mode => "0755", + } + + cron { "backup": + ensure => present, + command => $backup_script, + user => $user, + minute => 25, + hour => 3, + require => Concat[$backup_script], + } + + concat::fragment { "backup_head": + target => $backup_script, + content => template("role/backup/backup_head.sh.erb"), + order => "01-50", + } + + concat::fragment { "backup_tail": + target => $backup_script, + content => template("role/backup/backup_tail.sh.erb"), + order => "99-50", + } + + $backups.each |$infos| { + $dirname = $infos["name"] + $login = $infos["login"] + $host = $infos["host"] + $dest = "$login@$host" + $base = "$mountpoint/$dirname" + $nbr = $infos["nbr"] + $order_dirname = $infos["order"] + + file { $base: + ensure => "directory", + owner => $user, + group => $group, + require => Mount[$mountpoint], + } -> + file { "$base/older": + ensure => "directory", + owner => $user, + group => $group, + } -> + file { "$base/rsync_output": + ensure => "directory", + owner => $user, + group => $group, + } + + concat::fragment { "backup_${dirname}_head": + target => $backup_script, + content => template("role/backup/backup_dirname_head.sh.erb"), + order => "$order_dirname-01", + } + + concat::fragment { "backup_${dirname}_tail": + target => $backup_script, + content => template("role/backup/backup_dirname_tail.sh.erb"), + order => "$order_dirname-99", + } + + $infos["parts"].each |$part| { + $local_folder = $part["local_folder"] + $remote_folder = $part["remote_folder"] + $exclude_from = $part["exclude_from"] + $files_from = $part["files_from"] + $args = $part["args"] + $order_part = $part["order"] + + file { "$base/$local_folder": + ensure => "directory", + owner => $user, + group => $group, + require => File[$base], + } + + concat::fragment { "backup_${dirname}_${local_folder}": + target => $backup_script, + content => template("role/backup/backup_dirname_part.sh.erb"), + order => "$order_dirname-$order_part", + } + } + } +} diff --git a/modules/role/templates/backup/backup_dirname_head.sh.erb b/modules/role/templates/backup/backup_dirname_head.sh.erb new file mode 100644 index 0000000..e20cfd3 --- /dev/null +++ b/modules/role/templates/backup/backup_dirname_head.sh.erb @@ -0,0 +1,27 @@ +##### <%= @dirname %> ##### +DEST="<%= @dest %>" +BASE="<%= @base %>" +OLD_BAK_BASE=$BASE/older/j +BAK_BASE=${OLD_BAK_BASE}0 +RSYNC_OUTPUT=$BASE/rsync_output +NBR=<%= @nbr %> + +if ! ssh \ + -o PreferredAuthentications=publickey \ + -o StrictHostKeyChecking=yes \ + -o ClearAllForwardings=yes \ + $DEST backup; then + echo "Fichier de verrouillage backup sur $DEST ou impossible de se connecter" >&2 + skip=$DEST +fi + +rm -rf ${OLD_BAK_BASE}${NBR} +for j in `seq -w $(($NBR-1)) -1 0`; do + [ ! -d ${OLD_BAK_BASE}$j ] && continue + mv ${OLD_BAK_BASE}$j ${OLD_BAK_BASE}$(($j+1)) +done +mkdir $BAK_BASE +mv $RSYNC_OUTPUT $BAK_BASE +mkdir $RSYNC_OUTPUT + +if [ "$skip" != "$DEST" ]; then diff --git a/modules/role/templates/backup/backup_dirname_part.sh.erb b/modules/role/templates/backup/backup_dirname_part.sh.erb new file mode 100644 index 0000000..ec662c4 --- /dev/null +++ b/modules/role/templates/backup/backup_dirname_part.sh.erb @@ -0,0 +1,26 @@ +### <%= @dirname %> <%= @local_folder %> ### +LOCAL="<%= @local_folder %>" +REMOTE="<%= @remote_folder %>" + +cd $BASE/$LOCAL +cat > $EXCL_FROM < +EOF +cat > $FILES_FROM < +EOF + +OUT=$RSYNC_OUTPUT/$LOCAL +rsync -XAavbrz --fake-super -e ssh --numeric-ids --delete \ + --backup-dir=$BAK_BASE/$LOCAL \ +<%- unless @args.empty? -%> + <%= @args %>\ +<% end -%> +<%- unless @exclude_from.empty? -%> + --exclude-from=$EXCL_FROM \ +<% end -%> +<%- unless @files_from.empty? -%> + --files-from=$FILES_FROM \ +<% end -%> + $DEST:$REMOTE . > $OUT || true +### End <%= @dirname %> <%= @local_folder %> ### diff --git a/modules/role/templates/backup/backup_dirname_tail.sh.erb b/modules/role/templates/backup/backup_dirname_tail.sh.erb new file mode 100644 index 0000000..6b16c9d --- /dev/null +++ b/modules/role/templates/backup/backup_dirname_tail.sh.erb @@ -0,0 +1,4 @@ + + ssh $DEST sh -c "date > .last_backup" +fi # [ "$skip" != "$DEST" ] +##### End <%= @dirname %> ##### diff --git a/modules/role/templates/backup/backup_head.sh.erb b/modules/role/templates/backup/backup_head.sh.erb new file mode 100644 index 0000000..be9f5bf --- /dev/null +++ b/modules/role/templates/backup/backup_head.sh.erb @@ -0,0 +1,20 @@ +#!/bin/bash +MAILTO="<%= @mailto %>" + +EXCL_FROM=`mktemp` +FILES_FROM=`mktemp` +TMP_STDERR=`mktemp` + +on_exit() { + if [ -s "$TMP_STDERR" ]; then + cat "$TMP_STDERR" | mail -Ssendwait -s "save_distant rsync error" "$MAILTO" + fi + rm -f $TMP_STDERR $EXCL_FROM $FILES_FROM +} + +trap "on_exit" EXIT + +exec 2> "$TMP_STDERR" +exec < /dev/null + +set -e diff --git a/modules/role/templates/backup/backup_immae_eu.sh.erb b/modules/role/templates/backup/backup_immae_eu.sh.erb new file mode 100644 index 0000000..4fab30e --- /dev/null +++ b/modules/role/templates/backup/backup_immae_eu.sh.erb @@ -0,0 +1,79 @@ +#!/bin/bash +DEST="<%= @dest %>" +MAILTO="<%= @mailto %>" +BASE="<%= @base %>" +OLD_BAK_BASE=$BASE/older/j +BAK_BASE=${OLD_BAK_BASE}0 +RSYNC_OUTPUT=$BASE/rsync_output +NBR=7 + +TMP=`mktemp` +TMP_STDERR=`mktemp` + +trap "rm -f $TMP $TMP_STDERR" EXIT + +exec 2> "$TMP_STDERR" + +set -e +if ! `ssh -o ClearAllForwardings=yes $DEST backup`; then + echo "Fichier de verrouillage backup sur $DEST" + exit 1 +fi + +rm -rf ${OLD_BAK_BASE}${NBR} +for j in `seq -w $(($NBR-1)) -1 0`; do + [ ! -d ${OLD_BAK_BASE}$j ] && continue + mv ${OLD_BAK_BASE}$j ${OLD_BAK_BASE}$(($j+1)) +done +mkdir $BAK_BASE +mv $RSYNC_OUTPUT $BAK_BASE +mkdir $RSYNC_OUTPUT + +############## +NAME="home" +FOLDER="/home/immae" + +cd $BASE/$NAME +cat > $TMP < $OUT || true + +############## +NAME="system" +FOLDER="/" + +cd $BASE/$NAME +cat > $TMP < $OUT || true + +############## +ssh $DEST sh -c "date > .last_backup" + +if [ -s "$TMP_STDERR" ]; then + cat "$TMP_STDERR" | mail -Ssendwait -s "save_distant rsync error" "$MAILTO" +fi diff --git a/modules/role/templates/backup/backup_tail.sh.erb b/modules/role/templates/backup/backup_tail.sh.erb new file mode 100644 index 0000000..e69de29 diff --git a/modules/role/templates/backup/ssh_host_changed.info.erb b/modules/role/templates/backup/ssh_host_changed.info.erb new file mode 100644 index 0000000..ebf202e --- /dev/null +++ b/modules/role/templates/backup/ssh_host_changed.info.erb @@ -0,0 +1,4 @@ +Host <%= @host %> added, please send <%= @user %> key if necessary. +<%- if File.exist?("/home/#{@user}/.ssh/id_rsa.pub") %> + <%= File.read("/home/#{@user}/.ssh/id_rsa.pub") %> +<% end -%> diff --git a/modules/role/templates/backup/ssh_key_changed.info.erb b/modules/role/templates/backup/ssh_key_changed.info.erb new file mode 100644 index 0000000..43fd2ec --- /dev/null +++ b/modules/role/templates/backup/ssh_key_changed.info.erb @@ -0,0 +1,5 @@ +ssh key of <%= @user %> changed, +please update hosts: +<%- @hosts.each do |host| %> + - <%= host %> +<% end -%> diff --git a/modules/ssh_keygen b/modules/ssh_keygen new file mode 160000 index 0000000..ca53363 --- /dev/null +++ b/modules/ssh_keygen @@ -0,0 +1 @@ +Subproject commit ca53363249b58af96f90cb810c7c51dda8ba803b -- cgit v1.2.3 From a55a5e4aa468d550fcdf5bbeafe42dea13773e37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 19 Mar 2018 16:39:02 +0100 Subject: Fix load error for puppet --- modules/base_installation/lib/puppet/functions/ldap_data.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/base_installation/lib/puppet/functions/ldap_data.rb b/modules/base_installation/lib/puppet/functions/ldap_data.rb index ff8d779..0c92d89 100644 --- a/modules/base_installation/lib/puppet/functions/ldap_data.rb +++ b/modules/base_installation/lib/puppet/functions/ldap_data.rb @@ -10,7 +10,7 @@ Puppet::Functions.create_function(:ldap_data) do begin require 'ldap' require 'puppet/util/ldap/connection' - rescue + rescue LoadError context.not_found return end -- cgit v1.2.3 From f25ad097f24e0d39d5dd9ac2cef39760e671e08f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 19 Mar 2018 17:13:37 +0100 Subject: =?UTF-8?q?Don=E2=80=99t=20mount=20unavailables=20mounts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- environments/global/common.yaml | 8 ++++---- modules/base_installation/manifests/fstab.pp | 18 ------------------ modules/base_installation/manifests/init.pp | 2 -- modules/base_installation/manifests/params.pp | 1 - modules/profile/manifests/fstab.pp | 18 ++++++++++++++++++ modules/role/manifests/backup.pp | 1 + 6 files changed, 23 insertions(+), 25 deletions(-) delete mode 100644 modules/base_installation/manifests/fstab.pp create mode 100644 modules/profile/manifests/fstab.pp diff --git a/environments/global/common.yaml b/environments/global/common.yaml index 4836f6e..094e0ff 100644 --- a/environments/global/common.yaml +++ b/environments/global/common.yaml @@ -1,6 +1,6 @@ --- lookup_options: - base_installation::mounts: + profile::fstab::mounts: merge: unique classes: merge: deep @@ -22,9 +22,6 @@ base_installation::ldap_cn: "%{facts.ec2_metadata.hostname}" base_installation::ldap_server: "ldap.immae.eu" base_installation::ldap_uri: "ldaps://ldap.immae.eu" # FIXME: get all mounts without needing that hack? -base_installation::mounts: - - "%{facts.ldapvar.self.vars.mounts.0}" - - "%{facts.ldapvar.self.vars.mounts.1}" base_installation::puppet_conf_path: "/etc/puppetlabs/puppet" base_installation::puppet_code_path: "/etc/puppetlabs/code" base_installation::puppet_pass_seed: "/etc/puppetlabs/puppet/password_seed" @@ -32,6 +29,9 @@ base_installation::puppet_ssl_path: "/etc/puppetlabs/ssl" base_installation::system_locales: ["fr_FR.UTF-8", "en_US.UTF-8"] base_installation::system_timezone: "Europe/Paris" base_installation::system_users: [] # Fetched via ldap +profile::fstab::mounts: + - "%{facts.ldapvar.self.vars.mounts.0}" + - "%{facts.ldapvar.self.vars.mounts.1}" profile::xmr_stak::mining_pool: "" # Fetched via ldap profile::xmr_stak::wallet: "" # Fetched via ldap profile::mail::mailhub: "" # Fetched via ldap diff --git a/modules/base_installation/manifests/fstab.pp b/modules/base_installation/manifests/fstab.pp deleted file mode 100644 index f012e76..0000000 --- a/modules/base_installation/manifests/fstab.pp +++ /dev/null @@ -1,18 +0,0 @@ -class base_installation::fstab ( - $mounts = $base_installation::mounts -) inherits base_installation { - $mounts.each |$mount| { - unless empty($mount) { - $infos = split($mount, ';') - - file { $infos[0]: - ensure => directory, - } -> - mount { $infos[0]: - ensure => mounted, - device => "UUID=${infos[1]}", - fstype => $infos[2] - } - } - } -} diff --git a/modules/base_installation/manifests/init.pp b/modules/base_installation/manifests/init.pp index cb1cdda..998f8ff 100644 --- a/modules/base_installation/manifests/init.pp +++ b/modules/base_installation/manifests/init.pp @@ -6,7 +6,6 @@ class base_installation ( Optional[String] $ldap_dn = $base_installation::params::ldap_dn, Optional[String] $ldap_server = $base_installation::params::ldap_server, Optional[String] $ldap_uri = $base_installation::params::ldap_uri, - Optional[Array[String]] $mounts = $base_installation::params::mounts, Optional[String] $puppet_code_path = $base_installation::params::puppet_code_path, Optional[String] $puppet_conf_path = $base_installation::params::puppet_conf_path, Optional[String] $puppet_pass_seed = $base_installation::params::puppet_pass_seed, @@ -31,5 +30,4 @@ class base_installation ( contain ::base_installation::users contain ::base_installation::package_managers contain ::base_installation::puppet - contain ::base_installation::fstab } diff --git a/modules/base_installation/manifests/params.pp b/modules/base_installation/manifests/params.pp index 2a9fe6a..5ade838 100644 --- a/modules/base_installation/manifests/params.pp +++ b/modules/base_installation/manifests/params.pp @@ -10,7 +10,6 @@ class base_installation::params { $ldap_cert_path = "/etc/ssl/certs/ca-certificates.crt" $ldap_uri = "ldaps://ldap.example.com" $ldap_server = "ldap.example.com" - $mounts = [] $real_hostname = "example.com" $system_hostname = "example.com" $system_locales = ["en_US.UTF-8"] diff --git a/modules/profile/manifests/fstab.pp b/modules/profile/manifests/fstab.pp new file mode 100644 index 0000000..5f2e58e --- /dev/null +++ b/modules/profile/manifests/fstab.pp @@ -0,0 +1,18 @@ +class profile::fstab ( + Optional[Array] $mounts = [] +) { + $mounts.each |$mount| { + unless empty($mount) { + $infos = split($mount, ';') + + file { $infos[0]: + ensure => directory, + } -> + mount { $infos[0]: + ensure => mounted, + device => "UUID=${infos[1]}", + fstype => $infos[2] + } + } + } +} diff --git a/modules/role/manifests/backup.pp b/modules/role/manifests/backup.pp index edfd5e0..d7ba486 100644 --- a/modules/role/manifests/backup.pp +++ b/modules/role/manifests/backup.pp @@ -8,6 +8,7 @@ class role::backup ( ) { include "base_installation" + include "profile::fstab" include "profile::mail" include "profile::tools" include "profile::xmr_stak" -- cgit v1.2.3 From b45f55437f8239acf5576f4f5282b68eb933db32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 19 Mar 2018 18:44:43 +0100 Subject: Other fixes --- modules/base_installation/manifests/cronie.pp | 4 ++-- scripts/ovh_cloud_instance/arch_host_puppet_configuration_script.sh | 1 + scripts/ovh_vps_ssd/arch_host_puppet_configuration_script.sh | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/base_installation/manifests/cronie.pp b/modules/base_installation/manifests/cronie.pp index e8f3e20..2235470 100644 --- a/modules/base_installation/manifests/cronie.pp +++ b/modules/base_installation/manifests/cronie.pp @@ -19,13 +19,13 @@ class base_installation::cronie inherits base_installation { } cron { 'puppet-apply': ensure => present, - command => "cd $base_installation::puppet_code_path ; puppet_apply $base_installation::puppet_code_path/manifests/site.pp", + command => "cd $base_installation::puppet_code_path ; /usr/local/sbin/puppet_apply $base_installation::puppet_code_path/manifests/site.pp", user => root, minute => '*/20' } cron { 'puppet-apply-reboot': ensure => present, - command => "cd $base_installation::puppet_code_path ; puppet_apply $base_installation::puppet_code_path/manifests/site.pp", + command => "cd $base_installation::puppet_code_path ; /usr/local/sbin/puppet_apply $base_installation::puppet_code_path/manifests/site.pp", user => root, special => "reboot" } diff --git a/scripts/ovh_cloud_instance/arch_host_puppet_configuration_script.sh b/scripts/ovh_cloud_instance/arch_host_puppet_configuration_script.sh index b58e255..6aae8a7 100755 --- a/scripts/ovh_cloud_instance/arch_host_puppet_configuration_script.sh +++ b/scripts/ovh_cloud_instance/arch_host_puppet_configuration_script.sh @@ -9,3 +9,4 @@ sudo cp /tmp/arch_puppet_configuration_script.sh "$MOUNTPOINT/root/" sudo arch-chroot "$MOUNTPOINT" /root/arch_puppet_configuration_script.sh "$git_branch" "$environment" +sudo umount "$MOUNTPOINT" diff --git a/scripts/ovh_vps_ssd/arch_host_puppet_configuration_script.sh b/scripts/ovh_vps_ssd/arch_host_puppet_configuration_script.sh index dab8fac..35f46fd 100755 --- a/scripts/ovh_vps_ssd/arch_host_puppet_configuration_script.sh +++ b/scripts/ovh_vps_ssd/arch_host_puppet_configuration_script.sh @@ -11,3 +11,4 @@ cp /tmp/arch_puppet_configuration_script.sh "$MOUNTPOINT/root/" /tmp/root.x86_64/bin/arch-chroot "$MOUNTPOINT" /root/arch_puppet_configuration_script.sh "$git_branch" "$environment" +umount "$MOUNTPOINT" -- cgit v1.2.3 From 4772e48cc9e2e6ed39776b007cbf0d5514f4c7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 19 Mar 2018 19:06:22 +0100 Subject: Populate keyring at install --- modules/pacman | 2 +- scripts/arch_install_script.sh | 3 +++ scripts/ovh_cloud_instance/arch_host_script.sh | 2 +- scripts/ovh_vps_ssd/arch_chroot_script.sh | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/pacman b/modules/pacman index 8db3b97..5668cfa 160000 --- a/modules/pacman +++ b/modules/pacman @@ -1 +1 @@ -Subproject commit 8db3b97081230d5d89f8f1467eec05bf19b03f3c +Subproject commit 5668cfa6dff56d5965ab2143fd0aeb322518d9c8 diff --git a/scripts/arch_install_script.sh b/scripts/arch_install_script.sh index 21e202f..d2c6107 100755 --- a/scripts/arch_install_script.sh +++ b/scripts/arch_install_script.sh @@ -6,6 +6,9 @@ CODE_PATH="/etc/puppetlabs/code" rm -rf $CODE_PATH +pacman-key --init +pacman-key --populate archlinux + git clone -b $git_branch --recursive https://git.immae.eu/perso/Immae/Projets/Puppet.git $CODE_PATH puppet apply --environment $environment --tags base_installation --test $CODE_PATH/manifests/site.pp # The password seed requires puppet to be run twice diff --git a/scripts/ovh_cloud_instance/arch_host_script.sh b/scripts/ovh_cloud_instance/arch_host_script.sh index 3408563..42dcc4a 100755 --- a/scripts/ovh_cloud_instance/arch_host_script.sh +++ b/scripts/ovh_cloud_instance/arch_host_script.sh @@ -36,7 +36,7 @@ for i in /mnt/*; do done ##### /FIXME #### -sudo pacstrap /mnt base git puppet +sudo pacstrap -G /mnt base git puppet echo "$PART / auto defaults 0 1" | sudo tee /mnt/etc/fstab # /Install very basic system diff --git a/scripts/ovh_vps_ssd/arch_chroot_script.sh b/scripts/ovh_vps_ssd/arch_chroot_script.sh index 9dc5c46..7b7887f 100755 --- a/scripts/ovh_vps_ssd/arch_chroot_script.sh +++ b/scripts/ovh_vps_ssd/arch_chroot_script.sh @@ -21,7 +21,7 @@ for i in /mnt/*; do done ##### /FIXME #### -pacstrap /mnt base git puppet +pacstrap -G /mnt base git puppet echo "$PART / auto defaults 0 1" > /mnt/etc/fstab -- cgit v1.2.3 From f7d7967b445e59a46b3c2546c10eb9cc48ea42a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 19 Mar 2018 20:32:45 +0100 Subject: Add missing package for backup --- modules/role/manifests/backup.pp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/role/manifests/backup.pp b/modules/role/manifests/backup.pp index d7ba486..7a0c275 100644 --- a/modules/role/manifests/backup.pp +++ b/modules/role/manifests/backup.pp @@ -14,6 +14,8 @@ class role::backup ( include "profile::xmr_stak" include "profile::known_hosts" + ensure_packages(["rsync"]) + ssh_keygen { $user: notify => Notify_refresh["notify-backup-sshkey-change"] } -- cgit v1.2.3 From 5a1aab82ab45c14db40026bb7b1f19d0ee8a3cb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 19 Mar 2018 21:58:18 +0100 Subject: Change backup user id --- environments/global/roles/backup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/global/roles/backup.yaml b/environments/global/roles/backup.yaml index 52befe2..cdd5f09 100644 --- a/environments/global/roles/backup.yaml +++ b/environments/global/roles/backup.yaml @@ -5,7 +5,7 @@ role::backup::user: "backup" role::backup::group: "backup" base_installation::system_users: - username: "%{lookup('role::backup::user')}" - userid: 976 + userid: 2000 system: true password: "!!" -- cgit v1.2.3