From 5a4069dddbb8f41644f749e900016a2f0654aa65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz> Date: Thu, 18 Mar 2021 14:59:20 +0100 Subject: [PATCH] Switch puppet mode from master-less to master --- deploy.tf | 30 ++++++++++++++++++- deployments/hadoop-single/plugin.py | 21 +++++-------- .../{single.pp.tmpl => site.pp.tmpl} | 0 deployments/hadoop/plugin.py | 23 ++++++-------- image/build.sh | 2 +- image/hadoop.sh | 4 ++- image/scripts/hadoop-single-setup.sh | 11 +++++-- orchestrate.py | 8 ++++- 8 files changed, 66 insertions(+), 33 deletions(-) rename deployments/hadoop-single/{single.pp.tmpl => site.pp.tmpl} (100%) diff --git a/deploy.tf b/deploy.tf index f3e4cb7..70cdee6 100644 --- a/deploy.tf +++ b/deploy.tf @@ -12,6 +12,7 @@ terraform { locals { ord = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"] keyfile = "ssh-key.${var.domain}.txt" + master_fqdn = "${data.template_file.user_data_common[0].vars.host}.${var.domain}" } data "openstack_compute_keypair_v2" "userkey" { @@ -82,11 +83,38 @@ users: - ALL=(ALL) NOPASSWD:ALL %{ if count.index == 0 ~} +bootcmd: + - cloud-init-per once puppet-start systemctl mask puppet + - cloud-init-per once puppet-start systemctl mask puppet-master + - cloud-init-per once manifests-dir mkdir -p /etc/puppet/code/environments/production/manifests + +packages: + - puppet-master + +puppet: + conf: + agent: + server: ${local.master_fqdn} + write_files: - path: /home/deployadm/.ssh/id_rsa permissions: '0600' content: | - ${indent(6, openstack_compute_keypair_v2.localkey.private_key)~} + ${indent(6, openstack_compute_keypair_v2.localkey.private_key)} + - path: /etc/puppet/autosign.conf + permissions: '0644' + content: | + ${local.master_fqdn} + /^${var.node_hostname}\d+\.${var.domain}$$/ +%{ else ~} + +bootcmd: + - cloud-init-per once puppet-start systemctl mask puppet + +puppet: + conf: + agent: + server: ${local.master_fqdn} %{ endif ~} runcmd: diff --git a/deployments/hadoop-single/plugin.py b/deployments/hadoop-single/plugin.py index 3dff629..109baee 100644 --- a/deployments/hadoop-single/plugin.py +++ b/deployments/hadoop-single/plugin.py @@ -6,14 +6,7 @@ class ComponentHadoopSingle(deployments.hadoop.plugin.ComponentHadoopCommon): def action(self, action): if action == 'files': - template = self.generate('deployments/hadoop-single/single.pp.tmpl', 'single.pp', 0o600) - if template: - self.params['hdfs_deployed'] = 'true' - result = template.substitute(self.params) - print('-> single2.pp') - with open('single2.pp', 'w') as f: - os.chmod('single2.pp', 0o600) - f.write(result) + self.generate('deployments/hadoop-single/site.pp.tmpl', 'site.pp', 0o600) self.generate('deployments/hadoop/adduser.sh.tmpl', 'hadoop-adduser.sh', 0o755) self.generate('deployments/hadoop/krb5.conf.tmpl', 'krb5.conf.hadoop', 0o644) @@ -24,15 +17,17 @@ class ComponentHadoopSingle(deployments.hadoop.plugin.ComponentHadoopCommon): sensitive = 'SENSITIVE=true ' return [ ['ansible', '-i', './inventory', '-m', 'copy', '-a', - 'src=single.pp dest=/root mode=0600', 'all'], - ['ansible', '-i', './inventory', '-m', 'copy', '-a', - 'src=single2.pp dest=/root mode=0600', 'all'], + 'src=site.pp dest=/etc/puppet/code/environments/production/manifests mode=0600 ' + + 'owner=puppet', 'master'], ['ansible', '-i', './inventory', '-m', 'copy', '-a', 'src=hadoop-adduser.sh dest=/usr/local/sbin mode=0755', 'all'], ['ansible', '-i', './inventory', '-m', 'shell', '-a', - 'puppet apply --test /root/single.pp >> stage1.log 2>&1; echo $?', 'all'], + 'puppet agent --test >> stage1.log 2>&1; echo $?', 'all'], + ['ansible', '-i', './inventory', '-m', 'shell', '-a', + r'sed -i "s/^\\(\$hdfs_deployed\\s*=\\s*\\).*/\\1true/" /etc/puppet/code/' + + 'environments/production/manifests/site.pp', 'master'], ['ansible', '-i', './inventory', '-m', 'shell', '-a', - 'puppet apply --test /root/single2.pp >> stage2.log 2>&1; echo $?', 'all'], + 'puppet agent --test >> stage2.log 2>&1; echo $?', 'all'], ['ansible', '-i', './inventory', '-m', 'shell', '-a', '%s/usr/local/sbin/hadoop-adduser.sh %s' % (sensitive, self.config['image_user']), 'all'], diff --git a/deployments/hadoop-single/single.pp.tmpl b/deployments/hadoop-single/site.pp.tmpl similarity index 100% rename from deployments/hadoop-single/single.pp.tmpl rename to deployments/hadoop-single/site.pp.tmpl diff --git a/deployments/hadoop/plugin.py b/deployments/hadoop/plugin.py index c72ad5b..b83e528 100644 --- a/deployments/hadoop/plugin.py +++ b/deployments/hadoop/plugin.py @@ -47,15 +47,8 @@ class ComponentHadoop(ComponentHadoopCommon): def action(self, action): if action == 'files': - template = self.generate('deployments/%s/site.pp.tmpl' % self.config['type'], - 'site.pp', 0o600) - if template: - self.params['hdfs_deployed'] = 'true' - site = template.substitute(self.params) - print('-> site2.pp') - with open('site2.pp', 'w') as f: - os.chmod('site2.pp', 0o600) - f.write(site) + self.generate('deployments/%s/site.pp.tmpl' % self.config['type'], + 'site.pp', 0o600) self.generate('deployments/hadoop/adduser.sh.tmpl', 'hadoop-adduser.sh', 0o755) self.generate('deployments/hadoop/krb5.conf.tmpl', 'krb5.conf.hadoop', 0o644) @@ -66,15 +59,17 @@ class ComponentHadoop(ComponentHadoopCommon): sensitive = 'SENSITIVE=true ' return [ ['ansible', '-i', './inventory', '-m', 'copy', '-a', - 'src=site.pp dest=/root mode=0600', 'all'], - ['ansible', '-i', './inventory', '-m', 'copy', '-a', - 'src=site2.pp dest=/root mode=0600', 'all'], + 'src=site.pp dest=/etc/puppet/code/environments/production/manifests mode=0600 ' + + 'owner=puppet', 'master'], ['ansible', '-i', './inventory', '-m', 'copy', '-a', 'src=hadoop-adduser.sh dest=/usr/local/sbin mode=0755', 'all'], ['ansible', '-i', './inventory', '-m', 'shell', '-a', - 'puppet apply --test /root/site.pp >> stage1.log 2>&1; echo $?', 'all'], + 'puppet agent --test >> stage1.log 2>&1; echo $?', 'all'], + ['ansible', '-i', './inventory', '-m', 'shell', '-a', + r'sed -i "s/^\\(\$hdfs_deployed\\s*=\\s*\\).*/\\1true/" /etc/puppet/code/' + + 'environments/production/manifests/site.pp', 'master'], ['ansible', '-i', './inventory', '-m', 'shell', '-a', - 'puppet apply --test /root/site2.pp >> stage2.log 2>&1; echo $?', 'all'], + 'puppet agent --test >> stage2.log 2>&1; echo $?', 'all'], ['ansible', '-i', './inventory', '-m', 'shell', '-a', '%s/usr/local/sbin/hadoop-adduser.sh %s' % (sensitive, self.config['image_user']), 'all'], diff --git a/image/build.sh b/image/build.sh index 1f8326d..f8b7912 100755 --- a/image/build.sh +++ b/image/build.sh @@ -3,7 +3,7 @@ sed 's/MASTER=.*/MASTER="`hostname -f`"/' ../hadoop-adduser.sh > hadoop-adduser.sh chmod +x hadoop-adduser.sh touch -r ../deployments/hadoop/adduser.sh.tmpl hadoop-adduser.sh -touch -r ../deployments/hadoop-single/single.pp.tmpl ../single.pp ../single2.pp +touch -r ../deployments/hadoop-single/site.pp.tmpl ../site.pp sudo /usr/sbin/build-openstack-debian-image \ --release stretch \ diff --git a/image/hadoop.sh b/image/hadoop.sh index 5e30768..b6a6b30 100755 --- a/image/hadoop.sh +++ b/image/hadoop.sh @@ -42,7 +42,9 @@ cp -vp Puppetfile $BODI_CHROOT_PATH/etc/puppet/code/ chroot $BODI_CHROOT_PATH bash -c 'cd /etc/puppet/code; librarian-puppet install' cp -vp hadoop-adduser.sh $BODI_CHROOT_PATH/usr/local/sbin/ cp -vp scripts/*.sh $BODI_CHROOT_PATH/usr/local/sbin/ -cp -vp ../single.pp ../single2.pp $BODI_CHROOT_PATH/root/ +mkdir -p $BODI_CHROOT_PATH/etc/puppet/code/environments/production/manifests +install -v -m 0600 ../site.pp $BODI_CHROOT_PATH/etc/puppet/code/environments/production/manifests/ +chroot $BODI_CHROOT_PATH chown puppet /etc/puppet/code/environments/production/manifests/site.pp cp -vp motd $BODI_CHROOT_PATH/etc/motd egrep '^\s*GSSAPIAuthentication\s+yes' $BODI_CHROOT_PATH/etc/ssh/sshd_config || echo 'GSSAPIAuthentication yes' >> $BODI_CHROOT_PATH/etc/ssh/sshd_config diff --git a/image/scripts/hadoop-single-setup.sh b/image/scripts/hadoop-single-setup.sh index c9f94d7..abc27c6 100755 --- a/image/scripts/hadoop-single-setup.sh +++ b/image/scripts/hadoop-single-setup.sh @@ -12,9 +12,16 @@ fi mkdir /data 2>/dev/null || : -puppet apply --test /root/single.pp 2>&1 | tee stage1.log +apt-get install -y puppet-master +hostname -f > /etc/puppet/autosign.conf +systemctl enable puppet-master +service puppet-master restart + +puppet config set server `hostname -f` +puppet agent --test 2>&1 | tee stage1.log if test $? = 0 -o $? = 2; then - puppet apply --test /root/single2.pp 2>&1 | tee stage2.log + sed -i "s/^\\(\$hdfs_deployed\\s*=\\s*\\).*/\\1true/" /etc/puppet/code/environments/production/manifests/site.pp + puppet agent --test 2>&1 | tee stage2.log else echo "First stage setup failed" exit $? diff --git a/orchestrate.py b/orchestrate.py index e9b6bae..284339c 100755 --- a/orchestrate.py +++ b/orchestrate.py @@ -216,7 +216,13 @@ if 'wait' in args.actions: if 'deployment' in args.actions: print('== deployment ==') - commands = [] + commands = [ + ['ansible', '-i', './inventory', '-m', 'command', '-a', 'systemctl enable puppet-master', + 'master'], + # started with the old /etc/hosts + ['ansible', '-i', './inventory', '-m', 'command', '-a', 'service puppet-master restart', + 'master'], + ] perform_action('deployment', commands) fperf.write('[%f] Elapsed time\n' % elapsed) -- GitLab