From 5a4069dddbb8f41644f749e900016a2f0654aa65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz>
Date: Thu, 18 Mar 2021 14:59:20 +0100
Subject: [PATCH] Switch puppet mode from master-less to master
---
deploy.tf | 30 ++++++++++++++++++-
deployments/hadoop-single/plugin.py | 21 +++++--------
.../{single.pp.tmpl => site.pp.tmpl} | 0
deployments/hadoop/plugin.py | 23 ++++++--------
image/build.sh | 2 +-
image/hadoop.sh | 4 ++-
image/scripts/hadoop-single-setup.sh | 11 +++++--
orchestrate.py | 8 ++++-
8 files changed, 66 insertions(+), 33 deletions(-)
rename deployments/hadoop-single/{single.pp.tmpl => site.pp.tmpl} (100%)
diff --git a/deploy.tf b/deploy.tf
index f3e4cb7..70cdee6 100644
--- a/deploy.tf
+++ b/deploy.tf
@@ -12,6 +12,7 @@ terraform {
locals {
ord = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"]
keyfile = "ssh-key.${var.domain}.txt"
+ master_fqdn = "${data.template_file.user_data_common[0].vars.host}.${var.domain}"
}
data "openstack_compute_keypair_v2" "userkey" {
@@ -82,11 +83,38 @@ users:
- ALL=(ALL) NOPASSWD:ALL
%{ if count.index == 0 ~}
+bootcmd:
+ - cloud-init-per once puppet-start systemctl mask puppet
+ - cloud-init-per once puppet-start systemctl mask puppet-master
+ - cloud-init-per once manifests-dir mkdir -p /etc/puppet/code/environments/production/manifests
+
+packages:
+ - puppet-master
+
+puppet:
+ conf:
+ agent:
+ server: ${local.master_fqdn}
+
write_files:
- path: /home/deployadm/.ssh/id_rsa
permissions: '0600'
content: |
- ${indent(6, openstack_compute_keypair_v2.localkey.private_key)~}
+ ${indent(6, openstack_compute_keypair_v2.localkey.private_key)}
+ - path: /etc/puppet/autosign.conf
+ permissions: '0644'
+ content: |
+ ${local.master_fqdn}
+ /^${var.node_hostname}\d+\.${var.domain}$$/
+%{ else ~}
+
+bootcmd:
+ - cloud-init-per once puppet-start systemctl mask puppet
+
+puppet:
+ conf:
+ agent:
+ server: ${local.master_fqdn}
%{ endif ~}
runcmd:
diff --git a/deployments/hadoop-single/plugin.py b/deployments/hadoop-single/plugin.py
index 3dff629..109baee 100644
--- a/deployments/hadoop-single/plugin.py
+++ b/deployments/hadoop-single/plugin.py
@@ -6,14 +6,7 @@ class ComponentHadoopSingle(deployments.hadoop.plugin.ComponentHadoopCommon):
def action(self, action):
if action == 'files':
- template = self.generate('deployments/hadoop-single/single.pp.tmpl', 'single.pp', 0o600)
- if template:
- self.params['hdfs_deployed'] = 'true'
- result = template.substitute(self.params)
- print('-> single2.pp')
- with open('single2.pp', 'w') as f:
- os.chmod('single2.pp', 0o600)
- f.write(result)
+ self.generate('deployments/hadoop-single/site.pp.tmpl', 'site.pp', 0o600)
self.generate('deployments/hadoop/adduser.sh.tmpl', 'hadoop-adduser.sh', 0o755)
self.generate('deployments/hadoop/krb5.conf.tmpl', 'krb5.conf.hadoop', 0o644)
@@ -24,15 +17,17 @@ class ComponentHadoopSingle(deployments.hadoop.plugin.ComponentHadoopCommon):
sensitive = 'SENSITIVE=true '
return [
['ansible', '-i', './inventory', '-m', 'copy', '-a',
- 'src=single.pp dest=/root mode=0600', 'all'],
- ['ansible', '-i', './inventory', '-m', 'copy', '-a',
- 'src=single2.pp dest=/root mode=0600', 'all'],
+ 'src=site.pp dest=/etc/puppet/code/environments/production/manifests mode=0600 ' +
+ 'owner=puppet', 'master'],
['ansible', '-i', './inventory', '-m', 'copy', '-a',
'src=hadoop-adduser.sh dest=/usr/local/sbin mode=0755', 'all'],
['ansible', '-i', './inventory', '-m', 'shell', '-a',
- 'puppet apply --test /root/single.pp >> stage1.log 2>&1; echo $?', 'all'],
+ 'puppet agent --test >> stage1.log 2>&1; echo $?', 'all'],
+ ['ansible', '-i', './inventory', '-m', 'shell', '-a',
+ r'sed -i "s/^\\(\$hdfs_deployed\\s*=\\s*\\).*/\\1true/" /etc/puppet/code/' +
+ 'environments/production/manifests/site.pp', 'master'],
['ansible', '-i', './inventory', '-m', 'shell', '-a',
- 'puppet apply --test /root/single2.pp >> stage2.log 2>&1; echo $?', 'all'],
+ 'puppet agent --test >> stage2.log 2>&1; echo $?', 'all'],
['ansible', '-i', './inventory', '-m', 'shell', '-a',
'%s/usr/local/sbin/hadoop-adduser.sh %s' % (sensitive, self.config['image_user']),
'all'],
diff --git a/deployments/hadoop-single/single.pp.tmpl b/deployments/hadoop-single/site.pp.tmpl
similarity index 100%
rename from deployments/hadoop-single/single.pp.tmpl
rename to deployments/hadoop-single/site.pp.tmpl
diff --git a/deployments/hadoop/plugin.py b/deployments/hadoop/plugin.py
index c72ad5b..b83e528 100644
--- a/deployments/hadoop/plugin.py
+++ b/deployments/hadoop/plugin.py
@@ -47,15 +47,8 @@ class ComponentHadoop(ComponentHadoopCommon):
def action(self, action):
if action == 'files':
- template = self.generate('deployments/%s/site.pp.tmpl' % self.config['type'],
- 'site.pp', 0o600)
- if template:
- self.params['hdfs_deployed'] = 'true'
- site = template.substitute(self.params)
- print('-> site2.pp')
- with open('site2.pp', 'w') as f:
- os.chmod('site2.pp', 0o600)
- f.write(site)
+ self.generate('deployments/%s/site.pp.tmpl' % self.config['type'],
+ 'site.pp', 0o600)
self.generate('deployments/hadoop/adduser.sh.tmpl', 'hadoop-adduser.sh', 0o755)
self.generate('deployments/hadoop/krb5.conf.tmpl', 'krb5.conf.hadoop', 0o644)
@@ -66,15 +59,17 @@ class ComponentHadoop(ComponentHadoopCommon):
sensitive = 'SENSITIVE=true '
return [
['ansible', '-i', './inventory', '-m', 'copy', '-a',
- 'src=site.pp dest=/root mode=0600', 'all'],
- ['ansible', '-i', './inventory', '-m', 'copy', '-a',
- 'src=site2.pp dest=/root mode=0600', 'all'],
+ 'src=site.pp dest=/etc/puppet/code/environments/production/manifests mode=0600 ' +
+ 'owner=puppet', 'master'],
['ansible', '-i', './inventory', '-m', 'copy', '-a',
'src=hadoop-adduser.sh dest=/usr/local/sbin mode=0755', 'all'],
['ansible', '-i', './inventory', '-m', 'shell', '-a',
- 'puppet apply --test /root/site.pp >> stage1.log 2>&1; echo $?', 'all'],
+ 'puppet agent --test >> stage1.log 2>&1; echo $?', 'all'],
+ ['ansible', '-i', './inventory', '-m', 'shell', '-a',
+ r'sed -i "s/^\\(\$hdfs_deployed\\s*=\\s*\\).*/\\1true/" /etc/puppet/code/' +
+ 'environments/production/manifests/site.pp', 'master'],
['ansible', '-i', './inventory', '-m', 'shell', '-a',
- 'puppet apply --test /root/site2.pp >> stage2.log 2>&1; echo $?', 'all'],
+ 'puppet agent --test >> stage2.log 2>&1; echo $?', 'all'],
['ansible', '-i', './inventory', '-m', 'shell', '-a',
'%s/usr/local/sbin/hadoop-adduser.sh %s' % (sensitive, self.config['image_user']),
'all'],
diff --git a/image/build.sh b/image/build.sh
index 1f8326d..f8b7912 100755
--- a/image/build.sh
+++ b/image/build.sh
@@ -3,7 +3,7 @@
sed 's/MASTER=.*/MASTER="`hostname -f`"/' ../hadoop-adduser.sh > hadoop-adduser.sh
chmod +x hadoop-adduser.sh
touch -r ../deployments/hadoop/adduser.sh.tmpl hadoop-adduser.sh
-touch -r ../deployments/hadoop-single/single.pp.tmpl ../single.pp ../single2.pp
+touch -r ../deployments/hadoop-single/site.pp.tmpl ../site.pp
sudo /usr/sbin/build-openstack-debian-image \
--release stretch \
diff --git a/image/hadoop.sh b/image/hadoop.sh
index 5e30768..b6a6b30 100755
--- a/image/hadoop.sh
+++ b/image/hadoop.sh
@@ -42,7 +42,9 @@ cp -vp Puppetfile $BODI_CHROOT_PATH/etc/puppet/code/
chroot $BODI_CHROOT_PATH bash -c 'cd /etc/puppet/code; librarian-puppet install'
cp -vp hadoop-adduser.sh $BODI_CHROOT_PATH/usr/local/sbin/
cp -vp scripts/*.sh $BODI_CHROOT_PATH/usr/local/sbin/
-cp -vp ../single.pp ../single2.pp $BODI_CHROOT_PATH/root/
+mkdir -p $BODI_CHROOT_PATH/etc/puppet/code/environments/production/manifests
+install -v -m 0600 ../site.pp $BODI_CHROOT_PATH/etc/puppet/code/environments/production/manifests/
+chroot $BODI_CHROOT_PATH chown puppet /etc/puppet/code/environments/production/manifests/site.pp
cp -vp motd $BODI_CHROOT_PATH/etc/motd
egrep '^\s*GSSAPIAuthentication\s+yes' $BODI_CHROOT_PATH/etc/ssh/sshd_config || echo 'GSSAPIAuthentication yes' >> $BODI_CHROOT_PATH/etc/ssh/sshd_config
diff --git a/image/scripts/hadoop-single-setup.sh b/image/scripts/hadoop-single-setup.sh
index c9f94d7..abc27c6 100755
--- a/image/scripts/hadoop-single-setup.sh
+++ b/image/scripts/hadoop-single-setup.sh
@@ -12,9 +12,16 @@ fi
mkdir /data 2>/dev/null || :
-puppet apply --test /root/single.pp 2>&1 | tee stage1.log
+apt-get install -y puppet-master
+hostname -f > /etc/puppet/autosign.conf
+systemctl enable puppet-master
+service puppet-master restart
+
+puppet config set server `hostname -f`
+puppet agent --test 2>&1 | tee stage1.log
if test $? = 0 -o $? = 2; then
- puppet apply --test /root/single2.pp 2>&1 | tee stage2.log
+ sed -i "s/^\\(\$hdfs_deployed\\s*=\\s*\\).*/\\1true/" /etc/puppet/code/environments/production/manifests/site.pp
+ puppet agent --test 2>&1 | tee stage2.log
else
echo "First stage setup failed"
exit $?
diff --git a/orchestrate.py b/orchestrate.py
index e9b6bae..284339c 100755
--- a/orchestrate.py
+++ b/orchestrate.py
@@ -216,7 +216,13 @@ if 'wait' in args.actions:
if 'deployment' in args.actions:
print('== deployment ==')
- commands = []
+ commands = [
+ ['ansible', '-i', './inventory', '-m', 'command', '-a', 'systemctl enable puppet-master',
+ 'master'],
+ # started with the old /etc/hosts
+ ['ansible', '-i', './inventory', '-m', 'command', '-a', 'service puppet-master restart',
+ 'master'],
+ ]
perform_action('deployment', commands)
fperf.write('[%f] Elapsed time\n' % elapsed)
--
GitLab