From 25758a41b612957790d5eb28664affa41a580571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz> Date: Sat, 2 Jan 2021 22:46:59 +0100 Subject: [PATCH] Hadoop: pre-install Kerberos in the image, adduser script, beeline test with security --- image/.gitignore | 2 ++ image/HOWTO.sh | 5 ++++- image/hadoop.sh | 5 +++-- image/tests/common.sh | 5 +++-- 4 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 image/.gitignore diff --git a/image/.gitignore b/image/.gitignore new file mode 100644 index 0000000..be81188 --- /dev/null +++ b/image/.gitignore @@ -0,0 +1,2 @@ +build-image.log +hadoop-adduser.sh diff --git a/image/HOWTO.sh b/image/HOWTO.sh index d165d04..41cb1b6 100755 --- a/image/HOWTO.sh +++ b/image/HOWTO.sh @@ -1,6 +1,9 @@ #! /bin/sh -xe +sed 's/MASTER=.*/MASTER="`hostname -f`"/' ../hadoop-adduser.sh > hadoop-adduser.sh +chmod +x hadoop-adduser.sh + /usr/sbin/build-openstack-debian-image \ --release stretch \ - --extra-packages acl,default-jre-headless,git,gnupg,librarian-puppet,qemu-guest-agent,puppet \ + --extra-packages acl,default-jre-headless,git,gnupg,krb5-user,librarian-puppet,qemu-guest-agent,puppet \ --image-size 4 \ --hook-script ./hadoop.sh 2>&1 | tee build-image.log diff --git a/image/hadoop.sh b/image/hadoop.sh index ec9f829..5c7b539 100755 --- a/image/hadoop.sh +++ b/image/hadoop.sh @@ -13,7 +13,7 @@ case "$BODI_RELEASE" in ver='1.5.0' # 1.5.0 ;; esac -DAEMONS="$db_daemon hadoop-hdfs-datanode hadoop-hdfs-namenode hadoop-httpfs hadoop-mapreduce-historyserver hadoop-yarn-nodemanager hadoop-yarn-resourcemanager hbase-master hbase-regionserver hive-metastore hive-server2 spark-history-server zookeeper-server" +DAEMONS="$db_daemon krb5-admin-server krb5-kdc krb5-kpropd hadoop-hdfs-datanode hadoop-hdfs-namenode hadoop-httpfs hadoop-mapreduce-historyserver hadoop-yarn-nodemanager hadoop-yarn-resourcemanager hbase-master hbase-regionserver hive-metastore hive-server2 spark-history-server zookeeper-server" wget https://dist.apache.org/repos/dist/release/bigtop/KEYS -O - | chroot $BODI_CHROOT_PATH apt-key add - cp -vp apt/*.pref $BODI_CHROOT_PATH/etc/apt/preferences.d/ @@ -30,7 +30,7 @@ for d in $DAEMONS; do echo '#! /bin/sh' > $BODI_CHROOT_PATH/etc/init.d/${d} chmod +x $BODI_CHROOT_PATH/etc/init.d/${d} done -DEBIAN_FRONTEND=noninteractive chroot $BODI_CHROOT_PATH apt-get install -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' -y hadoop-doc hadoop-hdfs-namenode hadoop-httpfs hadoop-hdfs-datanode hadoop-mapreduce-historyserver hadoop-yarn-resourcemanager hadoop-yarn-nodemanager hbase-master hbase-regionserver hive hive-hbase hive-hcatalog hive-metastore hive-server2 $jdbc maven ant mariadb-client mariadb-common mariadb-server spark-core spark-history-server spark-python zookeeper-server +DEBIAN_FRONTEND=noninteractive chroot $BODI_CHROOT_PATH apt-get install -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' -y hadoop-doc hadoop-hdfs-namenode hadoop-httpfs hadoop-hdfs-datanode hadoop-mapreduce-historyserver hadoop-yarn-resourcemanager hadoop-yarn-nodemanager hbase-master hbase-regionserver hive hive-hbase hive-hcatalog hive-metastore hive-server2 krb5-admin-server krb5-kdc krb5-kpropd $jdbc maven ant mariadb-client mariadb-common mariadb-server spark-core spark-history-server spark-python zookeeper-server for d in $DAEMONS; do mv -v $BODI_CHROOT_PATH/etc/init.d/${d}.dpkg-dist $BODI_CHROOT_PATH/etc/init.d/${d} done @@ -43,6 +43,7 @@ cp -vp single.pp $BODI_CHROOT_PATH/root sed 's/\(\$hdfs_deployed\s*=\s*\).*/\1true/' single.pp > $BODI_CHROOT_PATH/root/single2.pp chroot $BODI_CHROOT_PATH touch -r /root/single.pp /root/single2.pp cp -vp scripts/*.sh $BODI_CHROOT_PATH/usr/local/sbin/ +cp -vp hadoop-adduser.sh $BODI_CHROOT_PATH/root/ # tests mkdir -p $BODI_CHROOT_PATH/opt/hadoop-tests diff --git a/image/tests/common.sh b/image/tests/common.sh index a4203f4..1fcd036 100644 --- a/image/tests/common.sh +++ b/image/tests/common.sh @@ -1,9 +1,10 @@ #! /bin/sh -e +REALM='HADOOP' HIVE_HOSTNAME=`hostname -f` HIVE_DB=`id -un`_test -JDBC_URL="jdbc:hive2://${HIVE_HOSTNAME}:10000/${HIVE_DB}" -#JDBC_URL="jdbc:hive2://${HIVE_HOSTNAME}:10000/${HIVE_DB};principal=hive/${HIVE_HOSTNAME}@${REALM}" +#JDBC_URL="jdbc:hive2://${HIVE_HOSTNAME}:10000/${HIVE_DB}" +JDBC_URL="jdbc:hive2://${HIVE_HOSTNAME}:10000/${HIVE_DB};principal=hive/${HIVE_HOSTNAME}@${REALM}" if test ! -f ~/.hadoop-test-`hostname -f`.txt; then hive -e "DROP DATABASE ${HIVE_DB} CASCADE" 2>/dev/null || : -- GitLab