From 8566a7323311659a72ca604e1b9e2b06824361ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz> Date: Mon, 21 Dec 2020 23:57:53 +0100 Subject: [PATCH] Hadoop image: speed up - install everything with disabled daemons --- image/hadoop.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/image/hadoop.sh b/image/hadoop.sh index f817e85..ec9f829 100755 --- a/image/hadoop.sh +++ b/image/hadoop.sh @@ -1,16 +1,20 @@ #! /bin/sh case "$BODI_RELEASE" in stretch) + db_daemon='mysql' jdbc='libmysql-java' os_maj=9 ver='1.5.0' # 1.4.0 1.5.0 ;; *) + db_daemon='mariadb' jdbc='libmariadb-java' os_maj=10 ver='1.5.0' # 1.5.0 ;; esac +DAEMONS="$db_daemon hadoop-hdfs-datanode hadoop-hdfs-namenode hadoop-httpfs hadoop-mapreduce-historyserver hadoop-yarn-nodemanager hadoop-yarn-resourcemanager hbase-master hbase-regionserver hive-metastore hive-server2 spark-history-server zookeeper-server" + wget https://dist.apache.org/repos/dist/release/bigtop/KEYS -O - | chroot $BODI_CHROOT_PATH apt-key add - cp -vp apt/*.pref $BODI_CHROOT_PATH/etc/apt/preferences.d/ cat <<EOF > $BODI_CHROOT_PATH/etc/apt/sources.list.d/bigtop.list @@ -22,7 +26,15 @@ EOF # download and pre-install chroot $BODI_CHROOT_PATH apt-get update chroot $BODI_CHROOT_PATH apt-get install -y hadoop hadoop-client hadoop-hdfs hadoop-mapreduce hadoop-yarn hbase hive-jdbc python-scipy zookeeper -chroot $BODI_CHROOT_PATH apt-get install -dy hadoop-doc hadoop-hdfs-namenode hadoop-httpfs hadoop-hdfs-datanode hadoop-mapreduce-historyserver hadoop-yarn-resourcemanager hadoop-yarn-nodemanager hbase-master hbase-regionserver hive hive-hbase hive-hcatalog hive-metastore hive-server2 $jdbc maven ant mariadb-client mariadb-common mariadb-server spark-core spark-history-server spark-python zookeeper-server +for d in $DAEMONS; do + echo '#! /bin/sh' > $BODI_CHROOT_PATH/etc/init.d/${d} + chmod +x $BODI_CHROOT_PATH/etc/init.d/${d} +done +DEBIAN_FRONTEND=noninteractive chroot $BODI_CHROOT_PATH apt-get install -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' -y hadoop-doc hadoop-hdfs-namenode hadoop-httpfs hadoop-hdfs-datanode hadoop-mapreduce-historyserver hadoop-yarn-resourcemanager hadoop-yarn-nodemanager hbase-master hbase-regionserver hive hive-hbase hive-hcatalog hive-metastore hive-server2 $jdbc maven ant mariadb-client mariadb-common mariadb-server spark-core spark-history-server spark-python zookeeper-server +for d in $DAEMONS; do + mv -v $BODI_CHROOT_PATH/etc/init.d/${d}.dpkg-dist $BODI_CHROOT_PATH/etc/init.d/${d} +done +chroot $BODI_CHROOT_PATH systemctl disable ${DAEMONS} # setup cp -vp Puppetfile $BODI_CHROOT_PATH/etc/puppet/code/ -- GitLab