hadoop.sh

#! /bin/sh
case "$BODI_RELEASE" in
	stretch)
		db_daemon='mysql'
		jdbc='libmysql-java'
		os_maj=9
		ver='1.5.0' # 1.4.0 1.5.0
		;;
	*)
		db_daemon='mariadb'
		jdbc='libmariadb-java'
		os_maj=10
		ver='3.0.0' # 1.5.0, 3.0.0
		;;
esac
DAEMONS="$db_daemon krb5-admin-server krb5-kdc krb5-kpropd hadoop-hdfs-datanode hadoop-hdfs-namenode hadoop-httpfs hadoop-mapreduce-historyserver hadoop-yarn-nodemanager hadoop-yarn-resourcemanager hbase-master hbase-regionserver hive-metastore hive-server2 spark-history-server zookeeper-server"

# workaround for bigtop-utils jsvc
d="$BODI_CHROOT_PATH/usr/lib/jvm/default-java/lib"; if test -e "$d" -a ! -e "$d/amd64"; then ln -fsv . "$d/amd64"; fi

wget https://dist.apache.org/repos/dist/release/bigtop/KEYS -O - | chroot $BODI_CHROOT_PATH apt-key add -
cp -vp apt/*.pref $BODI_CHROOT_PATH/etc/apt/preferences.d/
cat <<EOF > $BODI_CHROOT_PATH/etc/apt/sources.list.d/bigtop.list
# FIXME: local mirror instead of apache
# initial setup (local mirror)
#deb http://scientific.zcu.cz/repos/hadoop/bigtop/${ver}/debian/${os_maj}/amd64 bigtop contrib
#deb-src http://scientific.zcu.cz/repos/hadoop/bigtop/${ver}/debian/${os_maj}/amd64 bigtop contrib
deb http://repos.bigtop.apache.org/releases/${ver}/debian/${os_maj}/amd64 bigtop contrib
deb-src http://repos.bigtop.apache.org/releases/${ver}/debian/${os_maj}/amd64 bigtop contrib
EOF

# download and pre-install
chroot $BODI_CHROOT_PATH apt-get update
chroot $BODI_CHROOT_PATH apt-get install -dy ansible fail2ban less man mc puppet-master rsync vim wget
chroot $BODI_CHROOT_PATH apt-get install -y hadoop hadoop-client hadoop-hdfs hadoop-mapreduce hadoop-yarn hbase hive-jdbc python-scipy zookeeper
for d in $DAEMONS; do
	echo '#! /bin/sh' > $BODI_CHROOT_PATH/etc/init.d/${d}
	chmod +x $BODI_CHROOT_PATH/etc/init.d/${d}
done
DEBIAN_FRONTEND=noninteractive chroot $BODI_CHROOT_PATH apt-get install -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' -y hadoop-doc hadoop-hdfs-namenode hadoop-httpfs hadoop-hdfs-datanode hadoop-mapreduce-historyserver hadoop-yarn-resourcemanager hadoop-yarn-nodemanager hbase-master hbase-regionserver hive hive-hbase hive-hcatalog hive-metastore hive-server2 krb5-admin-server krb5-kdc krb5-kpropd $jdbc maven ant mariadb-client mariadb-common mariadb-server spark-core spark-history-server spark-python zookeeper-server
for d in $DAEMONS; do
	mv -v $BODI_CHROOT_PATH/etc/init.d/${d}.dpkg-dist $BODI_CHROOT_PATH/etc/init.d/${d}
done
chroot $BODI_CHROOT_PATH systemctl disable ${DAEMONS}
cat <<EOF > $BODI_CHROOT_PATH/etc/apt/sources.list.d/bigtop.list
# initial setup
deb http://repos.bigtop.apache.org/releases/${ver}/debian/${os_maj}/amd64 bigtop contrib
deb-src http://repos.bigtop.apache.org/releases/${ver}/debian/${os_maj}/amd64 bigtop contrib
EOF

# tar archiver
# wget https://gitlab.cesnet.cz/702/HADOOP/tar/-/jobs/1259/artifacts/raw/target/hadoop-tar-2.0.0.jar -P $BODI_CHROOT_PATH/usr/local/share/
wget --timestamping https://maven.civ.zcu.cz/repository/maven-releases/org/apache/hadoop/tar/hadoop-tar/2.0.0/hadoop-tar-2.0.0.jar -P $BODI_CHROOT_PATH/usr/local/share/
ln -sfv hadoop-tar-2.0.0.jar $BODI_CHROOT_PATH/usr/local/share/hadoop-tar.jar

# setup
cp -vp Puppetfile $BODI_CHROOT_PATH/etc/puppet/code/
chroot $BODI_CHROOT_PATH bash -c 'cd /etc/puppet/code; librarian-puppet install'
cp -vp hadoop-adduser.sh $BODI_CHROOT_PATH/usr/local/sbin/
cp -vp scripts/*.sh $BODI_CHROOT_PATH/usr/local/sbin/
mkdir -p $BODI_CHROOT_PATH/etc/puppet/code/environments/production/manifests
install -v -m 0600 ../site.pp $BODI_CHROOT_PATH/etc/puppet/code/environments/production/manifests/
chroot $BODI_CHROOT_PATH chown puppet /etc/puppet/code/environments/production/manifests/site.pp
cp -vp motd $BODI_CHROOT_PATH/etc/motd
egrep '^\s*GSSAPIAuthentication\s+yes' $BODI_CHROOT_PATH/etc/ssh/sshd_config || echo 'GSSAPIAuthentication yes' >> $BODI_CHROOT_PATH/etc/ssh/sshd_config

# tests
mkdir -p $BODI_CHROOT_PATH/opt/hadoop-tests
cp -vp tests/*.sh $BODI_CHROOT_PATH/opt/hadoop-tests/