From 97ae8684d37ad48b76f7203a690ea2f3f2ed9276 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz>
Date: Sun, 20 Dec 2020 17:52:08 +0100
Subject: [PATCH] Hadoop image: BigTop 1.5.0 + dynamic Debian versions support
Note: BitTop 1.5.0 requires Java < 11 ==> remaining at Debian 9 for now.
---
image/HOWTO.sh | 2 +-
image/hadoop.sh | 18 +++++++++++++++---
image/single.pp | 28 ++++++++++++++++++----------
3 files changed, 34 insertions(+), 14 deletions(-)
diff --git a/image/HOWTO.sh b/image/HOWTO.sh
index 50a44bf..d165d04 100755
--- a/image/HOWTO.sh
+++ b/image/HOWTO.sh
@@ -2,5 +2,5 @@
/usr/sbin/build-openstack-debian-image \
--release stretch \
--extra-packages acl,default-jre-headless,git,gnupg,librarian-puppet,qemu-guest-agent,puppet \
- --image-size 3 \
+ --image-size 4 \
--hook-script ./hadoop.sh 2>&1 | tee build-image.log
diff --git a/image/hadoop.sh b/image/hadoop.sh
index e7b1411..dcf4e5a 100755
--- a/image/hadoop.sh
+++ b/image/hadoop.sh
@@ -1,16 +1,28 @@
#! /bin/sh
+case "$BODI_RELEASE" in
+ stretch)
+ jdbc='libmysql-java'
+ os_maj=9
+ ver='1.5.0' # 1.4.0 1.5.0
+ ;;
+ *)
+ jdbc='libmariadb-java'
+ os_maj=10
+ ver='1.5.0' # 1.5.0
+ ;;
+esac
wget https://dist.apache.org/repos/dist/release/bigtop/KEYS -O - | chroot $BODI_CHROOT_PATH apt-key add -
cp -vp apt/*.pref $BODI_CHROOT_PATH/etc/apt/preferences.d/
cat <<EOF > $BODI_CHROOT_PATH/etc/apt/sources.list.d/bigtop.list
# initial setup
-deb http://repos.bigtop.apache.org/releases/1.4.0/debian/9/amd64 bigtop contrib
-deb-src http://repos.bigtop.apache.org/releases/1.4.0/debian/9/amd64 bigtop contrib
+deb http://repos.bigtop.apache.org/releases/${ver}/debian/${os_maj}/amd64 bigtop contrib
+deb-src http://repos.bigtop.apache.org/releases/${ver}/debian/${os_maj}/amd64 bigtop contrib
EOF
# download and pre-install
chroot $BODI_CHROOT_PATH apt-get update
chroot $BODI_CHROOT_PATH apt-get install -y hadoop hadoop-client hadoop-hdfs hadoop-mapreduce hadoop-yarn hbase hive-jdbc python-scipy zookeeper
-chroot $BODI_CHROOT_PATH apt-get install -dy hadoop-doc hadoop-hdfs-namenode hadoop-httpfs hadoop-hdfs-datanode hadoop-mapreduce-historyserver hadoop-yarn-resourcemanager hadoop-yarn-nodemanager hbase-master hbase-regionserver hive hive-hbase hive-hcatalog hive-metastore hive-server2 libmysql-java maven ant mariadb-client mariadb-common mariadb-server spark-core spark-history-server spark-python zookeeper-server
+chroot $BODI_CHROOT_PATH apt-get install -dy hadoop-doc hadoop-hdfs-namenode hadoop-httpfs hadoop-hdfs-datanode hadoop-mapreduce-historyserver hadoop-yarn-resourcemanager hadoop-yarn-nodemanager hbase-master hbase-regionserver hive hive-hbase hive-hcatalog hive-metastore hive-server2 $jdbc maven ant mariadb-client mariadb-common mariadb-server spark-core spark-history-server spark-python zookeeper-server
# setup
cp -vp Puppetfile $BODI_CHROOT_PATH/etc/puppet/code/
diff --git a/image/single.pp b/image/single.pp
index ad0a8ce..7164581 100644
--- a/image/single.pp
+++ b/image/single.pp
@@ -1,5 +1,7 @@
-$distribution = 'bigtop'
+$distribution = 'bigtop' # cloudera, bigtop
+
$hdfs_deployed = false
+$realm = ''
$ssl = false
$master = $::fqdn
@@ -10,16 +12,24 @@ $nodes = [$::fqdn]
$zookeepers = [
$master,
]
-$realm = ''
if $distribution == 'bigtop' {
- $version = '1.4.0'
+ $version = '1.5.0' # 1.4.0, 1.5.0
$hadoop_version = 2
- $hive_schema_file = 'hive-schema-2.3.0.mysql.sql'
} elsif $distribution == 'cloudera' {
$version = '6.3.0'
$hadoop_version = 3
- $hive_schema_file = 'hive-schema-2.1.1.mysql.sql'
+}
+$hive_schema_file = "${distribution}-${version}" ? {
+ 'bigtop-1.4.0' => 'hive-schema-2.3.0.mysql.sql',
+ 'bigtop-1.5.0' => 'hive-schema-2.3.0.mysql.sql',
+ 'cloudera-6.3.0' => 'hive-schema-2.1.1.mysql.sql',
+ 'cloudera-6.3.2' => 'hive-schema-2.1.1.mysql.sql',
+ default => undef, # stringify_facts=false required
+}
+$db_type = "${operatingsystem}-${operatingsystemmajrelease}" ? {
+ 'Debian-9' => 'mysql',
+ default => 'mariadb',
}
class{'hadoop':
@@ -84,8 +94,7 @@ class{'hive':
features => {
'manager' => true,
},
- #db => 'mariadb',
- db => 'mysql',
+ db => $db_type,
db_password => 'good-password',
schema_file => $hive_schema_file,
}
@@ -94,8 +103,7 @@ class{'hive':
# acl => true,
# #defaultFS =>
# hdfs_hostname => $master,
-# #db => 'mariadb',
-# db => 'mysql',
+# db => $db_type,
# db_password => 'good-password',
# oozie_hostname => "$master-disabled",
# realm => $realm,
@@ -139,7 +147,7 @@ include ::site_hadoop::role::simple
include ::hadoop::httpfs
class { 'mysql::bindings':
java_enable => true,
- #java_package_name => 'libmariadb-java',
+ java_package_name => "lib${db_type}-java",
}
class { 'mysql::server':
root_password => 'root',
--
GitLab