Skip to content
Snippets Groups Projects
Commit 97ae8684 authored by František Dvořák's avatar František Dvořák
Browse files

Hadoop image: BigTop 1.5.0 + dynamic Debian versions support

Note: BitTop 1.5.0 requires Java < 11 ==> remaining at Debian 9 for now.
parent ef3b0a48
No related branches found
No related tags found
No related merge requests found
Pipeline #415 passed
......@@ -2,5 +2,5 @@
/usr/sbin/build-openstack-debian-image \
--release stretch \
--extra-packages acl,default-jre-headless,git,gnupg,librarian-puppet,qemu-guest-agent,puppet \
--image-size 3 \
--image-size 4 \
--hook-script ./hadoop.sh 2>&1 | tee build-image.log
#! /bin/sh
case "$BODI_RELEASE" in
stretch)
jdbc='libmysql-java'
os_maj=9
ver='1.5.0' # 1.4.0 1.5.0
;;
*)
jdbc='libmariadb-java'
os_maj=10
ver='1.5.0' # 1.5.0
;;
esac
wget https://dist.apache.org/repos/dist/release/bigtop/KEYS -O - | chroot $BODI_CHROOT_PATH apt-key add -
cp -vp apt/*.pref $BODI_CHROOT_PATH/etc/apt/preferences.d/
cat <<EOF > $BODI_CHROOT_PATH/etc/apt/sources.list.d/bigtop.list
# initial setup
deb http://repos.bigtop.apache.org/releases/1.4.0/debian/9/amd64 bigtop contrib
deb-src http://repos.bigtop.apache.org/releases/1.4.0/debian/9/amd64 bigtop contrib
deb http://repos.bigtop.apache.org/releases/${ver}/debian/${os_maj}/amd64 bigtop contrib
deb-src http://repos.bigtop.apache.org/releases/${ver}/debian/${os_maj}/amd64 bigtop contrib
EOF
# download and pre-install
chroot $BODI_CHROOT_PATH apt-get update
chroot $BODI_CHROOT_PATH apt-get install -y hadoop hadoop-client hadoop-hdfs hadoop-mapreduce hadoop-yarn hbase hive-jdbc python-scipy zookeeper
chroot $BODI_CHROOT_PATH apt-get install -dy hadoop-doc hadoop-hdfs-namenode hadoop-httpfs hadoop-hdfs-datanode hadoop-mapreduce-historyserver hadoop-yarn-resourcemanager hadoop-yarn-nodemanager hbase-master hbase-regionserver hive hive-hbase hive-hcatalog hive-metastore hive-server2 libmysql-java maven ant mariadb-client mariadb-common mariadb-server spark-core spark-history-server spark-python zookeeper-server
chroot $BODI_CHROOT_PATH apt-get install -dy hadoop-doc hadoop-hdfs-namenode hadoop-httpfs hadoop-hdfs-datanode hadoop-mapreduce-historyserver hadoop-yarn-resourcemanager hadoop-yarn-nodemanager hbase-master hbase-regionserver hive hive-hbase hive-hcatalog hive-metastore hive-server2 $jdbc maven ant mariadb-client mariadb-common mariadb-server spark-core spark-history-server spark-python zookeeper-server
# setup
cp -vp Puppetfile $BODI_CHROOT_PATH/etc/puppet/code/
......
$distribution = 'bigtop'
$distribution = 'bigtop' # cloudera, bigtop
$hdfs_deployed = false
$realm = ''
$ssl = false
$master = $::fqdn
......@@ -10,16 +12,24 @@ $nodes = [$::fqdn]
$zookeepers = [
$master,
]
$realm = ''
if $distribution == 'bigtop' {
$version = '1.4.0'
$version = '1.5.0' # 1.4.0, 1.5.0
$hadoop_version = 2
$hive_schema_file = 'hive-schema-2.3.0.mysql.sql'
} elsif $distribution == 'cloudera' {
$version = '6.3.0'
$hadoop_version = 3
$hive_schema_file = 'hive-schema-2.1.1.mysql.sql'
}
$hive_schema_file = "${distribution}-${version}" ? {
'bigtop-1.4.0' => 'hive-schema-2.3.0.mysql.sql',
'bigtop-1.5.0' => 'hive-schema-2.3.0.mysql.sql',
'cloudera-6.3.0' => 'hive-schema-2.1.1.mysql.sql',
'cloudera-6.3.2' => 'hive-schema-2.1.1.mysql.sql',
default => undef, # stringify_facts=false required
}
$db_type = "${operatingsystem}-${operatingsystemmajrelease}" ? {
'Debian-9' => 'mysql',
default => 'mariadb',
}
class{'hadoop':
......@@ -84,8 +94,7 @@ class{'hive':
features => {
'manager' => true,
},
#db => 'mariadb',
db => 'mysql',
db => $db_type,
db_password => 'good-password',
schema_file => $hive_schema_file,
}
......@@ -94,8 +103,7 @@ class{'hive':
# acl => true,
# #defaultFS =>
# hdfs_hostname => $master,
# #db => 'mariadb',
# db => 'mysql',
# db => $db_type,
# db_password => 'good-password',
# oozie_hostname => "$master-disabled",
# realm => $realm,
......@@ -139,7 +147,7 @@ include ::site_hadoop::role::simple
include ::hadoop::httpfs
class { 'mysql::bindings':
java_enable => true,
#java_package_name => 'libmariadb-java',
java_package_name => "lib${db_type}-java",
}
class { 'mysql::server':
root_password => 'root',
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment