diff --git a/image/HOWTO.sh b/image/HOWTO.sh index 50a44bf4f31ed652db9074051a551dd889bb0940..d165d04dccd8ea3563c1cd16426d296806c01bcd 100755 --- a/image/HOWTO.sh +++ b/image/HOWTO.sh @@ -2,5 +2,5 @@ /usr/sbin/build-openstack-debian-image \ --release stretch \ --extra-packages acl,default-jre-headless,git,gnupg,librarian-puppet,qemu-guest-agent,puppet \ - --image-size 3 \ + --image-size 4 \ --hook-script ./hadoop.sh 2>&1 | tee build-image.log diff --git a/image/hadoop.sh b/image/hadoop.sh index e7b1411969859dc9a21f36177cb40e0f1f6d6bb5..dcf4e5a8eeca5ba2f146974167401219a60b83ca 100755 --- a/image/hadoop.sh +++ b/image/hadoop.sh @@ -1,16 +1,28 @@ #! /bin/sh +case "$BODI_RELEASE" in + stretch) + jdbc='libmysql-java' + os_maj=9 + ver='1.5.0' # 1.4.0 1.5.0 + ;; + *) + jdbc='libmariadb-java' + os_maj=10 + ver='1.5.0' # 1.5.0 + ;; +esac wget https://dist.apache.org/repos/dist/release/bigtop/KEYS -O - | chroot $BODI_CHROOT_PATH apt-key add - cp -vp apt/*.pref $BODI_CHROOT_PATH/etc/apt/preferences.d/ cat <<EOF > $BODI_CHROOT_PATH/etc/apt/sources.list.d/bigtop.list # initial setup -deb http://repos.bigtop.apache.org/releases/1.4.0/debian/9/amd64 bigtop contrib -deb-src http://repos.bigtop.apache.org/releases/1.4.0/debian/9/amd64 bigtop contrib +deb http://repos.bigtop.apache.org/releases/${ver}/debian/${os_maj}/amd64 bigtop contrib +deb-src http://repos.bigtop.apache.org/releases/${ver}/debian/${os_maj}/amd64 bigtop contrib EOF # download and pre-install chroot $BODI_CHROOT_PATH apt-get update chroot $BODI_CHROOT_PATH apt-get install -y hadoop hadoop-client hadoop-hdfs hadoop-mapreduce hadoop-yarn hbase hive-jdbc python-scipy zookeeper -chroot $BODI_CHROOT_PATH apt-get install -dy hadoop-doc hadoop-hdfs-namenode hadoop-httpfs hadoop-hdfs-datanode hadoop-mapreduce-historyserver hadoop-yarn-resourcemanager hadoop-yarn-nodemanager hbase-master hbase-regionserver hive hive-hbase hive-hcatalog hive-metastore hive-server2 libmysql-java maven ant mariadb-client mariadb-common mariadb-server spark-core spark-history-server spark-python zookeeper-server +chroot $BODI_CHROOT_PATH apt-get install -dy hadoop-doc hadoop-hdfs-namenode hadoop-httpfs hadoop-hdfs-datanode hadoop-mapreduce-historyserver hadoop-yarn-resourcemanager hadoop-yarn-nodemanager hbase-master hbase-regionserver hive hive-hbase hive-hcatalog hive-metastore hive-server2 $jdbc maven ant mariadb-client mariadb-common mariadb-server spark-core spark-history-server spark-python zookeeper-server # setup cp -vp Puppetfile $BODI_CHROOT_PATH/etc/puppet/code/ diff --git a/image/single.pp b/image/single.pp index ad0a8cec80ba8e203d77e70d2c3e00e6ae2719a4..7164581e94435531a56e24506b32adfd5480aa88 100644 --- a/image/single.pp +++ b/image/single.pp @@ -1,5 +1,7 @@ -$distribution = 'bigtop' +$distribution = 'bigtop' # cloudera, bigtop + $hdfs_deployed = false +$realm = '' $ssl = false $master = $::fqdn @@ -10,16 +12,24 @@ $nodes = [$::fqdn] $zookeepers = [ $master, ] -$realm = '' if $distribution == 'bigtop' { - $version = '1.4.0' + $version = '1.5.0' # 1.4.0, 1.5.0 $hadoop_version = 2 - $hive_schema_file = 'hive-schema-2.3.0.mysql.sql' } elsif $distribution == 'cloudera' { $version = '6.3.0' $hadoop_version = 3 - $hive_schema_file = 'hive-schema-2.1.1.mysql.sql' +} +$hive_schema_file = "${distribution}-${version}" ? { + 'bigtop-1.4.0' => 'hive-schema-2.3.0.mysql.sql', + 'bigtop-1.5.0' => 'hive-schema-2.3.0.mysql.sql', + 'cloudera-6.3.0' => 'hive-schema-2.1.1.mysql.sql', + 'cloudera-6.3.2' => 'hive-schema-2.1.1.mysql.sql', + default => undef, # stringify_facts=false required +} +$db_type = "${operatingsystem}-${operatingsystemmajrelease}" ? { + 'Debian-9' => 'mysql', + default => 'mariadb', } class{'hadoop': @@ -84,8 +94,7 @@ class{'hive': features => { 'manager' => true, }, - #db => 'mariadb', - db => 'mysql', + db => $db_type, db_password => 'good-password', schema_file => $hive_schema_file, } @@ -94,8 +103,7 @@ class{'hive': # acl => true, # #defaultFS => # hdfs_hostname => $master, -# #db => 'mariadb', -# db => 'mysql', +# db => $db_type, # db_password => 'good-password', # oozie_hostname => "$master-disabled", # realm => $realm, @@ -139,7 +147,7 @@ include ::site_hadoop::role::simple include ::hadoop::httpfs class { 'mysql::bindings': java_enable => true, - #java_package_name => 'libmariadb-java', + java_package_name => "lib${db_type}-java", } class { 'mysql::server': root_password => 'root',