diff --git a/hadoop/ctx.yaml b/hadoop/ctx.yaml index 9edcd625fa1ef0ff03599e146a7211b84cb51a3d..f9478250d49c922068f8834f24d6e640615bc756 100644 --- a/hadoop/ctx.yaml +++ b/hadoop/ctx.yaml @@ -8,9 +8,41 @@ merge_type: packages: - default-jre-headless + - git - gnupg + - librarian-puppet - puppet +write_files: + - path: /etc/puppet/code/Puppetfile + permissions: '0644' + content: | + #!/usr/bin/env ruby + #^syntax detection + forge "https://forgeapi.puppetlabs.com" + mod 'cesnet-site_hadoop', + :git => 'https://github.com/MetaCenterCloudPuppet/cesnet-site_hadoop/' + mod 'cesnet-hadoop', + :git => 'https://github.com/MetaCenterCloudPuppet/cesnet-hadoop/' + mod 'cesnet-hadoop_lib', + :git => 'https://github.com/MetaCenterCloudPuppet/cesnet-hadoop_lib/' + mod 'cesnet-hbase', + :git => 'https://github.com/MetaCenterCloudPuppet/cesnet-hbase/' + mod 'cesnet-hue', + :git => 'https://github.com/MetaCenterCloudPuppet/cesnet-hue/' + mod 'cesnet-hive', + :git => 'https://github.com/MetaCenterCloudPuppet/cesnet-hive/' + mod 'cesnet-impala', + :git => 'https://github.com/MetaCenterCloudPuppet/cesnet-impala/' + mod 'cesnet-oozie', + :git => 'https://github.com/MetaCenterCloudPuppet/cesnet-oozie/' + mod 'cesnet-pig', + :git => 'https://github.com/MetaCenterCloudPuppet/cesnet-pig/' + mod 'cesnet-spark', + :git => 'https://github.com/MetaCenterCloudPuppet/cesnet-spark/' + mod 'cesnet-zookeeper', + :git => 'https://github.com/MetaCenterCloudPuppet/cesnet-zookeeper/' + runcmd: - - puppet module install cesnet/site_hadoop + - cd /etc/puppet/code && HOME=/root librarian-puppet install --verbose - mkdir /data diff --git a/hadoop/site.pp.tmpl b/hadoop/site.pp.tmpl index db565f3233412094aa4b5fb6cb3c0d99e6603a1f..d93dc209140bca87ac977c7374d15ad218488a0c 100644 --- a/hadoop/site.pp.tmpl +++ b/hadoop/site.pp.tmpl @@ -1,5 +1,7 @@ $$distribution = '${distribution}' + $$hdfs_deployed = ${hdfs_deployed} +$$realm = '${realm}' $$ssl = false $$master = '${master_hostname}.${domain}' @@ -10,16 +12,26 @@ $$nodes = suffix(${nodes}, '.${domain}') $$zookeepers = [ $$master, ] -$$realm = '${realm}' if $$distribution == 'bigtop' { - $$version = '1.4.0' - $$hadoop_version = 2 - $$hive_schema_file = 'hive-schema-2.3.0.mysql.sql' + $$version = '1.5.0' # 1.4.0, 1.5.0 + $$hadoop_version = 2 + $$oozie_version = 4 } elsif $$distribution == 'cloudera' { - $$version = '6.3.0' - $$hadoop_version = 3 - $$hive_schema_file = 'hive-schema-2.1.1.mysql.sql' + $$version = '6.3.0' + $$hadoop_version = 3 + $$oozie_version = 5 +} +$$hive_schema_file = "$${distribution}-$${version}" ? { + 'bigtop-1.4.0' => 'hive-schema-2.3.0.mysql.sql', + 'bigtop-1.5.0' => 'hive-schema-2.3.0.mysql.sql', + 'cloudera-6.3.0' => 'hive-schema-2.1.1.mysql.sql', + 'cloudera-6.3.2' => 'hive-schema-2.1.1.mysql.sql', + default => undef, # stringify_facts=false required +} +$$db_type = "$${operatingsystem}-$${operatingsystemmajrelease}" ? { + 'Debian-9' => 'mysql', + default => 'mariadb', } class{'hadoop': @@ -84,21 +96,19 @@ class{'hive': features => { 'manager' => true, }, - #db => 'mariadb', - db => 'mysql', + db => $$db_type, db_password => 'good-password', schema_file => $$hive_schema_file, } #class { 'oozie': # acl => true, -# #defaultFS => -# hdfs_hostname => $$master, -# #db => 'mariadb', -# db => 'mysql', +# db => $$db_type, # db_password => 'good-password', -# oozie_hostname => "$$master-disabled", +# oozie_hostname => $$master, +# oozie_sharelib => '/usr/lib/oozie/oozie-sharelib.tar.gz', # realm => $$realm, +# version => $$oozie_version, #} class { 'spark': @@ -120,8 +130,9 @@ class{'site_hadoop': distribution => $$distribution, version => $$version, users => [ - 'hawking', + '${image_user}', 'example', + 'hawking', ], accounting_enable => false, hbase_enable => true, @@ -139,7 +150,7 @@ node /${master_hostname}\..*/ { include ::hadoop::httpfs class { 'mysql::bindings': java_enable => true, - #java_package_name => 'libmariadb-java', + java_package_name => "lib$${db_type}-java", } class { 'mysql::server': root_password => 'root',