Skip to content
Snippets Groups Projects
Commit d3725c87 authored by František Dvořák's avatar František Dvořák
Browse files

Hadoop: update context and orchestration according to the latest Hadoop image

parent 187eb967
No related branches found
No related tags found
No related merge requests found
...@@ -8,9 +8,41 @@ merge_type: ...@@ -8,9 +8,41 @@ merge_type:
packages: packages:
- default-jre-headless - default-jre-headless
- git
- gnupg - gnupg
- librarian-puppet
- puppet - puppet
write_files:
- path: /etc/puppet/code/Puppetfile
permissions: '0644'
content: |
#!/usr/bin/env ruby
#^syntax detection
forge "https://forgeapi.puppetlabs.com"
mod 'cesnet-site_hadoop',
:git => 'https://github.com/MetaCenterCloudPuppet/cesnet-site_hadoop/'
mod 'cesnet-hadoop',
:git => 'https://github.com/MetaCenterCloudPuppet/cesnet-hadoop/'
mod 'cesnet-hadoop_lib',
:git => 'https://github.com/MetaCenterCloudPuppet/cesnet-hadoop_lib/'
mod 'cesnet-hbase',
:git => 'https://github.com/MetaCenterCloudPuppet/cesnet-hbase/'
mod 'cesnet-hue',
:git => 'https://github.com/MetaCenterCloudPuppet/cesnet-hue/'
mod 'cesnet-hive',
:git => 'https://github.com/MetaCenterCloudPuppet/cesnet-hive/'
mod 'cesnet-impala',
:git => 'https://github.com/MetaCenterCloudPuppet/cesnet-impala/'
mod 'cesnet-oozie',
:git => 'https://github.com/MetaCenterCloudPuppet/cesnet-oozie/'
mod 'cesnet-pig',
:git => 'https://github.com/MetaCenterCloudPuppet/cesnet-pig/'
mod 'cesnet-spark',
:git => 'https://github.com/MetaCenterCloudPuppet/cesnet-spark/'
mod 'cesnet-zookeeper',
:git => 'https://github.com/MetaCenterCloudPuppet/cesnet-zookeeper/'
runcmd: runcmd:
- puppet module install cesnet/site_hadoop - cd /etc/puppet/code && HOME=/root librarian-puppet install --verbose
- mkdir /data - mkdir /data
$$distribution = '${distribution}' $$distribution = '${distribution}'
$$hdfs_deployed = ${hdfs_deployed} $$hdfs_deployed = ${hdfs_deployed}
$$realm = '${realm}'
$$ssl = false $$ssl = false
$$master = '${master_hostname}.${domain}' $$master = '${master_hostname}.${domain}'
...@@ -10,16 +12,26 @@ $$nodes = suffix(${nodes}, '.${domain}') ...@@ -10,16 +12,26 @@ $$nodes = suffix(${nodes}, '.${domain}')
$$zookeepers = [ $$zookeepers = [
$$master, $$master,
] ]
$$realm = '${realm}'
if $$distribution == 'bigtop' { if $$distribution == 'bigtop' {
$$version = '1.4.0' $$version = '1.5.0' # 1.4.0, 1.5.0
$$hadoop_version = 2 $$hadoop_version = 2
$$hive_schema_file = 'hive-schema-2.3.0.mysql.sql' $$oozie_version = 4
} elsif $$distribution == 'cloudera' { } elsif $$distribution == 'cloudera' {
$$version = '6.3.0' $$version = '6.3.0'
$$hadoop_version = 3 $$hadoop_version = 3
$$hive_schema_file = 'hive-schema-2.1.1.mysql.sql' $$oozie_version = 5
}
$$hive_schema_file = "$${distribution}-$${version}" ? {
'bigtop-1.4.0' => 'hive-schema-2.3.0.mysql.sql',
'bigtop-1.5.0' => 'hive-schema-2.3.0.mysql.sql',
'cloudera-6.3.0' => 'hive-schema-2.1.1.mysql.sql',
'cloudera-6.3.2' => 'hive-schema-2.1.1.mysql.sql',
default => undef, # stringify_facts=false required
}
$$db_type = "$${operatingsystem}-$${operatingsystemmajrelease}" ? {
'Debian-9' => 'mysql',
default => 'mariadb',
} }
class{'hadoop': class{'hadoop':
...@@ -84,21 +96,19 @@ class{'hive': ...@@ -84,21 +96,19 @@ class{'hive':
features => { features => {
'manager' => true, 'manager' => true,
}, },
#db => 'mariadb', db => $$db_type,
db => 'mysql',
db_password => 'good-password', db_password => 'good-password',
schema_file => $$hive_schema_file, schema_file => $$hive_schema_file,
} }
#class { 'oozie': #class { 'oozie':
# acl => true, # acl => true,
# #defaultFS => # db => $$db_type,
# hdfs_hostname => $$master,
# #db => 'mariadb',
# db => 'mysql',
# db_password => 'good-password', # db_password => 'good-password',
# oozie_hostname => "$$master-disabled", # oozie_hostname => $$master,
# oozie_sharelib => '/usr/lib/oozie/oozie-sharelib.tar.gz',
# realm => $$realm, # realm => $$realm,
# version => $$oozie_version,
#} #}
class { 'spark': class { 'spark':
...@@ -120,8 +130,9 @@ class{'site_hadoop': ...@@ -120,8 +130,9 @@ class{'site_hadoop':
distribution => $$distribution, distribution => $$distribution,
version => $$version, version => $$version,
users => [ users => [
'hawking', '${image_user}',
'example', 'example',
'hawking',
], ],
accounting_enable => false, accounting_enable => false,
hbase_enable => true, hbase_enable => true,
...@@ -139,7 +150,7 @@ node /${master_hostname}\..*/ { ...@@ -139,7 +150,7 @@ node /${master_hostname}\..*/ {
include ::hadoop::httpfs include ::hadoop::httpfs
class { 'mysql::bindings': class { 'mysql::bindings':
java_enable => true, java_enable => true,
#java_package_name => 'libmariadb-java', java_package_name => "lib$${db_type}-java",
} }
class { 'mysql::server': class { 'mysql::server':
root_password => 'root', root_password => 'root',
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment