Skip to content
Snippets Groups Projects
Commit e12a7048 authored by František Dvořák's avatar František Dvořák
Browse files

Hadoop: experiments with HDFS-only flavor

parent 86729ba8
Branches
No related tags found
No related merge requests found
#cloud-config
merge_type:
- name: list
settings: [append]
- name: dict
settings: [recurse_array]
packages:
- default-jre-headless
- gnupg
- puppet
runcmd:
- puppet module install cesnet/site_hadoop
- mkdir /data
import hadoop.plugin
class Component(hadoop.plugin.ComponentHadoop):
pass
$$distribution = '${distribution}'
$$hdfs_deployed = ${hdfs_deployed}
$$ssl = false
$$master = '${master_hostname}.${domain}'
$$frontends = [
'${master_hostname}.${domain}',
]
$$nodes = suffix(${nodes}, '.${domain}')
$$zookeepers = [
$$master,
]
$$realm = '${realm}'
if $$distribution == 'bigtop' {
$$version = '1.4.0'
$$hadoop_version = 2
$$hive_schema_file = 'hive-schema-2.3.0.mysql.sql'
} elsif $$distribution == 'cloudera' {
$$version = '6.3.0'
$$hadoop_version = 3
$$hive_schema_file = 'hive-schema-2.1.1.mysql.sql'
}
class{'hadoop':
acl => true,
hdfs_hostname => $$master,
httpfs_hostnames => [
$$master,
],
frontends => $$frontends,
oozie_hostnames => [
$$master,
],
slaves => $$nodes,
zookeeper_hostnames => $$zookeepers,
hdfs_name_dirs => [
'/data',
],
hdfs_data_dirs => [
'/data',
],
cluster_name => '${domain}',
https => $$ssl,
realm => $$realm,
features => {
'yellowmanager' => true,
'aggregation' => true,
},
properties => {
'dfs.replication' => 2,
'hadoop.proxyuser.hive.groups' => "hive,impala,oozie,users",
'hadoop.proxyuser.hive.hosts' => "*",
},
version => $$hadoop_version,
hdfs_deployed => $$hdfs_deployed,
}
class{'hive':
hdfs_hostname => $$master,
metastore_hostname => $$master,
server2_hostname => $$master,
zookeeper_hostnames => $$zookeepers,
realm => $$realm,
features => {
'manager' => true,
},
#db => 'mariadb',
db => 'mysql',
db_password => 'good-password',
schema_file => $$hive_schema_file,
}
class { 'spark':
historyserver_hostname => $$master,
environment => {
'LD_LIBRARY_PATH' => '/usr/lib/hadoop/lib/native:$${LD_LIBRARY_PATH}',
'SPARK_YARN_USER_ENV' => 'LD_LIBRARY_PATH=$${LD_LIBRARY_PATH},$${SPARK_YARN_USER_ENV}',
},
#jar_enable => true,
realm => $$realm,
}
class { '::zookeeper':
hostnames => $$zookeepers,
realm => $$realm,
}
class{'site_hadoop':
distribution => $$distribution,
version => $$version,
users => [
'hawking',
'example',
],
accounting_enable => false,
hbase_enable => false,
nfs_frontend_enable => false,
oozie_enable => false,
pig_enable => false,
spark_enable => true,
}
node /${master_hostname}\..*/ {
include ::site_hadoop::role::master_hdfs
include ::site_hadoop::role::frontend
include ::hadoop::httpfs
class { 'mysql::bindings':
java_enable => true,
#java_package_name => 'libmariadb-java',
}
class { 'mysql::server':
root_password => 'root',
}
}
node /${node_hostname}\d*\..*/ {
include ::site_hadoop::role::slave
}
......@@ -3,10 +3,11 @@ import string
DEFAULT_DISTRIBUTION = 'bigtop'
class Component:
class ComponentHadoop:
def __init__(self, args, config, hosts):
self.args = args
self.config = config
self.params = {
'distribution': DEFAULT_DISTRIBUTION,
'domain': config['domain'],
......@@ -23,7 +24,7 @@ class Component:
print('-> site.pp')
if not self.args.dry_run:
template = None
with open('hadoop/site.pp.tmpl', 'r') as f:
with open('%s/site.pp.tmpl' % self.config['type'], 'r') as f:
template = string.Template(f.read())
if template:
site = template.substitute(self.params)
......@@ -34,3 +35,7 @@ class Component:
return [
['ansible', '-i', './inventory', '-m', 'copy', '-a', 'src=site.pp dest=/root', 'all'],
]
class Component(ComponentHadoop):
pass
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment