site.pp.tmpl

$$distribution = '${distribution}' # cloudera, bigtop

$$hdfs_deployed = ${hdfs_deployed}
$$realm = '${realm}'
$$ssl = false

$$master = '${master_hostname}.${domain}'
$$frontends = [
  '${master_hostname}.${domain}',
]
$$nodes = suffix(${nodes}, '.${domain}')
$$zookeepers = [
  $$master,
]

if $$distribution == 'bigtop' {
  $$version = '1.5.0' # 1.4.0, 1.5.0
  $$hadoop_version = 2
} elsif $$distribution == 'cloudera' {
  $$version = '6.3.0'
  $$hadoop_version = 3
}

$$principals = suffix(concat(
  prefix(concat([$$master], $$nodes), 'host/'),
  prefix(concat([$$master], $$nodes), 'HTTP/'),
  ["httpfs/$$master"],
  prefix($$nodes, 'dn/'),
  ["nfs/$$master"],
  ["nn/$$master"],
  ["zookeeper/$$master"]
), "@$${realm}")

stage { 'kerberos':
  before => Stage['main'],
}

class{"kerberos":
  kadmin_hostname    => $$master,
  admin_principal    => "puppet/admin@$${realm}",
  admin_password     => '$kerberos_admin_password',
  master_password    => '$kerberos_master_password',
  realm              => $$realm,
  default_attributes => {
    'requires_preauth' => true,
  },
  default_policy     => 'default_host',
  stage              => 'kerberos',
}

class{'hadoop':
  acl                    => true,
  hdfs_hostname          => $$master,
  httpfs_hostnames       => [
    $$master,
  ],
  frontends              => $$frontends,
  oozie_hostnames        => [
    $$master,
  ],
  slaves                 => $$nodes,
  zookeeper_hostnames    => $$zookeepers,
  hdfs_name_dirs         => [
    '/data',
  ],
  hdfs_data_dirs         => $data_dirs,
  cluster_name           => '${domain}',
  https                  => $$ssl,
  realm                  => $$realm,
  features               => {
    'yellowmanager' => true,
    'aggregation'   => true,
  },
  properties             => {
    'dfs.replication' => 2,
    'hadoop.proxyuser.hive.groups' => "hive,impala,oozie,users",
    #'hadoop.proxyuser.hive.groups' => "*",
    'hadoop.proxyuser.hive.hosts' => "*",
    'yarn.app.mapreduce.am.env' => 'LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:$$LD_LIBRARY_PATH',
    # increase virtual memory limit for Spark
    'yarn.nodemanager.vmem-pmem-ratio' => 5,
  },
  version                => $$hadoop_version,
  hdfs_deployed          => $$hdfs_deployed,
}

class { '::zookeeper':
  hostnames => $$zookeepers,
  realm     => $$realm,
}

class{'site_hadoop':
  distribution        => $$distribution,
  version             => $$version,
  accounting_enable   => false,
  hbase_enable        => false,
  hive_enable         => false,
  nfs_frontend_enable => false,
  oozie_enable        => false,
  pig_enable          => false,
  spark_enable        => false,
}

group{$image_user:
  ensure => 'present',
}
->
user{$image_user:
  gid        => $image_user,
  groups     => ['users'],
  managehome => true,
  shell      => '/bin/bash',
}

class local_kerberos {
  file{'/etc/security/keytab':
    ensure => 'directory',
    owner  => 'root',
    group  => 'root',
    mode   => '0755',
  }

  File['/etc/security/keytab'] -> Kerberos::Keytab <| |>

  file{'/etc/security/http-auth-signature-secret':
    content => '$http_signature_secret',
    mode    => '0600',
    owner   => 'root',
    group   => 'root',
  }
}

class local_kerberos_master {
  include local_kerberos

  kerberos::policy{'default':
    ensure    => 'present',
    minlength => 6,
    history   => 2,
  }

  kerberos::policy{'default_host':
    ensure    => 'present',
    minlength => 6,
  }

  kerberos::principal{$$::kerberos::admin_principal:
    ensure   => 'present',
    password => $$::kerberos::admin_password,
  }

  kerberos::principal{$$principals:}

  kerberos::keytab{'/etc/krb5.keytab':
    principals => ["host/$${::fqdn}@$${realm}"],
  }
  kerberos::keytab{'/etc/security/keytab/http.service.keytab':
    principals => ["HTTP/$${::fqdn}@$${realm}"],
  }
  kerberos::keytab{'/etc/security/keytab/httpfs.service.keytab':
    principals => ["httpfs/$${::fqdn}@$${realm}"],
  }
  # works only locally on Kerberos admin server!
  kerberos::keytab{'/etc/security/keytab/httpfs-http.service.keytab':
    principals => [
      "httpfs/$${::fqdn}@$${realm}",
      "HTTP/$${::fqdn}@$${realm}",
    ],
  }
  kerberos::keytab{'/etc/security/keytab/nfs.service.keytab':
    principals => ["nfs/$${::fqdn}@$${realm}"],
  }
  kerberos::keytab{'/etc/security/keytab/nn.service.keytab':
    principals => ["nn/$${::fqdn}@$${realm}"],
  }
  kerberos::keytab{'/etc/security/keytab/zookeeper.service.keytab':
    principals => ["zookeeper/$${::fqdn}@$${realm}"],
  }
}

class local_kerberos_node {
  include local_kerberos

  # this will use kerberos::admin_principal and kerberos::admin_password parameters
  kerberos::keytab{'/etc/krb5.keytab':
    principals => ["host/$${::fqdn}@$${realm}"],
    wait       => 600,
  }
  kerberos::keytab{'/etc/security/keytab/dn.service.keytab':
    principals => ["dn/$${::fqdn}@$${realm}"],
    wait       => 600,
  }
  kerberos::keytab{'/etc/security/keytab/http.service.keytab':
    principals => ["HTTP/$${::fqdn}@$${realm}"],
    wait       => 600,
  }
}

node /${master_hostname}\..*/ {
  include ::site_hadoop::role::master_hdfs
  include ::site_hadoop::role::frontend
  include ::hadoop::httpfs

  class{'local_kerberos_master':
    stage => 'kerberos',
  }
}

node /${node_hostname}\d*\..*/ {
  include ::site_hadoop::role::slave

  class{'local_kerberos_node':
    stage => 'kerberos',
  }
}