From c266172e4a484ffc5a912e9981ac7134062f5d9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz>
Date: Wed, 17 Mar 2021 09:08:52 +0100
Subject: [PATCH] Hadoop: better workaround of virtual memory limit for
spark-shell
---
deployments/hadoop-hdfs/site.pp.tmpl | 3 ++-
deployments/hadoop-single/single.pp.tmpl | 3 ++-
deployments/hadoop/site.pp.tmpl | 3 ++-
3 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/deployments/hadoop-hdfs/site.pp.tmpl b/deployments/hadoop-hdfs/site.pp.tmpl
index 8cf801c..f190f86 100644
--- a/deployments/hadoop-hdfs/site.pp.tmpl
+++ b/deployments/hadoop-hdfs/site.pp.tmpl
@@ -77,7 +77,8 @@ class{'hadoop':
#'hadoop.proxyuser.hive.groups' => "*",
'hadoop.proxyuser.hive.hosts' => "*",
'yarn.app.mapreduce.am.env' => 'LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:$$LD_LIBRARY_PATH',
- 'yarn.nodemanager.vmem-check-enabled' => 'false',
+ # increase virtual memory limit for Spark
+ 'yarn.nodemanager.vmem-pmem-ratio' => 5,
},
version => $$hadoop_version,
hdfs_deployed => $$hdfs_deployed,
diff --git a/deployments/hadoop-single/single.pp.tmpl b/deployments/hadoop-single/single.pp.tmpl
index b0a7a4b..db444db 100644
--- a/deployments/hadoop-single/single.pp.tmpl
+++ b/deployments/hadoop-single/single.pp.tmpl
@@ -101,7 +101,8 @@ class{'hadoop':
#'hadoop.proxyuser.hive.groups' => "*",
'hadoop.proxyuser.hive.hosts' => "*",
'yarn.app.mapreduce.am.env' => 'LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:$$LD_LIBRARY_PATH',
- 'yarn.nodemanager.vmem-check-enabled' => 'false',
+ # increase virtual memory limit for Spark
+ 'yarn.nodemanager.vmem-pmem-ratio' => 5,
},
version => $$hadoop_version,
hdfs_deployed => $$hdfs_deployed,
diff --git a/deployments/hadoop/site.pp.tmpl b/deployments/hadoop/site.pp.tmpl
index bf5c3dc..bb02113 100644
--- a/deployments/hadoop/site.pp.tmpl
+++ b/deployments/hadoop/site.pp.tmpl
@@ -99,7 +99,8 @@ class{'hadoop':
#'hadoop.proxyuser.hive.groups' => "*",
'hadoop.proxyuser.hive.hosts' => "*",
'yarn.app.mapreduce.am.env' => 'LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:$$LD_LIBRARY_PATH',
- 'yarn.nodemanager.vmem-check-enabled' => 'false',
+ # increase virtual memory limit for Spark
+ 'yarn.nodemanager.vmem-pmem-ratio' => 5,
},
version => $$hadoop_version,
hdfs_deployed => $$hdfs_deployed,
--
GitLab