Hadoop: better workaround of virtual memory limit for spark-shell

c266172e · František Dvořák · 57437b8f · c266172e · c266172e · c266172e
Commit c266172e authored Mar 17, 2021 by František Dvořák
--- a/deployments/hadoop-hdfs/site.pp.tmpl
+++ b/deployments/hadoop-hdfs/site.pp.tmpl
@@ -77,7 +77,8 @@ class{'hadoop':
    #'hadoop.proxyuser.hive.groups' => "*",
    'hadoop.proxyuser.hive.hosts' => "*",
    'yarn.app.mapreduce.am.env' => 'LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:$$LD_LIBRARY_PATH',
-    'yarn.nodemanager.vmem-check-enabled' => 'false',
+    # increase virtual memory limit for Spark
+    'yarn.nodemanager.vmem-pmem-ratio' => 5,
  },
  version                => $$hadoop_version,
  hdfs_deployed          => $$hdfs_deployed,

--- a/deployments/hadoop-single/single.pp.tmpl
+++ b/deployments/hadoop-single/single.pp.tmpl
@@ -101,7 +101,8 @@ class{'hadoop':
    #'hadoop.proxyuser.hive.groups' => "*",
    'hadoop.proxyuser.hive.hosts' => "*",
    'yarn.app.mapreduce.am.env' => 'LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:$$LD_LIBRARY_PATH',
-    'yarn.nodemanager.vmem-check-enabled' => 'false',
+    # increase virtual memory limit for Spark
+    'yarn.nodemanager.vmem-pmem-ratio' => 5,
  },
  version                => $$hadoop_version,
  hdfs_deployed          => $$hdfs_deployed,

--- a/deployments/hadoop/site.pp.tmpl
+++ b/deployments/hadoop/site.pp.tmpl
@@ -99,7 +99,8 @@ class{'hadoop':
    #'hadoop.proxyuser.hive.groups' => "*",
    'hadoop.proxyuser.hive.hosts' => "*",
    'yarn.app.mapreduce.am.env' => 'LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:$$LD_LIBRARY_PATH',
-    'yarn.nodemanager.vmem-check-enabled' => 'false',
+    # increase virtual memory limit for Spark
+    'yarn.nodemanager.vmem-pmem-ratio' => 5,
  },
  version                => $$hadoop_version,
  hdfs_deployed          => $$hdfs_deployed,