How to use the yarn.scheduler.maximum-allocation-mb function in yarn

To help you get started, we’ve selected a few yarn examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github apache / ambari / ambari-web / app / utils / configs / defaults_providers / hive_defaults_provider.js View on Github external
getDefaults: function (localDB) {
    var configs = this._super(localDB);
    if (configs['yarn.scheduler.maximum-allocation-mb'] != null && configs['mapreduce.map.memory.mb'] != null
      && configs['mapreduce.reduce.memory.mb'] != null) {
      var containerSize = configs['mapreduce.map.memory.mb'] > 2048 ? configs['mapreduce.map.memory.mb'] : configs['mapreduce.reduce.memory.mb'];
      containerSize = Math.min(configs['yarn.scheduler.maximum-allocation-mb'], containerSize);
      configs['hive.auto.convert.join.noconditionaltask.size'] = Math.round(containerSize / 3) * 1048576; // MB to Bytes
      configs['hive.tez.java.opts'] = "-server -Xmx" + Math.round(0.8 * containerSize) + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC";
      configs['hive.tez.container.size'] = containerSize;
    } else {
      jQuery.extend(configs, this.get('configsTemplate'));
    }
    return configs;
  }
github apache / ambari / ambari-web / app / utils / configs / defaults_providers / hive_defaults_provider.js View on Github external
getDefaults: function (localDB) {
    var configs = this._super(localDB);
    if (configs['yarn.scheduler.maximum-allocation-mb'] != null && configs['mapreduce.map.memory.mb'] != null
      && configs['mapreduce.reduce.memory.mb'] != null) {
      var containerSize = configs['mapreduce.map.memory.mb'] > 2048 ? configs['mapreduce.map.memory.mb'] : configs['mapreduce.reduce.memory.mb'];
      containerSize = Math.min(configs['yarn.scheduler.maximum-allocation-mb'], containerSize);
      configs['hive.auto.convert.join.noconditionaltask.size'] = Math.round(containerSize / 3) * 1048576; // MB to Bytes
      configs['hive.tez.java.opts'] = "-server -Xmx" + Math.round(0.8 * containerSize) + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC";
      configs['hive.tez.container.size'] = containerSize;
    } else {
      jQuery.extend(configs, this.get('configsTemplate'));
    }
    return configs;
  }
github apache / ambari / ambari-web / app / utils / configs / defaults_providers / yarn_defaults_provider.js View on Github external
getDefaults: function (localDB) {
    this._super();
    this.getClusterData(localDB);
    var configs = {};
    jQuery.extend(configs, this.get('configsTemplate'));
    if (!this.clusterDataIsValid()) {
      return configs;
    }
    configs['yarn.nodemanager.resource.memory-mb'] = Math.round(this.get('containers') * this.get('ramPerContainer'));
    configs['yarn.scheduler.minimum-allocation-mb'] = Math.floor(this.get('ramPerContainer'));
    configs['yarn.scheduler.maximum-allocation-mb'] = Math.round(this.get('containers') * this.get('ramPerContainer'));
    configs['yarn.app.mapreduce.am.resource.mb'] = Math.floor(this.get('amMemory'));
    configs['yarn.app.mapreduce.am.command-opts'] = "-Xmx" + Math.round(0.8 * this.get('amMemory')) + "m";
    configs['mapreduce.map.memory.mb'] = Math.floor(this.get('mapMemory'));
    configs['mapreduce.reduce.memory.mb'] = Math.floor(this.get('reduceMemory'));
    configs['mapreduce.map.java.opts'] = "-Xmx" + Math.round(0.8 * this.get('mapMemory')) + "m";
    configs['mapreduce.reduce.java.opts'] = "-Xmx" + Math.round(0.8 * this.get('reduceMemory')) + "m";
    configs['mapreduce.task.io.sort.mb'] = Math.round(Math.min(0.4 * this.get('mapMemory'), 1024));
    return configs;
  },
github apache / incubator-ambari / ambari-web / app / utils / configs / defaults_providers / yarn_defaults_provider.js View on Github external
getDefaults: function (localDB) {
    this._super();
    this.getClusterData(localDB);
    var configs = {};
    jQuery.extend(configs, this.get('configsTemplate'));
    if (!this.clusterDataIsValid()) {
      return configs;
    }
    configs['yarn.nodemanager.resource.memory-mb'] = Math.round(this.get('containers') * this.get('ramPerContainer'));
    configs['yarn.scheduler.minimum-allocation-mb'] = Math.round(this.get('ramPerContainer'));
    configs['yarn.scheduler.maximum-allocation-mb'] = Math.round(this.get('containers') * this.get('ramPerContainer'));
    configs['yarn.app.mapreduce.am.resource.mb'] = Math.round(this.get('amMemory'));
    configs['yarn.app.mapreduce.am.command-opts'] = "-Xmx" + Math.round(0.8 * this.get('amMemory')) + "m";
    configs['mapreduce.map.memory.mb'] = Math.round(this.get('mapMemory'));
    configs['mapreduce.reduce.memory.mb'] = Math.round(this.get('reduceMemory'));
    configs['mapreduce.map.java.opts'] = "-Xmx" + Math.round(0.8 * this.get('mapMemory')) + "m";
    configs['mapreduce.reduce.java.opts'] = "-Xmx" + Math.round(0.8 * this.get('reduceMemory')) + "m";
    configs['mapreduce.task.io.sort.mb'] = Math.round(0.4 * this.get('mapMemory'));
    return configs;
  },