Question

环境：

Hadoop 3.0.0
1 NameNode，5 DataNode

我在mapred-site.yml配置设置如下，仅限制同时运行的3地图任务：

<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
</property>
<property>
    <name>mapreduce.tasktracker.map.tasks.maximum</name>
    <value>3</value>
    <description>The maximum number of map tasks that will be run simultaneously by a task tracker.</description>
</property>
<property>
    <name>mapreduce.tasktracker.reduce.tasks.maximum</name>
    <value>3</value>
    <description>The maximum number of reduce tasks that will be run simultaneously by a task tracker.</description>
</property>

但是当我使用以下命令运行TestDFSIO基准测试时，最大实际运行的映射任务是8，似乎该设置不起作用：

yarn jar /opt/hadoop-3.0.0/share/hadoop/mapreduce/hadoop-mapreduce- 
client-jobclient-3.0.0-tests.jar \
TestDFSIO -storagePolicy HOT -write \
-nrFiles 500 -fileSize 1000MB -resFile /tmp/DFSIO-write.out

任何帮助将不胜感激。

Answer 1

该配置参数来自旧的Hadoop 1.x.据我所知，你使用的是3.0.0。试试这个：

var RippleAPI = require('ripple-lib').RippleAPI

var api = new RippleAPI({ server: 'wss://s.altnet.rippletest.net:51233' 
}) 

var myAddress = 'rhMhKToAwTyrSgeRXRUispp6hNGaR7Am5a'
var mySecret  = 'shFGQDTPdHSA9mKgATM4seuRoajF7'

api.on('error', function (errorCode, errorMessage) {
console.log(errorCode + ': ' + errorMessage)
})
 api.on('connected', function () {
console.log('connected')


console.log('getting account info for', myAddress)
api.getAccountInfo(myAddress).then(function(info){
console.log('getAccountInfo done, info: ', info)

var serverstate = api.getServerInfo().then(function (ss) {

  console.log('getServerState', ss)

  var _fee = (ss.validatedLedger.baseFeeXRP*1000*1000)+""

  api.getFee().then(function(e){
    console.log('Estimated fee= ', parseFloat(e)*1000*1000)
  })

  if(parseInt(_fee) > 12){
    _fee = 12
  }

  var transaction = {
      "TransactionType" : "Payment",
      "Account" : myAddress,
      "Fee" : _fee,
      "Destination" : "rLarfrfDe7fqtoVjk9HYS1aUENPekVEwEr",
      "DestinationTag" : 133,
      "Amount" : (1*100*1000)+ "99",
      "LastLedgerSequence" : ss.validatedLedger.ledgerVersion+4,
      "Sequence" : info.sequence
       //"Amount" : {
         //"currency" : "XRP",
         //"issuer" : "r4vEs94PiqBbPvDjqgZiomEdExeYStPt3r",
         //"value" : (1*1000*1000)+"50"
      // }
    }

   // transaction = {"tx_json" : transaction }

   console.log('Transaction: ', transaction)

   var txJSON = JSON.stringify(transaction)
   console.log(txJSON)

   var transactionSigned = api.sign(txJSON,mySecret)

   console.log('Signed Transaction: ', transactionSigned)
   // console.log(transactionSigned.signedTransaction)
   // transactionSigned.id (eg 884A6C2F0340EFAC231AAB20627C58CB9890EDA66E2FEA9B175BB61FE3CA2916)
   // = required to check details

   api.submit(transactionSigned.signedTransaction).then(function(data){
      console.log(data)

      console.log('Tentative Result: ', data.resultCode);
      console.log('Tentative Message: ', data.resultMessage);


      var checkTransactionStatus = setInterval(function(){
          console.log('Checking Transaction Resutls')

           api.getLedgerVersion().then(function(d){
            var ledgerVersion = parseInt(d)
            console.log('Ledger = @ version ', ledgerVersion)

            if(ledgerVersion > ss.validatedLedger.ledgerVersion && ledgerVersion <= ss.validatedLedger.ledgerVersion+4){
              console.log('... getting transaction ... ')

                 api.getTransaction(transactionSigned.id, {
                    minLedgerVersion: ss.validatedLedger.ledgerVersion,
                    maxLedgerVersion: ledgerVersion
                 }).then(function(d){

                    clearInterval(checkTransactionStatus)
                    console.log('<<<<<< getTransaction results: >>>>')
                    // console.dir(d, { depth: null })
                    console.dir(d.outcome, { depth: null })

                 }).catch(function(e){
                  console.log('Error getting Transaction: ', e)
                 })

            }
            if(ledgerVersion > ss.validatedLedger.ledgerVersion+4){
              console.log('>>>>>> EXPIRED <<<<<<<')
              clearInterval(checkTransactionStatus)
            }

           })

      }, 1000);

    }).catch(console.error);

  }).catch(console.error)
  }).catch(console.error)
})

api.on('disconnected', function (code) {  

console.log('disconnected, code: ', code)   
})

 api.connect()

 setTimeout(function(){
   api.disconnect()
 }, 100*1000)

您应该在运行NodeManager的每台主机上的yarn-site.xml中设置它。

如何限制映射任务的数量将在每个DataNode上同时运行

1 个答案: