Question

Hadoop作业停留在“接受”状态。不进入运行状态。
1名硕士； 3个奴隶
PC配置：64GB RAM，64位，512GB HDD（全部4个）
1. 在Redhat 7.6上使用Java（内部版本1.8.0_191-b12）完成Hadoop 2.9.1集群设置
2. 运行start-all.sh命令后：hadoop jar hadoop / share / hadoop / mapreduce / hadoop-mapreduce-examples-2.9.1.jar pi 2 1000

1. Bash Profile:


export JAVA_HOME=/home/hduser/jdk1.8.0_191
export HADOOP_HOME=/home/hduser/hadoop
export YARN_HOME=${HADOOP_HOME}
export HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_HOME}/lib/native
export HADOOP_OPTS=-Djava.net.preferIPv4Stack=true
export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop
export SPARK_HOME=/home/hduser/spark
export LD_LIBRARY_PATH=/home/hduser/hadoop/lib/native:$LD_LIBRARY_PATH
PATH=${JAVA_HOME}/bin:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:${SPARK_HOME}/bin:$PATH
#PATH=${JAVA_HOME}/bin:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:$PATH
export PATH     


2. hdfs-site.xml:

<configuration>
<property>
           <name>dfs.replication</name>
           <value>2</value>
</property>
<property>
           <name>dfs.datanode.data.dir</name>
           <value>file:///opt/dfs/datanode</value>
</property>
<property>
           <name>dfs.namenode.name.dir</name>
           <value>file:///opt/dfs/namenode</value>
</property>


3. core-site.xml:

<configuration>
<property>
        <name>hadoop.tmp.dir</name>
        <value>/home/hduser/app/hadoop/tmp</value>
</property>
<property>
           <name>fs.defaultFS</name>
           <value>hdfs://master:54310</value>
</property>
<!--<property>
           <name>dfs.permissions</name>
           <value>false</value>
</property>-->


4. mapred-site.xml:

<property>
        <name>mapred.job.tracker</name>
        <value>master:54311</value>
</property>
<property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
</property>
<property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>master:19888</value>
</property>
<property>
        <name>mapreduce.map.memory.mb</name>
        <value>2048</value>
</property>
<property>
        <name>mapreduce.reduce.memory.mb</name>
        <value>4096</value>
</property>
<property>
        <name>mapreduce.map.java.opts</name>
        <value>-Xmx1638m</value>
</property>
<property>
        <name>mapreduce.reduce.java.opts</name>
        <value>-Xmx3276m</value>
</property>
<property>
        <name>yarn.app.mapreduce.am.resource.mb</name>
        <value>1536</value>
</property>
<property>
        <name>yarn.app.mapreduce.am.commands-opts</name>
        <value>-Xmx1024m</value>
</property>


5. yarn-site.xml:

<property>
        <name>yarn.nodemanager.user-home-dir</name>
        <value>/opt/dfs</value>
</property>
<property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
</property>
<property>
        <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
        <name>yarn.resourcemanager.hostname</name>
        <value>master</value>
</property>
<property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>master:8088</value>
</property>
<property>
        <name>yarn.resourcemanager.address</name>
        <value>master:8032</value>
</property>
<property>
        <name>yarn.nodemanager.resource.memory-mb</name>
        <value>2200</value>
        <description>Amount of physical memory, in MB, that can be allocated for containers.</description>
</property>

<property>
        <name>yarn.scheduler.minimum-allocation-mb</name>
        <value>1024</value>
</property>
<property>
        <name>yarn.scheduler.maximum-allocation-mb</name>
        <value>8192</value>
</property>
<property>
        <name>yarn.scheduler.minimum-allocation-vcores</name>
        <value>1</value>
</property>
<property>
        <name>yarn.scheduler.maximum-allocation-vcores</name>
        <value>24</value>
</property>
<property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
</property>
<property>
        <name>yarn.nodemanager.pmem-check-enabled</name>
        <value>true</value>
</property>

<property>
        <name>yarn.nodemanager.vmem-pmem-ratio</name>
        <value>2.1</value>
</property>

<property>
        <name>yarn.nodemanager.resource.cpu-vcores</name>
        <value>16></value>
</property>

如何将“已接受” Hadoop工作状态更改为“正在运行”？

0 个答案: