Easy hadoop 向导伪分布式部署注意事项

只针对我的配置文件，需要建一些文件夹

伪分布式目录

mkdir -p /opt/data/hadoop/mapred/mrlocal
mkdir -p /opt/data/hadoop/mapred/mrsystem
mkdir -p /opt/data/hadoop/hdfs/name
mkdir -p /opt/data/hadoop/hdfs/data
mkdir -p /opt/data/hadoop/hdfs/namesecondary
mkdir -p /data1/hadoop/hdfs/name
mkdir -p /nfs/name
mkdir -p /data1/hadoop/hdfs/data

chown hadoop:hadoop /opt  -R

chown hadoop:hadoop /data1  -R

chown hadoop:hadoop /nfs  -R

完全分布式目录

mkdir -p /optfull/data/hadoop/mapred/mrlocal
mkdir -p /optfulldata/hadoop/mapred/mrsystem
mkdir -p /optfull/data/hadoop/hdfs/name
mkdir -p /optfull/data/hadoop/hdfs/data
mkdir -p /optfull/data/hadoop/hdfs/namesecondary
mkdir -p /data1full/hadoop/hdfs/name
mkdir -p /nfsfull/name
mkdir -p /data1full/hadoop/hdfs/data

chown hadoop:hadoop /optfull  -R

chown hadoop:hadoop /data1full  -R

chown hadoop:hadoop /nfsfull  -R

core-site.xml

<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://localhost:9000</value>

</property>
<property>
<name>fs.checkpoint.dir</name>
<value>/opt/data/hadoop/hdfs/namesecondary</value>

</property>
<property>
<name>fs.checkpoint.period</name>
<value>1800</value>

</property>
<property>
<name>fs.checkpoint.size</name>
<value>33554432</value>

</property>
<property>
<name>io.compression.codecs</name>
<value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec</value>

</property>
<property>
<name>fs.trash.interval</name>
<value>1440</value>
<description>H</description>
</property>
</configuration>

hdfs-site.xml

<configuration>
<property>
<name>dfs.name.dir</name>
<value>/opt/data/hadoop/hdfs/name,/data1/hadoop/hdfs/name,/nfs/name</value>
<description> </description>
</property>
<property>
<name>dfs.data.dir</name>
<value>/opt/data/hadoop/hdfs/data,/data1/hadoop/hdfs/data</value>
<description> </description>
</property>
<property>
<name>dfs.http.address</name>
<value>localhost:50070</value>
</property>
<property>
<name>dfs.secondary.http.address</name>
<value>localhost:50090</value>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.datanode.du.reserved</name>
<value>1073741824</value>
</property>
<property>
<name>dfs.block.size</name>
<value>134217728</value>
</property>
<property>
<name>dfs.permissions</name>
<value>false</value>
</property>
</configuration>

mapred.xml

<configuration>
<property>
<name>mapred.job.tracker</name>
<value>localhost:9001</value>
</property>
<property>
<name>mapred.local.dir</name>
<value>/opt/data/hadoop/mapred/mrlocal</value>
<final>true</final>
</property>
<property>
<name>mapred.system.dir</name>
<value>/opt/data/hadoop/mapred/mrsystem</value>
<final>true</final>
</property>
<property>
<name>mapred.tasktracker.map.tasks.maximum</name>
<value>2</value>
<final>true</final>
</property>
<property>
<name>mapred.tasktracker.reduce.tasks.maximum</name>
<value>1</value>
<final>true</final>
</property>
<property>
<name>io.sort.mb</name>
<value>32</value>
<final>true</final>
</property>
<property>
<name>mapred.child.java.opts</name>
<value>-Xmx64M</value>

</property>
<property>
<name>mapred.compress.map.output</name>
<value>true</value>

</property>
</configuration>