CDH——hadoop

sudo vi /etc/apt/sources.list.d/cloudera.listdeb [arch=amd64] http://archive.cloudera.com/cdh4/ubuntu/precise/amd64/cdh precise-cdh4 contribdeb-src http://archive.cloudera.com/cdh4/ubuntu/precise/amd64/cdh precise-cdh4 contribcurl -s http://archive.cloudera.com/cdh4/ubuntu/precise/amd64/cdh/archive.key | sudo apt-key add -sudo apt-get updatesudo apt-get install hadoop-0.20-mapreduce-jobtrackersudo apt-get install hadoop-hdfs-namenodesudo apt-get install hadoop-hdfs-secondarynamenodesudo apt-get install hadoop-0.20-mapreduce-tasktracker hadoop-hdfs-datanodesudo apt-get install hadoop-clientsudo vi /etc/hostnamesudo vi /etc/hosts#127.0.1.1    <host><ip> <host-fqdn> <host>##以下所有host都需要使用fqdn全域名模式##sudo cp -r /etc/hadoop/conf.empty/ /etc/hadoop/conf.dist/ls -al /etc/hadoop/conf.dist/sudo update-alternatives --display hadoop-confsudo update-alternatives --install /etc/hadoop/conf hadoop-conf /etc/hadoop/conf.dist 20sudo update-alternatives --set hadoop-conf /etc/hadoop/conf.distsudo update-alternatives --config hadoop-confsudo vi /etc/hadoop/conf/core-site.xml<property>  <name>fs.defaultFS</name>  <value>hdfs://<namenode-host>:8020</value></property><property>  <name>fs.trash.interval</name>  <value>10060</value></property>sudo vi /etc/hadoop/conf/hdfs-site.xml<!--<property>-->  <!--<name>dfs.name.dir</name>-->  <!--<value>/var/lib/hadoop-hdfs/cache/hdfs/dfs/name</value>--><!--</property>--><property>  <name>dfs.namenode.name.dir</name>  <value>/data/1/dfs/nn</value></property>sudo mkdir -p /data/1/dfs/nnsudo chown -R hdfs:hdfs /data/1/dfs/nnsudo chmod 700 /data/1/dfs/nnsudo -u hdfs hadoop namenode -formatsudo vi /etc/hadoop/conf/hdfs-site.xml<property>  <name>dfs.datanode.data.dir</name>  <value>/data/1/dfs/dn,/data/2/dfs/dn,/data/3/dfs/dn</value></property>sudo mkdir -p /data/1/dfs/dn /data/2/dfs/dn /data/3/dfs/dnsudo chown -R hdfs:hdfs /data/1/dfs/dn /data/2/dfs/dn /data/3/dfs/dnsudo vi /etc/hadoop/conf/mapred-site.xml<property>  <name>mapred.job.tracker</name>  <value><jobtracker-host>:8021</value></property><property>  <name>mapred.local.dir</name>  <value>/data/1/mapred/local,/data/2/mapred/local,/data/3/mapred/local</value></property>sudo mkdir -p /data/1/mapred/local /data/2/mapred/local /data/3/mapred/localsudo chown -R mapred:hadoop /data/1/mapred/local /data/2/mapred/local /data/3/mapred/localsudo vi /etc/hadoop/conf/slaves<datanode-host>sudo vi /etc/hadoop/conf/masters<secondary-namenode-host>sudo vi /etc/hadoop/conf/hdfs-site.xml<property>  <name>dfs.namenode.http-address</name>  <value><namenode-host>:50070</value></property>sudo vi /etc/hadoop/conf/hadoop-env.shexport HADOOP_MAPRED_HOME=/usr/lib/hadoop-0.20-mapreducesudo apt-get install hadoop-httpfssudo update-alternatives --display hadoop-httpfs-confsudo cp -r /etc/hadoop-httpfs/conf.empty/ /etc/hadoop-httpfs/conf.dist/ls -al /etc/hadoop-httpfs/conf.dist/sudo update-alternatives --install /etc/hadoop-httpfs/conf hadoop-httpfs-conf /etc/hadoop-httpfs/conf.dist 20sudo update-alternatives --set hadoop-httpfs-conf /etc/hadoop-httpfs/conf.distsudo update-alternatives --config hadoop-httpfs-confsudo vi /etc/hadoop/conf/core-site.xml<property>  <name>hadoop.proxyuser.httpfs.hosts</name>  <value>*</value></property><property>  <name>hadoop.proxyuser.httpfs.groups</name>  <value>*</value></property>sudo vi /etc/hadoop/conf/hdfs-site.xml<property>  <name>dfs.webhdfs.enabled</name>  <value>true</value></property>sudo vi /etc/security/limits.d/hdfs.confhdfs   - nofile 32768hdfs   - nproc  65536sudo vi /etc/security/limits.d/mapred.confmapred    - nofile 32768mapred    - nproc  65536sudo usermod -a -G hadoop $USERsudo -u hdfs hadoop fs -mkdir /tmpsudo -u hdfs hadoop fs -chmod -R 1777 /tmpsudo -u hdfs hadoop fs -mkdir /tmp/hadoop-mapred/mapred/systemsudo -u hdfs hadoop fs -chown mapred:hadoop /tmp/hadoop-mapred/mapred/systemsudo -u hdfs hadoop fs -mkdir /tmp/hadoop-mapred/mapred/staging/$USERsudo -u hdfs hadoop fs -chown -R user:hadoop /tmp/hadoop-mapred/mapred/staging/$USERsudo -u hdfs hadoop fs -ls -R /sudo -u hdfs hadoop fs -mkdir /user/$USERsudo -u hdfs hadoop fs -chown $USER /user/$USERfor x in `cd /etc/init.d ; ls hadoop-hdfs-*` ; do sudo service $x restart ; donefor x in `cd /etc/init.d ; ls hadoop-0.20-mapreduce-*` ; do sudo service $x restart ; donesudo $JAVA_HOME/bin/jpshttp://<namenode-host>:50070/http://<jobtracker-host>:50030/sudo service hadoop-httpfs restarthttp://<namenode-host>:14000/webhdfs/v1?op=gethomedirectory&user.name=<username>http://<namenode-host>:50070/webhdfs/v1/?op=liststatus&user.name=<username>hadoop fs -mkdir inputhadoop fs -put /etc/hadoop/conf/*.xml inputhadoop fs -ls inputhadoop jar /usr/lib/hadoop-0.20-mapreduce/hadoop-examples.jar grep input output23 'dfs[a-z.]+'hadoop fs -lshadoop fs -ls output23hadoop fs -cat output23/part-00000 | headhadoop fs -rm -r output*