大数据工程师：大数据的java基础 第十四周

 Eclipse中创建Hadoop项目的建议
– 没有Hadoop插件
– 下载并解压缩hadoop发布包
– 创建Java项目
– 将解压后hadoop源码包/src目录中core, hdfs, mapred, tool几个目录（其它几个源码根据需
要进行选择）copy到eclipse新建项目的src目录
– 右键点击eclipse项目，选择“ Properties” ，在弹出对话框中左边菜单选择“ Java Build
Path”
• 点击“ Source” 标签。先删除src这个目录，然后依次添加刚才copy过来的目录
• 点击当前对话框“ Libaries” ，点击“ Add External JARs” ，在弹出窗口中添加$HADOOP_HOME下
几个hadoop程序jar包，然后再次添加$HADOOP_HOME /lib、 $HADOOP_HOME /lib/jsp-2.1两个
目录下所有jar包，最后还要添加ANT项目lib目录下ant.jar文件

– 此时源码项目应该只有关于找不到sun.security包的错误了。这时我们还是在“ Libraries” 这
个标签中，展开jar包列表最低下的 “ JRE System Library” ，双击” Access rules” ，在弹
出窗口中点击“ add按钮”，然后在新对话框中“Resolution” 下拉框选择“Accessible” ，
“Rule Pattern” 填写*/，保存

正则表达式
 普通匹配
– 单词，例如cat，匹配cat、 catalog等等
 通配符”.”
– 匹配0至多个任意字符，如c.t，可匹配cat、 c#t、 coot、 ct等
 方括号
– 仅匹配方括号内的字符，如t[aeiou]n，可匹配tan、 ten、 tin、 ton、 tun
– 方括号内仅能匹配单个字符
 或括号(|)
– 可匹配任意一个选项，如t(a|oo)n，可匹配tan、 toon
 表示匹配次数的符号
– +: 1 – n次
– *: 0 – n次
– ？： 0或1次
– {n}：恰好n次
– {n, m}：n-m次

 常见符号
– d：数字，等价于[0-9]
– D：非数字，等价于[^0-9]
– s：空白符，等价于[\t\n\x0B\f\r]
– S：非空白符，等价于[^\t\n\x0B\f\r]
– w：单独字符，等价于[a-zA-Z0-9]
– W：非单独字符，等价于[^a-zA-Z0-9]
– ^：开头，如^java，为java开头的字符串

– $：结尾，java$，以java为结尾的行

package org.myorg;import java.io.*;import java.util.*;import org.apache.hadoop.fs.Path;import org.apache.hadoop.filecache.DistributedCache;import org.apache.hadoop.conf.*;import org.apache.hadoop.io.*;import org.apache.hadoop.mapred.*;import org.apache.hadoop.util.*; public class WordCount extends Configured implements Tool {    public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {      static enum Counters { INPUT_WORDS }      private final static IntWritable one = new IntWritable(1);     private Text word = new Text();      private boolean caseSensitive = true;     private Set<String> patternsToSkip = new HashSet<String>();     private long numRecords = 0;     private String inputFile;     public void configure(JobConf job) {       caseSensitive = job.getBoolean("wordcount.case.sensitive", true);       inputFile = job.get("map.input.file");       if (job.getBoolean("wordcount.skip.patterns", false)) {         Path[] patternsFiles = new Path[0];         try {           patternsFiles = DistributedCache.getLocalCacheFiles(job);         } catch (IOException ioe) {           System.err.println("Caught exception while getting cached files: " + StringUtils.stringifyException(ioe));         }         for (Path patternsFile : patternsFiles) {           parseSkipFile(patternsFile);         }       }     }     private void parseSkipFile(Path patternsFile) {       try {         BufferedReader fis = new BufferedReader(new FileReader(patternsFile.toString()));         String pattern = null;         while ((pattern = fis.readLine()) != null) {           patternsToSkip.add(pattern);         }       } catch (IOException ioe) {         System.err.println("Caught exception while parsing the cached file '" + patternsFile + "' : " + StringUtils.stringifyException(ioe));       }     }     public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {       String line = (caseSensitive) ? value.toString() : value.toString().toLowerCase();       for (String pattern : patternsToSkip) {         line = line.replaceAll(pattern, "");       }       StringTokenizer tokenizer = new StringTokenizer(line);       while (tokenizer.hasMoreTokens()) {         word.set(tokenizer.nextToken());         output.collect(word, one);         reporter.incrCounter(Counters.INPUT_WORDS, 1);       }       if ((++numRecords % 100) == 0) {         reporter.setStatus("Finished processing " + numRecords + " records " + "from the input file: " + inputFile);       }     }   }   public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {     public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {       int sum = 0;       while (values.hasNext()) {         sum += values.next().get();       }       output.collect(key, new IntWritable(sum));     }   }   public int run(String[] args) throws Exception {     JobConf conf = new JobConf(getConf(), WordCount.class);     conf.setJobName("wordcount");     conf.setOutputKeyClass(Text.class);     conf.setOutputValueClass(IntWritable.class);     conf.setMapperClass(Map.class);     conf.setCombinerClass(Reduce.class);     conf.setReducerClass(Reduce.class);     conf.setInputFormat(TextInputFormat.class);     conf.setOutputFormat(TextOutputFormat.class);     List<String> other_args = new ArrayList<String>();     for (int i=0; i < args.length; ++i) {       if ("-skip".equals(args[i])) {         DistributedCache.addCacheFile(new Path(args[++i]).toUri(), conf);         conf.setBoolean("wordcount.skip.patterns", true);       } else {         other_args.add(args[i]);       }     }     FileInputFormat.setInputPaths(conf, new Path(other_args.get(0)));     FileOutputFormat.setOutputPath(conf, new Path(other_args.get(1)));     JobClient.runJob(conf);     return 0;   }   public static void main(String[] args) throws Exception {     int res = ToolRunner.run(new Configuration(), new WordCount(), args);     System.exit(res);   }}

◆比如，在字符串包含验证时


//查找以Java开头,任意结尾的字符串
  Pattern pattern = Pattern.compile("^Java.*");
  Matcher matcher = pattern.matcher("Java不是人");
  boolean b= matcher.matches();
  //当条件满足时，将返回true，否则返回false
  System.out.println(b);




◆以多条件分割字符串时
Pattern pattern = Pattern.compile("[, |]+");
String[] strs = pattern.split("Java Hello World  Java,Hello,,World|Sun");
for (int i=0;i<strs.length;i++) {
    System.out.println(strs[i]);
}


◆文字替换（首次出现字符）
Pattern pattern = Pattern.compile("正则表达式");
Matcher matcher = pattern.matcher("正则表达式 Hello World,正则表达式 Hello World");
//替换第一个符合正则的数据
System.out.println(matcher.replaceFirst("Java"));


◆文字替换（全部）
Pattern pattern = Pattern.compile("正则表达式");
Matcher matcher = pattern.matcher("正则表达式 Hello World,正则表达式 Hello World");
//替换第一个符合正则的数据
System.out.println(matcher.replaceAll("Java"));




◆文字替换（置换字符）
Pattern pattern = Pattern.compile("正则表达式");
Matcher matcher = pattern.matcher("正则表达式 Hello World,正则表达式 Hello World ");
StringBuffer sbr = new StringBuffer();
while (matcher.find()) {
    matcher.appendReplacement(sbr, "Java");
}
matcher.appendTail(sbr);
System.out.println(sbr.toString());


◆验证是否为邮箱地址


String str="[e-mail]";
Pattern pattern = Pattern.compile("[\w\.\-]+@([\w\-]+\.)+[\w\-]+",Pattern.CASE_INSENSITIVE);
Matcher matcher = pattern.matcher(str);
System.out.println(matcher.matches());


◆去除html标记
Pattern pattern = Pattern.compile("<.+?>", Pattern.DOTALL);
Matcher matcher = pattern.matcher("<a href=\"index.html\">主页</a>");
String string = matcher.replaceAll("");
System.out.println(string);


◆查找html中对应条件字符串
Pattern pattern = Pattern.compile("href=\"(.+?)\"");
Matcher matcher = pattern.matcher("<a href=\"index.html\">主页</a>");
if(matcher.find())
  System.out.println(matcher.group(1));
}


◆截取http://地址
//截取url
Pattern pattern = Pattern.compile("(http://|https://){1}[//w//.//-/:]+");
Matcher matcher = pattern.matcher("dsdsds<http://dsds//gfgffdfd>fdf");
StringBuffer buffer = new StringBuffer();
while(matcher.find()){             
    buffer.append(matcher.group());       
    buffer.append("/r/n");             
System.out.println(buffer.toString());
}
       
◆替换指定{}中文字


String str = "Java目前的发展史是由{0}年-{1}年";
String[][] object={new String[]{"//{0//}","1995"},new String[]{"//{1//}","2007"}};
System.out.println(replace(str,object));


public static String replace(final String sourceString,Object[] object) {
            String temp=sourceString;   
            for(int i=0;i<object.length;i++){
                      String[] result=(String[])object[i];
               Pattern    pattern = Pattern.compile(result[0]);
               Matcher matcher = pattern.matcher(temp);
               temp=matcher.replaceAll(result[1]);
            }
            return temp;
}




◆以正则条件查询指定目录下文件


 //用于缓存文件列表
        private ArrayList files = new ArrayList();
        //用于承载文件路径
        private String _path;
        //用于承载未合并的正则公式
        private String _regexp;
       
        class MyFileFilter implements FileFilter {


              /**
               * 匹配文件名称
               */
              public boolean accept(File file) {
                try {
                  Pattern pattern = Pattern.compile(_regexp);
                  Matcher match = pattern.matcher(file.getName());               
                  return match.matches();
                } catch (Exception e) {
                  return true;
                }
              }
            }
       
        /**
         * 解析输入流
         * @param inputs
         */
        FilesAnalyze (String path,String regexp){
            getFileName(path,regexp);
        }
       
        /**
         * 分析文件名并加入files
         * @param input
         */
        private void getFileName(String path,String regexp) {
            //目录
              _path=path;
              _regexp=regexp;
              File directory = new File(_path);
              File[] filesFile = directory.listFiles(new MyFileFilter());
              if (filesFile == null) return;
              for (int j = 0; j < filesFile.length; j++) {
                files.add(filesFile[j]);
              }
              return;
            }
   
        /**
         * 显示输出信息
         * @param out
         */
        public void print (PrintStream out) {
            Iterator elements = files.iterator();
            while (elements.hasNext()) {
                File file=(File) elements.next();
                    out.println(file.getPath());   
            }
        }


        public static void output(String path,String regexp) {


            FilesAnalyze fileGroup1 = new FilesAnalyze(path,regexp);
            fileGroup1.print(System.out);
        }
   
        public static void main (String[] args) {
            output("C://","[A-z|.]*");
        }