hadoop高级应用-搜索提示

来源：互联网发布：李雪儿的网络歌曲编辑：程序博客网时间：2024/05/21 15:44

先搭建简单的web项目，基于struts,使用到了bootstrap。

界面：

web.xml

1 <filter>2         <filter-name>struts2</filter-name>3         <filter-class>org.apache.struts2.dispatcher.ng.filter.StrutsPrepareAndExecuteFilter</filter-class>4     </filter>5 6     <filter-mapping>7         <filter-name>struts2</filter-name>8         <url-pattern>/*</url-pattern>9     </filter-mapping>

struts.xml

 1 <struts> 2  3     <constant name="struts.devMode" value="true" /> 4     <constant name="struts.action.extension" value="action,," /> 5  6     <package name="ajax" extends="json-default"> 7  8         <action name="sug" class="org.admln.suggestion.sug"> 9             <result type="json"></result>10         </action>11     </package>12 13 </struts>

index.jsp

 1 <%@ page language="java" contentType="text/html; charset=utf-8" 2     pageEncoding="utf-8"%> 3 <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> 4 <html> 5 <head> 6 <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> 7 <title>search demo</title> 8 <link rel="stylesheet" href="http://code.jquery.com/ui/1.9.2/themes/base/jquery-ui.css"/> 9 <script type="text/javascript" src="http://code.jquery.com/jquery-1.8.3.js"></script>10 <script type="text/javascript" src="http://code.jquery.com/ui/1.9.2/jquery-ui.js"></script>11 <link rel="stylesheet" href="./bootstrap/css/bootstrap.min.css"/>12 13 <script type="text/javascript">14     $(document).ready(function() {15         $("#query").autocomplete({16             source:function(request,response){17                 $.ajax({18                     url:"ajax/sug.action",19                     dataType:"json",20                     data:{21                         query:$("#query").val()22                     },23                     success:function(data) {24                         response($.map(data.result,function(item) {25                             return {value:item}26                         }));27                     }28                 })29             },30             minLength:1,31         });32     })33 </script>34 </head>35 <body>36     <div class="container">37         <h1>Search Demo</h1>38         <div class="well">39             <form action="index.jsp">40                 <label>Search</label><br/>41                 <input id="query" name="query"/>42                 <input type="submit" value="查询"/>43             </form>44         </div>45     </div>46 </body>47 </html>

sug.java

 1 package org.admln.suggestion; 2  3 import java.util.ArrayList; 4 import java.util.HashSet; 5 import java.util.List; 6 import java.util.Set; 7  8 import redis.clients.jedis.Jedis; 9 10 import com.opensymphony.xwork2.ActionSupport;11 12 public class sug extends ActionSupport{13     String query;14     15     Set<String> result;16     17     public String execute() throws Exception {18         return SUCCESS;19     }20 21     public String getQuery() {22         return query;23     }24 25     public void setQuery(String query) {26         this.query = query;27     }28 29     public Set<String> getResult() {30         System.out.println(query);31         32         Jedis jedis = new Jedis("192.168.126.133");33         result = jedis.zrevrange(query, 0, 5);34         35         for(String t:result) {36             System.out.println(t);37         }38         return result;39     }40 41     public void setResult(Set<String> result) {42         this.result = result;43     }44 }

在centos上下载安装tomcat，修改配置文件

设置日志生成格式和目录：

添加tomcat manager用户

然后部署war包

安装redis

。。。

编写hadoop代码

WordCount.java

 1 package org.admln.mr; 2  3 import java.io.IOException; 4 import java.util.StringTokenizer; 5  6 import org.apache.hadoop.conf.Configuration; 7 import org.apache.hadoop.conf.Configured; 8 import org.apache.hadoop.fs.Path; 9 import org.apache.hadoop.io.IntWritable;10 import org.apache.hadoop.io.LongWritable;11 import org.apache.hadoop.io.Text;12 import org.apache.hadoop.mapreduce.Job;13 import org.apache.hadoop.mapreduce.Mapper;14 import org.apache.hadoop.mapreduce.Reducer;15 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;16 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;17 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;18 import org.apache.hadoop.util.Tool;19 import org.apache.hadoop.util.ToolRunner;20 21 public class WordCount extends Configured implements Tool {22 23     public static class Map extends24             Mapper<LongWritable, Text, Text, IntWritable> {25 26         private final static IntWritable one = new IntWritable(1);27 28         public void map(LongWritable key, Text value, Context context)29                 throws IOException, InterruptedException {30             String line = value.toString();31 32             String a[] = line.split("\"");33             if (a[1].indexOf("index.jsp?query") > 0) {34                 String b[] = a[1].split("query=| ");35                 Text word = new Text(b[2]);36                 context.write(word, one);37             }38         }39     }40 41     public static class Reduce extends42             Reducer<Text, IntWritable, Text, IntWritable> {43 44         public void reduce(Text key, Iterable<IntWritable> values,45                 Context context) throws IOException, InterruptedException {46             int sum = 0;47 48             for (IntWritable val : values) {49                 sum += val.get();50             }51             context.write(key, new IntWritable(sum));52         }53     }54 55     public static void main(String[] args) throws Exception {56         int ret = ToolRunner.run(new WordCount(), args);57         System.exit(ret);58     }59 60     @Override61     public int run(String[] args) throws Exception {62         Configuration conf = getConf();63         Job job = new Job(conf, "Load Redis");64         job.setJarByClass(WordCount.class);65         job.setOutputKeyClass(Text.class);66         job.setOutputValueClass(IntWritable.class);67 68         job.setMapperClass(Map.class);69         job.setReducerClass(Reduce.class);70         job.setInputFormatClass(TextInputFormat.class);71         job.setOutputFormatClass(RedisOutputFormat.class);72 73         FileInputFormat.setInputPaths(job, new Path(args[0]));74         FileOutputFormat.setOutputPath(job, new Path(args[1]));75 76         return job.waitForCompletion(true) ? 0 : 1;77     }78 }

RedisOutputFormat.java

 1 package org.admln.mr; 2  3 import java.io.IOException; 4  5 import org.apache.hadoop.mapreduce.RecordWriter; 6 import org.apache.hadoop.mapreduce.TaskAttemptContext; 7 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 8  9 import redis.clients.jedis.Jedis;10 11 public class RedisOutputFormat<K,V> extends FileOutputFormat<K,V>{12     13     protected static class RedisRecordWriter<K,V> extends RecordWriter<K,V>{14         private Jedis jedis;15         16         RedisRecordWriter(Jedis jedis) {17             this.jedis=jedis;18         }19         public void close(TaskAttemptContext arg0) throws IOException,InterruptedException{20             jedis.disconnect();21         }22         public void write(K key,V value) throws IOException,23             InterruptedException{24             boolean nullkey = key == null;25             boolean nullvalue = value == null;26             27             if(nullkey||nullvalue){28                 return;29             }30 31             String s = key.toString();32             for(int i=0;i<s.length();i++) {33                 String k = s.substring(0,i+1);34                 int score = Integer.parseInt(value.toString());35                 jedis.zincrby(k,score,s);36             }37         }38             39     }40     public RecordWriter<K,V> getRecordWriter(TaskAttemptContext arg0) throws IOException,41         InterruptedException{42         return new RedisRecordWriter<K,V>(new Jedis("127.0.0.1"));43     }44 }