storm实时计算实例（socket实时接入)
来源：互联网 发布：超市利润怎么算法公式 编辑：程序博客网 时间：2024/04/30 01:39
介绍
实现了一个简单的从实时日志文件监听，写入socket服务器，再接入Storm计算的一个流程。
源码
日志监听实时写入socket服务器
package socket;import java.io.BufferedReader;import java.io.File; import java.io.IOException; import java.io.InputStreamReader;import java.io.PrintWriter;import java.io.RandomAccessFile; import java.net.Socket;import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; /* * 监测数据，通过socket远程发送到另外服务器 ，见MyServerMulti * ClientRead再通过服务器从socket里读 * */ public class LogViewToSocket { private long lastTimeFileSize = 0; //上次文件大小 /** * 实时输出日志信息 * @param logFile 日志文件 * @throws IOException */ public String getNewFile(File file){File[] fs=file.listFiles();long maxtime=0;String newfilename="";for (int i=0;i<fs.length;i++){if (fs[i].lastModified()>maxtime){maxtime=fs[i].lastModified();newfilename=fs[i].getAbsolutePath();}}return newfilename;} RandomAccessFile randomFile=null; String newfile=null; String thisfile=null; public void realtimeShowLog(final File logFile,final PrintWriter out) throws IOException{ newfile=getNewFile(logFile); //指定文件可读可写 randomFile = new RandomAccessFile(new File(newfile),"r"); //启动一个线程每1秒钟读取新增的日志信息 ScheduledExecutorService exec = Executors.newScheduledThreadPool(1); exec.scheduleWithFixedDelay(new Runnable(){ public void run() { try { //获得变化部分的 randomFile.seek(lastTimeFileSize); String tmp = ""; while( (tmp = randomFile.readLine())!= null) { System.out.println(new String(tmp.getBytes("ISO8859-1"))); out.println(new String(tmp.getBytes("ISO8859-1"))); out.flush(); } thisfile=getNewFile(logFile); if(!thisfile.equals(newfile)) { randomFile = new RandomAccessFile(new File(newfile),"r"); lastTimeFileSize=0; } else lastTimeFileSize = randomFile.length(); } catch (IOException e) { throw new RuntimeException(e); } } }, 0, 1, TimeUnit.SECONDS); } public static void main(String[] args) throws Exception { LogViewToSocket view = new LogViewToSocket(); Socket socket=new Socket("192.168.27.100",5678); PrintWriter out=new PrintWriter(socket.getOutputStream()); final File tmpLogFile = new File("/home/hadoop/test"); view.realtimeShowLog(tmpLogFile,out); // socket.close(); } } socket服务器处理
import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintWriter; import java.net.ServerSocket; import java.net.Socket; import java.net.SocketAddress;import java.util.*; public class MyServerMulti { private static Socket socket1;public static void main(String[] args) throws IOException { ServerSocket server = new ServerSocket(5678); int i=0; ArrayList<PrintWriter> outs=new ArrayList<PrintWriter>(); /* * 一个client socket发送数据过来， server端再发到其他client socket端 * */ Socket socket1=null; while (true) { Socket socket = server.accept(); i++; System.out.println(i); System.out.println(socket.getInetAddress()); PrintWriter out= new PrintWriter(socket.getOutputStream()); outs.add(out); if(i==1) socket1=socket; if(i==2) invoke(socket1,outs); } } private static void invoke(final Socket client, final ArrayList<PrintWriter> outs) throws IOException { new Thread(new Runnable() { public void run() { BufferedReader in = null; PrintWriter out = null; PrintWriter out1 = null; try { in = new BufferedReader(new InputStreamReader(client.getInputStream())); out = new PrintWriter(client.getOutputStream()); while (true) { String msg = in.readLine(); System.out.println(msg); out.println("Server received " + msg); out.flush(); /*数据转发送到多个client*/ for(int i=0;i<outs.size();i++) { out1=outs.get(i); System.out.println(i); System.out.println("send msg:"+msg); out1.println(msg); out1.flush(); } System.out.println(client.getInetAddress()); if (msg.equals("bye")) { break; } } } catch(IOException ex) { ex.printStackTrace(); } finally { try { in.close(); } catch (Exception e) {} try { out.close(); } catch (Exception e) {} try { client.close(); } catch (Exception e) {} } } }).start(); } } storm topology
import java.io.BufferedReader;import java.io.BufferedWriter;import java.io.File;import java.io.FileNotFoundException;import java.io.FileOutputStream;import java.io.FileReader;import java.io.FileWriter;import java.io.IOException;import java.io.InputStreamReader;import java.io.OutputStreamWriter;import java.io.PrintWriter;import java.io.RandomAccessFile;import java.net.Socket;import java.net.UnknownHostException;import java.util.Map; //import mytest.ThroughputTest.GenSpout; import backtype.storm.Config;import backtype.storm.LocalCluster;import backtype.storm.StormSubmitter;import backtype.storm.generated.AlreadyAliveException;import backtype.storm.generated.InvalidTopologyException;import backtype.storm.spout.SpoutOutputCollector;import backtype.storm.task.OutputCollector;import backtype.storm.task.TopologyContext;import backtype.storm.topology.BasicOutputCollector;import backtype.storm.topology.OutputFieldsDeclarer;import backtype.storm.topology.TopologyBuilder;import backtype.storm.topology.base.BaseBasicBolt;import backtype.storm.topology.base.BaseRichBolt;import backtype.storm.topology.base.BaseRichSpout;import backtype.storm.tuple.Fields;import backtype.storm.tuple.Tuple;import backtype.storm.tuple.Values;import backtype.storm.utils.Utils;/* * * * storm jar stormtest.jar socket.SocketProcess /home/hadoop/out_socket.txt true * */ public class SocketProcess { public static class SocketSpout extends BaseRichSpout { /** */ static Socket sock=null; static BufferedReader in=null; String str=null; private static final long serialVersionUID = 1L; private SpoutOutputCollector _collector; private BufferedReader br; private String dataFile; private BufferedWriter bw2; RandomAccessFile randomFile; private long lastTimeFileSize = 0; int cnt=0; //定义spout文件 SocketSpout(){ } //定义如何读取spout文件 @Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { // TODO Auto-generated method stub _collector = collector; try {sock=new Socket("192.168.27.100",5678); in= new BufferedReader(new InputStreamReader(sock.getInputStream())); } catch (UnknownHostException e) {// TODO Auto-generated catch blocke.printStackTrace();} catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();} } //获取下一个tuple的方法 @Override public void nextTuple() { // TODO Auto-generated method stub if(sock==null){ try {sock=new Socket("192.168.27.100",5678); in= new BufferedReader(new InputStreamReader(sock.getInputStream())); } catch (UnknownHostException e) {// TODO Auto-generated catch blocke.printStackTrace();} catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();} } while(true){ try {str = in.readLine();} catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();} System.out.println(str); _collector.emit(new Values(str)); if(str.equals("end")){ break; } } } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { // TODO Auto-generated method stub declarer.declare(new Fields("line")); } } public static class Process extends BaseRichBolt{ private String _seperator; private String _outFile; PrintWriter pw; private OutputCollector _collector; private BufferedWriter bw; public Process(String outFile) { this._outFile = outFile; } //把输出结果保存到外部文件里面。 @Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { // TODO Auto-generated method stub this._collector = collector; File out = new File(_outFile); try {// br = new BufferedWriter(new FileWriter(out)); bw = new BufferedWriter(new OutputStreamWriter( new FileOutputStream(out, true))); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } } //blot计算单元，把tuple中的数据添加一个bkeep和回车。然后保存到outfile指定的文件中。 @Override public void execute(Tuple input) { // TODO Auto-generated method stub String line = input.getString(0);// System.out.println(line); // String[] str = line.split(_seperator); // System.out.println(str[2]); try { bw.write(line+",bkeep"+"\n"); bw.flush(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } _collector.emit(new Values(line)); } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { // TODO Auto-generated method stub declarer.declare(new Fields("line")); } } public static void main(String[] argv) throws AlreadyAliveException, InvalidTopologyException{ String outFile = argv[0]; //输出文件 boolean distribute = Boolean.valueOf(argv[1]); //本地模式还是集群模式 TopologyBuilder builder = new TopologyBuilder(); //build一个topology builder.setSpout("spout", new SocketSpout(), 1); //指定spout builder.setBolt("bolt", new Process(outFile),1).shuffleGrouping("spout"); //指定bolt，包括bolt、process和grouping Config conf = new Config(); if(distribute){ StormSubmitter.submitTopology("SocketProcess", conf, builder.createTopology()); }else{ LocalCluster cluster = new LocalCluster(); cluster.submitTopology("SocketProcess", conf, builder.createTopology()); } } }最后执行
storm jar stormtest.jar socket.SocketProcess /home/hadoop/out_socket.txt truespout接受从socket服务器实时发送过来的数据，经过topology处理，最终将数据写入out_socket.txt文件
0 0
- storm实时计算实例（socket实时接入)
- 实时计算之storm
- storm实时计算
- storm实时计算
- 实时计算之storm
- 实时计算框架之二：Storm之入门实例
- Storm分布式实时计算模式
- 实时计算框架storm基础
- Storm实例：实时单词计数
- 实时计算storm流程架构总结
- Twitter的开放实时计算系统Storm
- 实时计算storm流程架构总结
- Storm 实战：构建大数据实时计算
- 实时计算storm流程架构总结
- 实时计算storm流程架构总结
- 实时计算storm流程架构总结
- Storm+Kafka实时计算框架搭建
- Storm实时分布式计算系统简介
- Jos - Doubts (keep updating)
- android socket connection
- 蓝牙基础知识进阶——链路控制操作
- Matering PowerShell 学习笔记-2
- 面试题
- storm实时计算实例（socket实时接入)
- 深入JAVA源码分析
- Java concurrent Framework之util.concurrent(包括atomic and locks)包概述
- Winform里面的缓存使用
- Java语言与C++语言的差异总结
- webservice+axis1.4接口开发
- HBase vs. BigTable Comparison - HBase对比BigTable
- 黑马程序员—5—Java基础：多态学习笔记和学习心得体会
- PHP里10个鲜为人知但却非常有用的函数
