Ping/Echo节点监测机制的实现

  在分布式系统中，集群中的节点需要被实时监测，并在节点死掉或重新启动时做出相应的处理。

  这是一个基于ping/echo机制的监测模块的实现：

  模块分为两部分，monitor部分和monitored部分，其中monitor被放置在monitor节点上运行，monitored被放置在需要被监测的节点上。

  

  monitor部分：

  主体部分是monitor包，其中的Monitor、NodePoolManager、RegisterHandler、Reconnector均为线程。

  NodePoolManager线程启动一个接受被监测节点注册的服务，它将一直循环，等待节点注册；

  NodePoolManager在接受到注册请求后，将新开一个RegisterHandler线程处理注册；

  Monitor线程将定期遍历已注册节点的列表，向它们发出ping请求，并根据反馈结果进行相应处理，如果节点失去联系，将发出警告，并新开一个Reconnector线程尝试重连。

  

  data包中的node存储被监测节点的信息。

  strategy包中的Deadstrategy是判定节点是否已死的策略类，隔离出易发生变动的判定策略，便于修改。

  monitored部分：

  主体部分是monitored包，其中EchoThread负责建立echo服务，循环等待monitor节点的ping请求，并返回echo表示节点运行正常。

  这里采用了在被监测节点上建立ping/echo通信的服务端，原因是，在这种设计中，在被监测节点宕掉后，monitor节点可以尝试重新连接被监测节点，而被监测节点在重启成功后会在相同的端口重新开启ping/echo通信的服务，这样重连成功后monitor节点可以重新恢复对该节点的监测。

  runner包中有两个Runner，其中MonitoredRunner用在第一次启动时，它将向monitor节点注册，然后建立ping/echo服务；而MonitoredReRunner用在宕机重启时，由于之前已经注册，它将直接建立ping/echo服务。

monitor代码：

monitor.Monitor

package monitor;import java.io.DataInputStream;import java.io.DataOutputStream;import java.io.IOException;import java.net.Socket;import java.util.Vector;import monitor.data.Node;import monitor.strategy.DeadStrategy;/** * 监视器，负责遍历已注册的节点，发出ping请求，并根据反馈结果判定节点是否存活 * */public class Monitor extends Thread {private int DEAD_TIMEOUT = 3000;//等待节点echo的最长时间private int TRAVERSAL_INTERVAL = 3000;//遍历节点列表的时间间隔private NodePoolManager poolManager;private DeadStrategy strategy;public Monitor(NodePoolManager poolManager, DeadStrategy strategy) {this.poolManager = poolManager;this.strategy = strategy;}public void run() {Vector<Node> pool = poolManager.getPool();while (true) {System.out.println("Monitor开始新一次遍历node列表");/* * 遍历node列表 */for (int i = 0; i < pool.size(); i++) {Node node = pool.get(i);if (strategy.isDead(node))continue;else node.resetTimeoutCount();Socket socket = null;try {/* * 从node中取出与被监测节点通信的socket */socket = node.getSocket();if (socket.isClosed()) {node.setDead(true);handleDeadNode(node);alarm(node);continue;}/* * 设置timeout时长 */socket.setSoTimeout(DEAD_TIMEOUT);DataInputStream input = new DataInputStream(socket.getInputStream());DataOutputStream output = new DataOutputStream(socket.getOutputStream());/* * 向被监测节点发出ping请求 */output.writeUTF("ping");output.flush();System.out.println("Monitor发出ping");/* * 从被监测节点接收echo响应 */String echo = "";try {echo = input.readUTF();} catch (Exception e) {/* * 超时 */node.increaseTimeoutCount();if (strategy.isDead(node)) {handleDeadNode(node);alarm(node);}continue;}if (echo.equals("echo") == false) {System.out.println("Not echo");}System.out.println("Monitor接受到echo");} catch (Exception e) {try {if (socket != null)socket.close();} catch (IOException e1) {e1.printStackTrace();}e.printStackTrace();}}try {sleep(TRAVERSAL_INTERVAL);} catch (Exception e) {e.printStackTrace();}}}/** * 对死掉的节点进行处理 * @param node */private void handleDeadNode(Node node) {/* * 新开一个尝试重新连接的线程，连接成功后，将节点设置为存活状态 */Reconnector reconnector = new Reconnector(node);reconnector.start();}/** * 节点死掉的提醒 * @param node */private void alarm(Node node) {System.out.println("Node " + node.getIp() + " is dead!");}}

monitor.NodePoolManager

package monitor;import java.io.BufferedReader;import java.io.FileReader;import java.io.IOException;import java.net.ServerSocket;import java.net.Socket;import java.util.Vector;import java.util.regex.Matcher;import java.util.regex.Pattern;import monitor.data.Node;/** * 管理被监测的节点列表，接收被监测节点的注册请求，并更新列表 * */public class NodePoolManager extends Thread {/* * 对被监测节点注册请求的处理会交给RegisterHandler线程去做，多个线程会同时操作pool，因此它必须是线程安全的，所以这里使用了vector */private Vector<Node> pool = new Vector<Node>();/* * monitor模块接受注册的服务端口 */private static int REGISTER_PORT;public Vector<Node> getPool() {return pool;}public void run() {initialize();ServerSocket serverSocket = null;try {/* * 建立注册服务，等待被监测节点注册 */serverSocket = new ServerSocket(REGISTER_PORT);System.out.println("Monitor建立注册服务");while (true) {/* * 接受到一个注册请求 */                Socket socket = serverSocket.accept();                System.out.println("Monitor接受到一个注册请求");                /*                 * 开启一个RegisterHandler线程处理注册请求                 */                RegisterHandler handler = new RegisterHandler(pool, socket);                handler.start();            }} catch (Exception e) {try {if (serverSocket != null)serverSocket.close();} catch (IOException e1) {e1.printStackTrace();}e.printStackTrace();}}/** * 读取配置文件，初始化注册端口 */private void initialize() {try {BufferedReader reader = new BufferedReader(new FileReader("monitorConfigFiles/config.ini"));String str = reader.readLine();Pattern pattern = Pattern.compile("\\s*\\w*\\s*=\\s*(\\w*)");Matcher matcher = pattern.matcher(str);if (matcher.find()) {try {REGISTER_PORT = Integer.parseInt(matcher.group(1));} catch (Exception e) {e.printStackTrace();}}} catch (Exception e) {e.printStackTrace();}}}

monitor.Reconnector

package monitor;import monitor.data.Node;/** * 尝试重新连接死掉节点的线程 * */public class Reconnector extends Thread{/* * 重新连接的间隔时间 */private int RECONNECT_TIME = 3000;private Node node;public Reconnector(Node node){this.node = node;}public void run(){try {while(true){/* * 根据node中记录的ip和port重新连接 */node.initializeSocket();if(!node.isDead()){alarm();break;}try {sleep(RECONNECT_TIME);} catch (Exception e) {e.printStackTrace();}}} catch (Exception e) {e.printStackTrace();}}/** * 重新连接成功的提醒 */private void alarm() {System.out.println("Node "+ node.getIp() + "重新连接成功");}}

monitor.RegisterHandler

package monitor;import java.io.DataInputStream;import java.net.Socket;import java.util.Vector;import monitor.data.Node;/** * 处理具体注册过程的线程 * */public class RegisterHandler extends Thread {private Vector<Node> vector;private Socket socket;public RegisterHandler(Vector<Node> vector,Socket socket) {this.vector = vector;this.socket = socket;}public void run() {try {/* * 从socket中获取被监测节点的ip和port */DataInputStream input = new DataInputStream(socket.getInputStream());String str = input.readUTF();String ip = socket.getInetAddress().getHostAddress();int port = Integer.parseInt(str);/* * 根据ip和port初始化被监测节点，更新node列表 */Node node = new Node(ip,port);System.out.println("Monitor完成一个注册："+ip+" "+port);vector.add(node);input.close();} catch (Exception e) {e.printStackTrace();}}}

monitor.data.Node

package monitor.data;import java.net.Socket;/** * 被监测节点 * */public class Node {/* * 节点是否失去连接 */private boolean isDead;/* * 节点超时次数 */private int timeoutCount;private String ip;private int port;private Socket socket;public Node(String ip,int port) {this.timeoutCount = 0;this.ip = ip;this.port = port;initializeSocket();}public boolean isDead() {return isDead;}public void setDead(boolean isDead) {this.isDead = isDead;}public int getTimeoutCount() {return timeoutCount;}public void increaseTimeoutCount() {timeoutCount++;}public void resetTimeoutCount() {timeoutCount = 0;}public String getIp() {return ip;}public void setIp(String ip) {this.ip = ip;}public int getPort() {return port;}public void setPort(int port) {this.port = port;}public Socket getSocket() {return socket;}/** * 根据ip和port连接运行ECHO服务的节点 */public void initializeSocket() {try {socket = new Socket(ip, port);isDead = false;} catch (Exception e) {isDead = true;e.printStackTrace();}}}

monitor.strategy.DeadStrategy

package monitor.strategy;import monitor.data.Node;/** * 判断节点是否死掉的策略接口 * */public interface DeadStrategy {public boolean isDead(Node node);}

monitor.strategy.impl.DefaultDeadStrategy

package monitor.strategy.impl;import monitor.data.Node;import monitor.strategy.DeadStrategy;/** * 判断节点是否死掉的策略 * */public class DefaultDeadStrategy implements DeadStrategy {private int MAX_TIME_OUT = 3;public boolean isDead(Node node) {/* * 节点socket被关闭，或者超时次数超过MAX_TIME_OUT，均被判定为死掉 */if (node.isDead() || node.getTimeoutCount() >= MAX_TIME_OUT) {return true;} else {return false;}}}

monitor.runner.MonitorRunner

package monitor.runner;import monitor.Monitor;import monitor.NodePoolManager;import monitor.strategy.DeadStrategy;import monitor.strategy.impl.DefaultDeadStrategy;/** * 启动Monitor节点 * */public class MonitorRunner {public static void runMonitor() {NodePoolManager poolManager = new NodePoolManager();DeadStrategy strategy = new DefaultDeadStrategy();Monitor monitor = new Monitor(poolManager,strategy);/* * 建立注册服务 */poolManager.start();/* * 开始监测 */monitor.start();}public static void main(String [] args) {MonitorRunner.runMonitor();}}

monitored代码：

monitored.EchoThread

package monitored;import java.io.BufferedReader;import java.io.DataInputStream;import java.io.DataOutputStream;import java.io.FileReader;import java.io.IOException;import java.net.ServerSocket;import java.net.Socket;import java.util.regex.Matcher;import java.util.regex.Pattern;/** * 被监测节点建立echo服务，接收monitor的ping请求，并返回echo，代表该节点存活 * 在被监测节点上建立echo服务，而不是利用向monitor节点注册时建立的socket进行通信，是为了应对被监测节点死掉后过一段时间又重新启动的情况，在这种情况下， * 被监测节点将会重新建立echo服务，monitor节点在被监测节点死掉的时间内可以不断尝试重新连接 * */public class EchoThread extends Thread {/* * echo服务的端口 */private int PORT;/* * monitor的ip */private String MONITOR_IP;public void run() {initialize();ServerSocket serverSocket = null;Socket socket = null;try {System.out.println("被监测节点建立pingecho服务");/* * 建立echo服务 */serverSocket = new ServerSocket(PORT);/* * 建立与monitor节点通信的socket */socket = serverSocket.accept();while (socket.getInetAddress().getHostAddress().equals(MONITOR_IP) == false) {socket.close();socket = serverSocket.accept();}System.out.println("被监测节点与monitor建立pingecho连接");/* * 接收monitor节点的ping请求，并返回echo */DataInputStream input = new DataInputStream(socket.getInputStream());DataOutputStream out = new DataOutputStream(socket.getOutputStream());while (true) {String ping = input.readUTF();if (ping.equals("ping") == false) {System.out.println("Not ping");}System.out.println("被监测节点接受到monitor发送的ping");System.out.println("被监测节点向monitor发送echo");out.writeUTF("echo");out.flush();}} catch (Exception e) {try {if (serverSocket != null)serverSocket.close();if (socket != null)socket.close();} catch (IOException e1) {e1.printStackTrace();}e.printStackTrace();}}/** * 读取配置文件，初始化echo服务端口和monitor的ip */private void initialize() {try {BufferedReader reader = new BufferedReader(new FileReader("monitoredConfigFiles/config.ini"));String str = reader.readLine();Pattern pattern = Pattern.compile("\\s*\\w*\\s*=\\s*(\\w*)");Matcher matcher = pattern.matcher(str);if (matcher.find()) {try {PORT = Integer.parseInt(matcher.group(1));} catch (Exception e) {e.printStackTrace();}}str = reader.readLine();pattern = Pattern.compile("\\s*\\w*\\s*=\\s*(\\S*)");matcher = pattern.matcher(str);if (matcher.find()) {try {MONITOR_IP = matcher.group(1);} catch (Exception e) {e.printStackTrace();}}} catch (Exception e) {e.printStackTrace();}}}

monitored.Register

package monitored;import java.io.BufferedReader;import java.io.DataOutputStream;import java.io.FileReader;import java.io.IOException;import java.net.Socket;import java.util.regex.Matcher;import java.util.regex.Pattern;/** * 被监测节点通过该类向monitor注册 * */public class Register {/* * 被监测节点的echo服务端口，monitor节点将使用此端口与被监测节点进行ping-ehco通信 */private int PORT;/* * monitor节点的ip */private String MONITOR_IP;/* * monitor节点上运行注册服务的port */private static int MONITOR_REGISTER_PORT;public void register() {initialize();Socket socket = null;try {System.out.println("被监测节点向monitor注册");/* * 向monitor节点注册 */socket = new Socket(MONITOR_IP, MONITOR_REGISTER_PORT);DataOutputStream out = new DataOutputStream(socket.getOutputStream());String str = PORT+"";out.writeUTF(str);out.flush();out.close();} catch (Exception e) {try {if (socket != null)socket.close();} catch (IOException e1) {e1.printStackTrace();}e.printStackTrace();}}/** * 读取配置文件，初始化monitor节点上的注册服务和被监测节点上的ping-echo服务的相关ip及端口 */private void initialize() {try {BufferedReader reader = new BufferedReader(new FileReader("monitoredConfigFiles/config.ini"));String str = reader.readLine();Pattern pattern = Pattern.compile("\\s*\\w*\\s*=\\s*(\\w*)");Matcher matcher = pattern.matcher(str);if (matcher.find()) {try {PORT = Integer.parseInt(matcher.group(1));} catch (Exception e) {e.printStackTrace();}}str = reader.readLine();pattern = Pattern.compile("\\s*\\w*\\s*=\\s*(\\S*)");matcher = pattern.matcher(str);if (matcher.find()) {try {MONITOR_IP = matcher.group(1);} catch (Exception e) {e.printStackTrace();}}str = reader.readLine();pattern = Pattern.compile("\\s*\\w*\\s*=\\s*(\\w*)");matcher = pattern.matcher(str);if (matcher.find()) {try {MONITOR_REGISTER_PORT = Integer.parseInt(matcher.group(1));} catch (Exception e) {e.printStackTrace();}}} catch (Exception e) {e.printStackTrace();}}}

monitored.runner.MonitoredRunner

package monitored.runner;import monitored.EchoThread;import monitored.Register;/** * 启动被监测节点 * */public class MonitoredRunner {public static void runMonitored() {EchoThread echoThread = new EchoThread();Register register = new Register();/* * 建立ECHO服务 */echoThread.start();/* * 向Monitor节点注册 */register.register();}public static void main(String [] args) {MonitoredRunner.runMonitored();}}

monitored.runner.MonitoredReRunner

package monitored.runner;import monitored.EchoThread;/** * 重新启动被监测节点，在节点重启后调用，将重新启动ECHO服务，但不需要向Monitor节点重新注册 * */public class MonitoredReRunner {public static void reRunMonitored() {EchoThread echoThread = new EchoThread();/* * 重新启动ECHO服务 */echoThread.start();}public static void main(String [] args) {MonitoredReRunner.reRunMonitored();}}