Yarn核心——RPC(3) YarnRPC

hadoop 1.0中RPC的序列化机制是WritableRpcEngine，Yarn RPC采用ProtocolBuffer。

(1) 类型结构

(2) 定义RPC协议

package com.jackniu.yarnrpc.pb.api;public interface Calculate {     int add(int num1, int num2);     int minus(int num1, int num2);}

(3) 定义Proto文件

CalculateMessage.protooption java_package ="com.jackniu.yarnrpc.pb.proto";option java_outer_classname = "CalculateMessage";option java_generic_services=true;option java_generate_equals_and_hash=true;message RequestProto{    required  string methodName=1;    required int32 num1 =2;    required int32  num2=3;}message ResponseProto{    required int32 result =1;}CalculateServer.protooption java_package ="com.jackniu.yarnrpc.pb.proto";option java_outer_classname = "Calculate";option java_generic_services=true;option java_generate_equals_and_hash=true;import "CalculateMessage.proto";service CalculateService{    rpc add(RequestProto) returns (ResponseProto);    rpc  minus(RequestProto) returns (ResponseProto);}生成对应的java文件 Calculate.java CalculateMessage.java

(3) 定义接口协议和protobuf文件的关联

package com.jackniu.yarnrpc.pb.api;import com.jackniu.yarnrpc.pb.proto.*;import com.jackniu.yarnrpc.pb.proto.Calculate;/** * Created by JackNiu on 2017/9/1. */public interface CalculatePB extends Calculate.CalculateService.BlockingInterface{}

(4) 定义服务器端 对信息进行接受 解析

package com.jackniu.yarnrpc.pb.ipc;import com.google.protobuf.BlockingService;import com.google.protobuf.Descriptors;import com.google.protobuf.Message;import com.jackniu.yarnrpc.pb.proto.CalculateMessage;import java.io.DataInputStream;import java.io.DataOutputStream;import java.io.IOException;import java.net.ServerSocket;import java.net.Socket;/** * Created by JackNiu on 2017/9/2. */public class Server extends Thread {    private Class<?> protocol;    private BlockingService impl;    private int port;    private ServerSocket ss;    public Server(Class<?> protocol, BlockingService impl,int port){        this.protocol = protocol;        this.impl = impl;        this.port = port;    }    @Override    public void run() {        System.out.println("accept... ");        Socket clientSocket = null;        DataOutputStream dos = null;        DataInputStream dis = null;        try {            ss = new ServerSocket(port);        } catch (IOException e) {            e.printStackTrace();        }        int testCount =10;        while(testCount -->0){            try{                clientSocket = ss.accept();                dos = new DataOutputStream(clientSocket.getOutputStream());                dis = new DataInputStream(clientSocket.getInputStream());                int dataLen = dis.readInt();                byte[] dataBuffer = new byte[dataLen];                int readCount = dis.read(dataBuffer);                byte[] result = processOneRpc(dataBuffer);                dos.writeInt(result.length);                dos.write(result);                dos.flush();            }catch(Exception e){                e.printStackTrace();            }        }        try{            dos.close();            dis.close();            ss.close();        }catch(Exception e){        };    }    public byte[] processOneRpc(byte[] data) throws Exception{        CalculateMessage.RequestProto request = CalculateMessage.RequestProto.parseFrom(data);        String methodName = request.getMethodName();        Descriptors.MethodDescriptor methodDescriptor = impl.getDescriptorForType().findMethodByName(methodName);        Message response = impl.callBlockingMethod(methodDescriptor,null,request);        return response.toByteArray();    }}

(5) 定义服务端服务Service，用以启动后台服务，并启动特定的事件服务，这里就是Server(Calculate的)

package com.jackniu.yarnrpc.pb.server.business;import com.google.protobuf.BlockingService;import com.jackniu.yarnrpc.pb.api.Calculate;import com.jackniu.yarnrpc.pb.ipc.Server;import java.lang.reflect.Constructor;import java.lang.reflect.InvocationTargetException;import java.lang.reflect.Method;/** * Created by JackNiu on 2017/9/2. */public class CalculateService implements Calculate {    private Server server = null;    private final Class protocol = Calculate.class;    private final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();    private final String protoPackage ="com.jackniu.yarnrpc.pb.proto";    private final String host = "localhost";    private final int port = 8888;    public CalculateService(){    }    public int add(int num1, int num2) {        return num1+num2;    }    public int minus(int num1, int num2) {        return num1-num2;    }    // return ： org.jackniu.yarnrpc.pb.api.CalculatePBServiceImpl    public Class<?> getPbServiceImplClass(){        String packageName = protocol.getPackage().getName();        String className = protocol.getSimpleName();        String pbServiceImplName =  packageName + "." + className +  "PBServiceImpl";        Class<?> clazz = null;        try{            clazz = Class.forName(pbServiceImplName, true, classLoader);        }catch(ClassNotFoundException e){            System.err.println(e.toString());        }        System.out.println(clazz);        return clazz;    }    public Class<?> getProtoClass(){        String className = protocol.getSimpleName();        String protoClazzName =  protoPackage + "." + className + "$" + className + "Service";        Class<?> clazz = null;        try{            clazz = Class.forName(protoClazzName, true, classLoader);        }catch(ClassNotFoundException e){            System.err.println(e.toString());        }        System.out.println(clazz);        return clazz;    }    public void createServer() {        Class<?> pbServiceImpl = getPbServiceImplClass();        Constructor<?> constructor = null;        try {            constructor = pbServiceImpl.getConstructor(protocol);            constructor.setAccessible(true);        } catch (NoSuchMethodException e) {            System.err.print(e.toString());        }        Object service = null;  // instance of CalculatePBServiceImpl        try {            service = constructor.newInstance(this);        } catch (InstantiationException e) {        } catch (IllegalArgumentException e) {        } catch (IllegalAccessException e) {        } catch (InvocationTargetException e) {        }        Class<?> pbProtocol = service.getClass().getInterfaces()[0];        System.out.println(pbProtocol);        Class<?> protoClazz = getProtoClass();        System.out.println(protoClazz);        Method method = null;        try {            method = protoClazz.getMethod("newReflectiveBlockingService", pbProtocol.getInterfaces()[0]);        } catch (Exception e) {            e.printStackTrace();        }        try {            createServer(pbProtocol,(BlockingService) method.invoke(null,service));        }catch (Exception e){            e.printStackTrace();        }    }    public void createServer(Class pbProtocol,BlockingService service){        server= new Server(pbProtocol,service,port);        server.start();    }    public void init(){        createServer();    }    public static void main(String[] args) {        CalculateService cs= new CalculateService();        cs.init();    }    }

(6) 接口协议实现，实现的是Proto定义的方法

package com.jackniu.yarnrpc.pb.api;import com.google.protobuf.RpcController;import com.google.protobuf.ServiceException;import com.jackniu.yarnrpc.pb.proto.CalculateMessage;/** * Created by JackNiu on 2017/9/2. */public class CalculatePBServiceImpl implements CalculatePB {    public Calculate real;    public CalculatePBServiceImpl(Calculate impl){        this.real = impl;    }    public CalculateMessage.ResponseProto add(RpcController controller, CalculateMessage.RequestProto request) throws ServiceException {        CalculateMessage.ResponseProto proto = CalculateMessage.ResponseProto.getDefaultInstance();        CalculateMessage.ResponseProto.Builder build = CalculateMessage.ResponseProto.newBuilder();        int add1 = request.getNum1();        int add2 = request.getNum2();        int sum = real.add(add1, add2);        CalculateMessage.ResponseProto result = null;        build.setResult(sum);        result = build.build();        return result;    }    public CalculateMessage.ResponseProto minus(RpcController controller, CalculateMessage.RequestProto request) throws ServiceException {        CalculateMessage.ResponseProto proto = CalculateMessage.ResponseProto.getDefaultInstance();        CalculateMessage.ResponseProto.Builder build = CalculateMessage.ResponseProto.newBuilder();        int add1 = request.getNum1();        int add2 = request.getNum2();        int sum = real.minus(add1, add2);        CalculateMessage.ResponseProto result = null;        build.setResult(sum);        result = build.build();        return result;    }}

(7) 客户端实现

package com.jackniu.yarnrpc.pb.client;import com.jackniu.yarnrpc.pb.api.Calculate;import com.jackniu.yarnrpc.pb.proto.CalculateMessage;import java.io.DataInputStream;import java.io.DataOutputStream;import java.net.Socket;import java.util.Random;/** * Created by JackNiu on 2017/9/2. */public class CalculateClient implements Calculate{    public int add(int num1, int num2) {        return socketoperation("add",num1,num2);    }    public int minus(int num1, int num2) {        return socketoperation("minus",num1,num2);    }    public int socketoperation(String op,int num1,int num2){        Socket s = null;        DataOutputStream out = null;        DataInputStream in = null;        int ret = 0;        try{            s= new Socket("localhost", 8888);            out = new DataOutputStream(s.getOutputStream());            in = new DataInputStream(s.getInputStream());            CalculateMessage.RequestProto.Builder builder = CalculateMessage.RequestProto.newBuilder();            builder.setMethodName(op);            builder.setNum1(num1);            builder.setNum2(num2);            CalculateMessage.RequestProto request = builder.build();            byte [] bytes = request.toByteArray();            out.writeInt(bytes.length);            out.write(bytes);            out.flush();            int dataLen = in.readInt();            byte[] data = new byte[dataLen];            int count = in.read(data);            if(count != dataLen){                System.err.println("something bad happened!");            }            CalculateMessage.ResponseProto result = CalculateMessage.ResponseProto.parseFrom(data);            System.out.println(num1 + " " + op + " " + num2 + "=" + result.getResult());            ret = result.getResult();        }catch(Exception e){            e.printStackTrace();        }finally {            try{                in.close();                out.close();                s.close();            }catch(Exception e){                e.printStackTrace();            }        }        return ret;    }    public static void main(String[] args) {        CalculateClient client = new CalculateClient();        int testCount = 5;        Random rand = new Random();        while(testCount-- > 0){            int a = rand.nextInt(100);            int b = rand.nextInt(100);            client.add(a,b);            client.minus(a, b);        }    }}

(8) 控制台输出

Server:class com.jackniu.yarnrpc.pb.api.CalculatePBServiceImplinterface com.jackniu.yarnrpc.pb.api.CalculatePBclass com.jackniu.yarnrpc.pb.proto.Calculate$CalculateServiceclass com.jackniu.yarnrpc.pb.proto.Calculate$CalculateServiceaccept... -----------------------------------------------client:90 add 96=18690 minus 96=-675 add 49=12475 minus 49=267 add 81=887 minus 81=-7441 add 93=13441 minus 93=-5276 add 88=164

整个YarnRpc的简单实现如上，还有很多丰富的地方，网络通信现在应该是Netty的Nio通信，原理还是很简单的，可以增加很多别的特性，看个人需求。 理解还是有点麻烦。
差不多能够理解Yarn 中的基于Protobuf的通信方式，源码理解也希望有帮助。

参考网址: http://standalone.iteye.com/blog/1727544