Hadoop中的RPC实现

Hadoop的RPC分为四个部分：
（1）序列化与反序列化
（2）利用java反射与动态代理实现函数调用
（3）基于Socket机制的网络传输
（4）基于Reactor模式的事件驱动IO模型的服务的处理

在IPC包中，最重要的3个类是Server,Client和RPC，它们具有层次化的结构。

1.RPC类是对Server、Client的具体化。在RPC类中规定，客户程序发出请求调用时，参数类型必须是Invocation；从服务器返回的值类型必须是ObjectWritable。

2.RPC类是对Server、Client的包装，简化用户的使用。如果一个类需充当服务器，只需通过RPC类的静态方法getServer获得Server实例，然后start。同时此类提供协议接口的实现。如果一个类充当客户端，可以通过getProxy或者waitForProxy获得一个实现了协议接口的proxy object，与服务器端交互。

Client端

Client向Server发送调用函数的信息：

public Writable call(Writable param, InetSocketAddress addr,                        UserGroupInformation ticket)                         throws InterruptedException, IOException {    Call call = new Call(param);    Connection connection = getConnection(addr, ticket, call);    connection.sendParam(call);                 // send the parameter    synchronized (call) {      while (!call.done) {        try {          call.wait();                           // wait for the result        } catch (InterruptedException ignored) {}      }      if (call.error != null) {        if (call.error instanceof RemoteException) {          call.error.fillInStackTrace();          throw call.error;        } else { // local exception          throw wrapException(addr, call.error);        }      } else {        return call.value;      }    }  }

在发送消息之前需要建立连接：

/** Connect to the server and set up the I/O streams. It then sends     * a header to the server and starts     * the connection thread that waits for responses.     */    private synchronized void setupIOstreams() {      if (socket != null || shouldCloseConnection.get()) {        return;      }            short ioFailures = 0;      short timeoutFailures = 0;      try {        if (LOG.isDebugEnabled()) {          LOG.debug("Connecting to "+remoteId.getAddress());        }        while (true) {          try {            this.socket = socketFactory.createSocket();            this.socket.setTcpNoDelay(tcpNoDelay);            // connection time out is 20s            this.socket.connect(remoteId.getAddress(), 20000);            this.socket.setSoTimeout(pingInterval);            break;          } catch (SocketTimeoutException toe) {            /* The max number of retries is 45,             * which amounts to 20s*45 = 15 minutes retries.             */            handleConnectionFailure(timeoutFailures++, 45, toe);          } catch (IOException ie) {            handleConnectionFailure(ioFailures++, maxRetries, ie);          }        }        this.in = new DataInputStream(new BufferedInputStream            (new PingInputStream(NetUtils.getInputStream(socket))));        this.out = new DataOutputStream            (new BufferedOutputStream(NetUtils.getOutputStream(socket)));        writeHeader();        // update last activity time        touch();        // start the receiver thread after the socket connection has been set up        start();      } catch (IOException e) {        markClosed(e);        close();      }    }

发送数据：

/** Initiates a call by sending the parameter to the remote server.     * Note: this is not called from the Connection thread, but by other     * threads.     */    public void sendParam(Call call) {      if (shouldCloseConnection.get()) {        return;      }      DataOutputBuffer d=null;      try {        synchronized (this.out) {          if (LOG.isDebugEnabled())            LOG.debug(getName() + " sending #" + call.id);                    //for serializing the          //data to be written          d = new DataOutputBuffer();          d.writeInt(call.id);          call.param.write(d);          byte[] data = d.getData();          int dataLength = d.getLength();          out.writeInt(dataLength);      //first put the data length          out.write(data, 0, dataLength);//write the data          out.flush();        }      } catch(IOException e) {        markClosed(e);      } finally {        //the buffer is just an in-memory buffer, but it is still polite to        // close early        IOUtils.closeStream(d);      }    }  

接受服务器响应，处理请求：

/* Receive a response.     * Because only one receiver, so no synchronization on in.     */    private void receiveResponse() {      if (shouldCloseConnection.get()) {        return;      }      touch();            try {        int id = in.readInt();                    // try to read an id        if (LOG.isDebugEnabled())          LOG.debug(getName() + " got value #" + id);        Call call = calls.remove(id);        boolean isError = in.readBoolean();     // read if error        if (isError) {          call.setException(new RemoteException( WritableUtils.readString(in),              WritableUtils.readString(in)));        } else {          Writable value = ReflectionUtils.newInstance(valueClass, conf);          value.readFields(in);                 // read value          call.setValue(value);        }      } catch (IOException e) {        markClosed(e);      }    }

Server端

开启Listener、Responder和Handler线程：

/** Starts the service.  Must be called before any calls will be handled. */  public synchronized void start() throws IOException {    responder.start();    listener.start();    handlers = new Handler[handlerCount];        for (int i = 0; i < handlerCount; i++) {      handlers[i] = new Handler(i);      handlers[i].start();    }  }

Listener线程：

@Override    public void run() {      LOG.info(getName() + ": starting");      SERVER.set(Server.this);      while (running) {        SelectionKey key = null;        try {          selector.select();          Iterator<SelectionKey> iter = selector.selectedKeys().iterator();          while (iter.hasNext()) {            key = iter.next();            iter.remove();            try {              if (key.isValid()) {                if (key.isAcceptable())                  doAccept(key);                else if (key.isReadable())                  doRead(key);              }            } catch (IOException e) {            }            key = null;          }        } catch (OutOfMemoryError e) {          // we can run out of memory if we have too many threads          // log the event and sleep for a minute and give           // some thread(s) a chance to finish          LOG.warn("Out of Memory in server select", e);          closeCurrentConnection(key, e);          cleanupConnections(true);          try { Thread.sleep(60000); } catch (Exception ie) {}        } catch (InterruptedException e) {          if (running) {                          // unexpected -- log it            LOG.info(getName() + " caught: " +                     StringUtils.stringifyException(e));          }        } catch (Exception e) {          closeCurrentConnection(key, e);        }        cleanupConnections(false);      }      LOG.info("Stopping " + this.getName());      synchronized (this) {        try {          acceptChannel.close();          selector.close();        } catch (IOException e) { }        selector= null;        acceptChannel= null;                // clean up all connections        while (!connectionList.isEmpty()) {          closeConnection(connectionList.remove(0));        }      }    }

接受Client请求：

 void doAccept(SelectionKey key) throws IOException,  OutOfMemoryError {      Connection c = null;      ServerSocketChannel server = (ServerSocketChannel) key.channel();      // accept up to 10 connections      for (int i=0; i<10; i++) {        SocketChannel channel = server.accept();        if (channel==null) return;        channel.configureBlocking(false);        channel.socket().setTcpNoDelay(tcpNoDelay);        SelectionKey readKey = channel.register(selector, SelectionKey.OP_READ);        c = new Connection(readKey, channel, System.currentTimeMillis());        readKey.attach(c);        synchronized (connectionList) {          connectionList.add(numConnections, c);          numConnections++;        }        if (LOG.isDebugEnabled())          LOG.debug("Server connection from " + c.toString() +              "; # active connections: " + numConnections +              "; # queued calls: " + callQueue.size());      }    }

处理Client请求：

void doRead(SelectionKey key) throws InterruptedException {      int count = 0;      Connection c = (Connection)key.attachment();      if (c == null) {        return;        }      c.setLastContact(System.currentTimeMillis());            try {        count = c.readAndProcess();      } catch (InterruptedException ieo) {        throw ieo;      } catch (Exception e) {        LOG.debug(getName() + ": readAndProcess threw exception " + e + ". Count of bytes read: " + count, e);        count = -1; //so that the (count < 0) block is executed      }      if (count < 0) {        if (LOG.isDebugEnabled())          LOG.debug(getName() + ": disconnecting client " +                     c.getHostAddress() + ". Number of active connections: "+                    numConnections);        closeConnection(c);        c = null;      }      else {        c.setLastContact(System.currentTimeMillis());      }    }   

doRead将处理的请求加入到callQueue队列，然后将交由handler进行处理。

Handler线程：

@Override    public void run() {      LOG.info(getName() + ": starting");      SERVER.set(Server.this);      long lastPurgeTime = 0;   // last check for old calls.      while (running) {        try {          waitPending();     // If a channel is being registered, wait.          writeSelector.select(PURGE_INTERVAL);          Iterator<SelectionKey> iter = writeSelector.selectedKeys().iterator();          while (iter.hasNext()) {            SelectionKey key = iter.next();            iter.remove();            try {              if (key.isValid() && key.isWritable()) {                  doAsyncWrite(key);              }            } catch (IOException e) {              LOG.info(getName() + ": doAsyncWrite threw exception " + e);            }          }          long now = System.currentTimeMillis();          if (now < lastPurgeTime + PURGE_INTERVAL) {            continue;          }          lastPurgeTime = now;          //          // If there were some calls that have not been sent out for a          // long time, discard them.          //          LOG.debug("Checking for old call responses.");          ArrayList<Call> calls;                    // get the list of channels from list of keys.          synchronized (writeSelector.keys()) {            calls = new ArrayList<Call>(writeSelector.keys().size());            iter = writeSelector.keys().iterator();            while (iter.hasNext()) {              SelectionKey key = iter.next();              Call call = (Call)key.attachment();              if (call != null && key.channel() == call.connection.channel) {                 calls.add(call);              }            }          }                    for(Call call : calls) {            try {              doPurge(call, now);            } catch (IOException e) {              LOG.warn("Error in purging old calls " + e);            }          }        } catch (OutOfMemoryError e) {          //          // we can run out of memory if we have too many threads          // log the event and sleep for a minute and give          // some thread(s) a chance to finish          //          LOG.warn("Out of Memory in server select", e);          try { Thread.sleep(60000); } catch (Exception ie) {}        } catch (Exception e) {          LOG.warn("Exception in Responder " +                    StringUtils.stringifyException(e));        }      }      LOG.info("Stopping " + this.getName());    }

执行call的调用：

public Writable call(Writable param, long receivedTime) throws IOException {      try {        Invocation call = (Invocation)param;        if (verbose) log("Call: " + call);                Method method =          implementation.getMethod(call.getMethodName(),                                   call.getParameterClasses());        long startTime = System.currentTimeMillis();        Object value = method.invoke(instance, call.getParameters());        int processingTime = (int) (System.currentTimeMillis() - startTime);        int qTime = (int) (startTime-receivedTime);        LOG.debug("Served: " + call.getMethodName() +            " queueTime= " + qTime +            " procesingTime= " + processingTime);        rpcMetrics.rpcQueueTime.inc(qTime);        rpcMetrics.rpcProcessingTime.inc(processingTime);MetricsTimeVaryingRate m = rpcMetrics.metricsList.get(call.getMethodName());if (m != null) {m.inc(processingTime);}else {rpcMetrics.metricsList.put(call.getMethodName(), new MetricsTimeVaryingRate(call.getMethodName()));m = rpcMetrics.metricsList.get(call.getMethodName());m.inc(processingTime);}        if (verbose) log("Return: "+value);        return new ObjectWritable(method.getReturnType(), value);      } catch (InvocationTargetException e) {        Throwable target = e.getTargetException();        if (target instanceof IOException) {          throw (IOException)target;        } else {          IOException ioe = new IOException(target.toString());          ioe.setStackTrace(target.getStackTrace());          throw ioe;        }      } catch (Throwable e) {        IOException ioe = new IOException(e.toString());        ioe.setStackTrace(e.getStackTrace());        throw ioe;      }    }  }

然后调用responder响应处理完成的call请求

//    // Enqueue a response from the application.    //    void doRespond(Call call) throws IOException {      synchronized (call.connection.responseQueue) {        call.connection.responseQueue.addLast(call);        if (call.connection.responseQueue.size() == 1) {          processResponse(call.connection.responseQueue, true);        }      }    }

Handler处理完call后，传递给Responser

// Processes one response. Returns true if there are no more pending    // data for this channel.    //    private boolean processResponse(LinkedList<Call> responseQueue,                                    boolean inHandler) throws IOException {      boolean error = true;      boolean done = false;       // there is more data for this channel.      int numElements = 0;      Call call = null;      try {        synchronized (responseQueue) {          //          // If there are no items for this channel, then we are done          //          numElements = responseQueue.size();          if (numElements == 0) {            error = false;            return true;              // no more data for this channel.          }          //          // Extract the first call          //          call = responseQueue.removeFirst();          SocketChannel channel = call.connection.channel;          if (LOG.isDebugEnabled()) {            LOG.debug(getName() + ": responding to #" + call.id + " from " +                      call.connection);          }          //          // Send as much data as we can in the non-blocking fashion          //          int numBytes = channelWrite(channel, call.response);          if (numBytes < 0) {            return true;          }          if (!call.response.hasRemaining()) {            call.connection.decRpcCount();            if (numElements == 1) {    // last call fully processes.              done = true;             // no more data for this channel.            } else {              done = false;            // more calls pending to be sent.            }            if (LOG.isDebugEnabled()) {              LOG.debug(getName() + ": responding to #" + call.id + " from " +                        call.connection + " Wrote " + numBytes + " bytes.");            }          } else {            //            // If we were unable to write the entire response out, then             // insert in Selector queue.             //            call.connection.responseQueue.addFirst(call);                        if (inHandler) {              // set the serve time when the response has to be sent later              call.timestamp = System.currentTimeMillis();                            incPending();              try {                // Wakeup the thread blocked on select, only then can the call                 // to channel.register() complete.                writeSelector.wakeup();                channel.register(writeSelector, SelectionKey.OP_WRITE, call);              } catch (ClosedChannelException e) {                //Its ok. channel might be closed else where.                done = true;              } finally {                decPending();              }            }            if (LOG.isDebugEnabled()) {              LOG.debug(getName() + ": responding to #" + call.id + " from " +                        call.connection + " Wrote partial " + numBytes +                         " bytes.");            }          }          error = false;              // everything went off well        }      } finally {        if (error && call != null) {          LOG.warn(getName()+", call " + call + ": output error");          done = true;               // error. no more data for this channel.          closeConnection(call.connection);        }      }      return done;    }

Responder线程：

@Override    public void run() {      LOG.info(getName() + ": starting");      SERVER.set(Server.this);      long lastPurgeTime = 0;   // last check for old calls.      while (running) {        try {          waitPending();     // If a channel is being registered, wait.          writeSelector.select(PURGE_INTERVAL);          Iterator<SelectionKey> iter = writeSelector.selectedKeys().iterator();          while (iter.hasNext()) {            SelectionKey key = iter.next();            iter.remove();            try {              if (key.isValid() && key.isWritable()) {                  doAsyncWrite(key);              }            } catch (IOException e) {              LOG.info(getName() + ": doAsyncWrite threw exception " + e);            }          }          long now = System.currentTimeMillis();          if (now < lastPurgeTime + PURGE_INTERVAL) {            continue;          }          lastPurgeTime = now;          //          // If there were some calls that have not been sent out for a          // long time, discard them.          //          LOG.debug("Checking for old call responses.");          ArrayList<Call> calls;                    // get the list of channels from list of keys.          synchronized (writeSelector.keys()) {            calls = new ArrayList<Call>(writeSelector.keys().size());            iter = writeSelector.keys().iterator();            while (iter.hasNext()) {              SelectionKey key = iter.next();              Call call = (Call)key.attachment();              if (call != null && key.channel() == call.connection.channel) {                 calls.add(call);              }            }          }                    for(Call call : calls) {            try {              doPurge(call, now);            } catch (IOException e) {              LOG.warn("Error in purging old calls " + e);            }          }        } catch (OutOfMemoryError e) {          //          // we can run out of memory if we have too many threads          // log the event and sleep for a minute and give          // some thread(s) a chance to finish          //          LOG.warn("Out of Memory in server select", e);          try { Thread.sleep(60000); } catch (Exception ie) {}        } catch (Exception e) {          LOG.warn("Exception in Responder " +                    StringUtils.stringifyException(e));        }      }      LOG.info("Stopping " + this.getName());    }

doAsyncWrite：

private void doAsyncWrite(SelectionKey key) throws IOException {      Call call = (Call)key.attachment();      if (call == null) {        return;      }      if (key.channel() != call.connection.channel) {        throw new IOException("doAsyncWrite: bad channel");      }      synchronized(call.connection.responseQueue) {        if (processResponse(call.connection.responseQueue, false)) {          try {            key.interestOps(0);          } catch (CancelledKeyException e) {            /* The Listener/reader might have closed the socket.             * We don't explicitly cancel the key, so not sure if this will             * ever fire.             * This warning could be removed.             */            LOG.warn("Exception while changing ops : " + e);          }        }      }    }<span style="font-family:SimHei;font-size:18px;"></span>