netty

Scalable IO in JavaDoug LeaState University of New York at [e-mail]://gee.cs.oswego.eduhttp://gee.cs.oswego.eduOutline" Scalable network services" Event-driven processing" Reactor patternBasic versionMultithreaded versionsOther variants" Walkthrough of java.nio nonblocking IO APIshttp://gee.cs.oswego.eduNetwork Services" Web services, Distributed Objects, etc" Most have same basic structure:Read requestDecode requestProcess serviceEncode replySend reply" But differ in nature and cost of each stepXML parsing, File transfer, Web page generation, computational services, ...http://gee.cs.oswego.eduClassic Service DesignsclientclientclientServerread decode compute encode sendread decode compute encode sendhandlerhandlerread decode compute encode sendhandlerEach handler may be started in its own threadhttp://gee.cs.oswego.eduClassic ServerSocket Loop class Server implements Runnable { public void run() { try { ServerSocket ss = new ServerSocket(PORT); while (!Thread.interrupted()) new Thread(new Handler(ss.accept())).start(); // or, single-threaded, or a thread pool } catch (IOException ex) { /* ... */ } } static class Handler implements Runnable { final Socket socket; Handler(Socket s) { socket = s; } public void run() { try { byte[] input = new byte[MAX_INPUT]; socket.getInputStream().read(input); byte[] output = process(input); socket.getOutputStream().write(output); } catch (IOException ex) { /* ... */ } } private byte[] process(byte[] cmd) { /* ... */ } }}Note: most exception handling elided from code exampleshttp://gee.cs.oswego.eduScalability Goals" Graceful degradation under increasing load (more clients)" Continuous improvement with increasing resources (CPU, memory, disk, bandwidth)" Also meet availability and performance goalsShort latenciesMeeting peak demandTunable quality of service" Divide-and-conquer is usually the best approach for achieving any scalability goalhttp://gee.cs.oswego.eduDivide and Conquer" Divide processing into small tasksEach task performs an action without blocking" Execute each task when it is enabledHere, an IO event usually serves as trigger" Basic mechanisms supported in java.nioNon-blocking reads and writesDispatch tasks associated with sensed IO events " Endless variation possible A family of event-driven designs read decode compute encode sendhandlerhttp://gee.cs.oswego.eduEvent-driven Designs" Usually more efficient than alternativesFewer resources " Don't usually need a thread per clientLess overhead" Less context switching, often less lockingBut dispatching can be slower" Must manually bind actions to events " Usually harder to programMust break up into simple non-blocking actions" Similar to GUI event-driven actions" Cannot eliminate all blocking: GC, page faults, etcMust keep track of logical state of servicehttp://gee.cs.oswego.eduBackground: Events in AWTEventEventButtonpublic void actionPerformed(...) { doSomething();}ActionListenerAWT threadAWT Event QueueEvent-driven IO uses similar ideas but in different designs...click!http://gee.cs.oswego.eduReactor Pattern" Reactor responds to IO events by dispatching the appropriate handlerSimilar to AWT thread" Handlers perform non-blocking actionsSimilar to AWT ActionListeners" Manage by binding handlers to eventsSimilar to AWT addActionListener" See Schmidt et al, Pattern-Oriented Software Architecture, Volume 2 (POSA2)Also Richard Stevens's networking books, Matt Welsh's SEDA framework, etchttp://gee.cs.oswego.eduBasic Reactor Designclientclientclientread decode compute encode sendread decode compute encode sendread decode compute encode sendReactoracceptordispatchSingle threaded versionhttp://gee.cs.oswego.edujava.nio Support" ChannelsConnections to files, sockets etc that support non-blocking reads" BuffersArray-like objects that can be directly read or written by Channels" SelectorsTell which of a set of Channels have IO events" SelectionKeysMaintain IO event status and bindings http://gee.cs.oswego.eduReactor 1: Setup class Reactor implements Runnable { final Selector selector; final ServerSocketChannel serverSocket; Reactor(int port) throws IOException { selector = Selector.open(); serverSocket = ServerSocketChannel.open(); serverSocket.socket().bind( new InetSocketAddress(port)); serverSocket.configureBlocking(false); SelectionKey sk = serverSocket.register(selector,  SelectionKey.OP_ACCEPT); sk.attach(new Acceptor());  } /*  Alternatively, use explicit SPI provider: SelectorProvider p = SelectorProvider.provider(); selector = p.openSelector(); serverSocket = p.openServerSocketChannel(); */ http://gee.cs.oswego.eduReactor 2: Dispatch Loop // class Reactor continued  public void run() { // normally in a new Thread try { while (!Thread.interrupted()) { selector.select(); Set selected = selector.selectedKeys(); Iterator it = selected.iterator(); while (it.hasNext()) dispatch((SelectionKey)(it.next()); selected.clear();  } } catch (IOException ex) { /* ... */ } } void dispatch(SelectionKey k) { Runnable r = (Runnable)(k.attachment()); if (r != null)  r.run(); } http://gee.cs.oswego.eduReactor 3: Acceptor // class Reactor continued  class Acceptor implements Runnable { // inner public void run() { try { SocketChannel c = serverSocket.accept(); if (c != null) new Handler(selector, c); } catch(IOException ex) { /* ... */ } }  }}clientclientclientread decode compute encode sendread decode compute encode sendread decode compute encode sendReactoracceptordispatchhttp://gee.cs.oswego.eduReactor 4: Handler setup final class Handler implements Runnable {  final SocketChannel socket; final SelectionKey sk; ByteBuffer input = ByteBuffer.allocate(MAXIN); ByteBuffer output = ByteBuffer.allocate(MAXOUT); static final int READING = 0, SENDING = 1; int state = READING; Handler(Selector sel, SocketChannel c)  throws IOException {  socket = c; c.configureBlocking(false); // Optionally try first read now sk = socket.register(sel, 0); sk.attach(this); sk.interestOps(SelectionKey.OP_READ); sel.wakeup(); } boolean inputIsComplete() { /* ... */ } boolean outputIsComplete() { /* ... */ } void process() { /* ... */ } http://gee.cs.oswego.eduReactor 5: Request handling // class Handler continued public void run() { try {  if (state == READING) read();  else if (state == SENDING) send(); } catch (IOException ex) { /* ... */ } } void read() throws IOException { socket.read(input); if (inputIsComplete()) { process();  state = SENDING;  // Normally also do first write now sk.interestOps(SelectionKey.OP_WRITE); } } void send() throws IOException { socket.write(output); if (outputIsComplete()) sk.cancel(); } }http://gee.cs.oswego.eduPer-State Handlers" A simple use of GoF State-Object patternRebind appropriate handler as attachment class Handler { // ... public void run() { // initial state is reader socket.read(input); if (inputIsComplete()) { process(); sk.attach(new Sender()); sk.interest(SelectionKey.OP_WRITE); sk.selector().wakeup(); } }  class Sender implements Runnable { public void run(){ // ... socket.write(output); if (outputIsComplete()) sk.cancel(); } } } http://gee.cs.oswego.eduMultithreaded Designs" Strategically add threads for scalabilityMainly applicable to multiprocessors" Worker ThreadsReactors should quickly trigger handlers" Handler processing slows down ReactorOffload non-IO processing to other threads" Multiple Reactor ThreadsReactor threads can saturate doing IODistribute load to other reactors" Load-balance to match CPU and IO rateshttp://gee.cs.oswego.eduWorker Threads" Offload non-IO processing to speed up Reactor thread Similar to POSA2 Proactor designs" Simpler than reworking compute-bound processing into event-driven formShould still be pure nonblocking computation " Enough processing to outweigh overhead" But harder to overlap processing with IO Best when can first read all input into a buffer" Use thread pool so can tune and controlNormally need many fewer threads than clientshttp://gee.cs.oswego.eduWorker Thread Poolsclientclientclientreaddecode compute encodesendreaddecode compute encodesendreaddecode compute encodesendReactorThreadPoolworkerthreadsacceptorqueued taskshttp://gee.cs.oswego.eduHandler with Thread Pool class Handler implements Runnable { // uses util.concurrent thread pool static PooledExecutor pool = new PooledExecutor(...); static final int PROCESSING = 3; // ... synchronized void read() { // ... socket.read(input); if (inputIsComplete()) { state = PROCESSING; pool.execute(new Processer()); } } synchronized void processAndHandOff() { process(); state = SENDING; // or rebind attachment sk.interest(SelectionKey.OP_WRITE); } class Processer implements Runnable { public void run() { processAndHandOff(); } }}http://gee.cs.oswego.eduCoordinating Tasks" HandoffsEach task enables, triggers, or calls next oneUsually fastest but can be brittle" Callbacks to per-handler dispatcherSets state, attachment, etcA variant of GoF Mediator pattern" QueuesFor example, passing buffers across stages" FuturesWhen each task produces a resultCoordination layered on top of join or wait/notifyhttp://gee.cs.oswego.eduUsing PooledExecutor" A tunable worker thread pool" Main method execute(Runnable r)" Controls for:The kind of task queue (any Channel)Maximum number of threadsMinimum number of threads"Warm" versus on-demand threadsKeep-alive interval until idle threads die " to be later replaced by new ones if necessarySaturation policy " block, drop, producer-runs, etchttp://gee.cs.oswego.eduMultiple Reactor Threads" Using Reactor PoolsUse to match CPU and IO ratesStatic or dynamic construction" Each with own Selector, Thread, dispatch loopMain acceptor distributes to other reactors Selector[] selectors; // also create threadsint next = 0;class Acceptor { // ...  public synchronized void run() { ... Socket connection = serverSocket.accept(); if (connection != null) new Handler(selectors[next], connection); if (++next == selectors.length) next = 0; }}http://gee.cs.oswego.eduUsing Multiple Reactorsclientclientclientreaddecode compute encodesendreaddecode compute encodesendreaddecode compute encodesendmainReactorThreadPoolworkerthreadsacceptorqueued taskssubReactorhttp://gee.cs.oswego.eduUsing other java.nio features" Multiple Selectors per ReactorTo bind different handlers to different IO eventsMay need careful synchronization to coordinate" File transferAutomated file-to-net or net-to-file copying" Memory-mapped filesAccess files via buffers" Direct buffersCan sometimes achieve zero-copy transferBut have setup and finalization overhead Best for applications with long-lived connectionshttp://gee.cs.oswego.eduConnection-Based Extensions" Instead of a single service request, Client connectsClient sends a series of messages/requestsClient disconnects" ExamplesDatabases and Transaction monitorsMulti-participant games, chat, etc" Can extend basic network service patternsHandle many relatively long-lived clientsTrack client and session state (including drops)Distribute services across multiple hostshttp://gee.cs.oswego.eduAPI Walkthrough" Buffer" ByteBuffer(CharBuffer, LongBuffer, etc not shown.)" Channel " SelectableChannel" SocketChannel " ServerSocketChannel" FileChannel" Selector" SelectionKeyhttp://gee.cs.oswego.eduBuffer abstract class Buffer { int capacity(); int position(); Buffer position(int newPosition); int limit(); Buffer limit(int newLimit); Buffer mark(); Buffer reset(); Buffer clear(); Buffer flip(); Buffer rewind(); int remaining(); boolean hasRemaining(); boolean isReadOnly();}position limit capacitymarka b chttp://gee.cs.oswego.eduByteBuffer (1) abstract class ByteBuffer extends Buffer { static ByteBuffer allocateDirect(int capacity); static ByteBuffer allocate(int capacity); static ByteBuffer wrap(byte[] src, int offset, int len); static ByteBuffer wrap(byte[] src); boolean isDirect(); ByteOrder order();  ByteBuffer order(ByteOrder bo); ByteBuffer slice(); ByteBuffer duplicate(); ByteBuffer compact(); ByteBuffer asReadOnlyBuffer();  byte get(); byte get(int index); ByteBuffer get(byte[] dst, int offset, int length); ByteBuffer get(byte[] dst); ByteBuffer put(byte b); ByteBuffer put(int index, byte b); ByteBuffer put(byte[] src, int offset, int length); ByteBuffer put(ByteBuffer src); ByteBuffer put(byte[] src); char getChar(); char getChar(int index); ByteBuffer putChar(char value); ByteBuffer putChar(int index, char value); CharBuffer asCharBuffer();http://gee.cs.oswego.eduByteBuffer (2) short getShort(); short getShort(int index); ByteBuffer putShort(short value); ByteBuffer putShort(int index, short value); ShortBuffer asShortBuffer(); int getInt(); int getInt(int index); ByteBuffer putInt(int value); ByteBuffer putInt(int index, int value); IntBuffer asIntBuffer(); long getLong(); long getLong(int index); ByteBuffer putLong(long value); ByteBuffer putLong(int index, long value); LongBuffer asLongBuffer(); float getFloat(); float getFloat(int index); ByteBuffer putFloat(float value); ByteBuffer putFloat(int index, float value); FloatBuffer asFloatBuffer(); double getDouble(); double getDouble(int index); ByteBuffer putDouble(double value); ByteBuffer putDouble(int index, double value); DoubleBuffer asDoubleBuffer();}http://gee.cs.oswego.eduChannel interface Channel { boolean isOpen(); void close() throws IOException;}interface ReadableByteChannel extends Channel { int read(ByteBuffer dst) throws IOException;}interface WritableByteChannel extends Channel { int write(ByteBuffer src) throws IOException;}interface ScatteringByteChannel extends ReadableByteChannel { int read(ByteBuffer[] dsts, int offset, int length) throws IOException; int read(ByteBuffer[] dsts) throws IOException;}interface GatheringByteChannel extends WritableByteChannel { int write(ByteBuffer[] srcs, int offset, int length) throws IOException; int write(ByteBuffer[] srcs) throws IOException;}http://gee.cs.oswego.eduSelectableChannel abstract class SelectableChannel implements Channel { int validOps(); boolean isRegistered(); SelectionKey keyFor(Selector sel); SelectionKey register(Selector sel, int ops) throws ClosedChannelException; void configureBlocking(boolean block)  throws IOException; boolean isBlocking(); Object blockingLock();}http://gee.cs.oswego.eduSocketChannel abstract class SocketChannel implements ByteChannel ... { static SocketChannel open() throws IOException; Socket socket(); int validOps(); boolean isConnected(); boolean isConnectionPending(); boolean isInputOpen(); boolean isOutputOpen(); boolean connect(SocketAddress remote) throws IOException; boolean finishConnect() throws IOException; void shutdownInput() throws IOException; void shutdownOutput() throws IOException; int read(ByteBuffer dst) throws IOException; int read(ByteBuffer[] dsts, int offset, int length)  throws IOException; int read(ByteBuffer[] dsts) throws IOException; int write(ByteBuffer src) throws IOException; int write(ByteBuffer[] srcs, int offset, int length)  throws IOException; int write(ByteBuffer[] srcs) throws IOException;}http://gee.cs.oswego.eduServerSocketChannel abstract class ServerSocketChannel extends ... { static ServerSocketChannel open() throws IOException; int validOps(); ServerSocket socket(); SocketChannel accept() throws IOException;}http://gee.cs.oswego.eduFileChannel abstract class FileChannel implements ... { int read(ByteBuffer dst); int read(ByteBuffer dst, long position); int read(ByteBuffer[] dsts, int offset, int length); int read(ByteBuffer[] dsts); int write(ByteBuffer src); int write(ByteBuffer src, long position); int write(ByteBuffer[] srcs, int offset, int length); int write(ByteBuffer[] srcs); long position(); void position(long newPosition); long size(); void truncate(long size); void force(boolean flushMetaDataToo); int transferTo(long position, int count, WritableByteChannel dst); int transferFrom(ReadableByteChannel src,  long position, int count); FileLock lock(long position, long size, boolean shared); FileLock lock(); FileLock tryLock(long pos, long size, boolean shared); FileLock tryLock(); static final int MAP_RO, MAP_RW, MAP_COW; MappedByteBuffer map(int mode, long position, int size);} NOTE: ALL methods throw IOExceptionhttp://gee.cs.oswego.eduSelector abstract class Selector { static Selector open() throws IOException; Set keys(); Set selectedKeys(); int selectNow() throws IOException; int select(long timeout) throws IOException; int select() throws IOException; void wakeup(); void close() throws IOException;}http://gee.cs.oswego.eduSelectionKey abstract class SelectionKey { static final int OP_READ, OP_WRITE,  OP_CONNECT, OP_ACCEPT; SelectableChannel channel(); Selector selector(); boolean isValid(); void cancel(); int interestOps(); void interestOps(int ops); int readyOps(); boolean isReadable(); boolean isWritable(); boolean isConnectable(); boolean isAcceptable(); Object attach(Object ob); Object attachment();}