java ByteBufferStringOperation.java 非阻塞爬虫..

package common;import java.io.UnsupportedEncodingException;import java.nio.ByteBuffer;public class ByteBufferStringOperation {public static String ByteBufferToString(ByteBuffer buffer)throws UnsupportedEncodingException {buffer.flip();byte[] array = new byte[buffer.limit()];for (int i = 0; i < buffer.limit(); i++) {array[i] = buffer.get();}String content = new String(array, "utf8");array = null;return content;}public static ByteBuffer StringToByteBuffer(String content, int size)throws UnsupportedEncodingException {if (size <= 0) {System.err.println("invalid size");return null;}ByteBuffer buffer = ByteBuffer.allocate(size);buffer.put(content.getBytes("utf8"));return buffer;}public static void main(String[] args) {try {String jj = "你丫的";ByteBuffer buffer = ByteBufferStringOperation.StringToByteBuffer(jj, 1024 * 256);String fuck;fuck = ByteBufferStringOperation.ByteBufferToString(buffer);System.out.println(fuck);} catch (UnsupportedEncodingException e) {// TODO Auto-generated catch blocke.printStackTrace();}}}

 

package client;import java.io.IOException;import java.net.InetSocketAddress;import java.nio.ByteBuffer;import java.nio.channels.SelectionKey;import java.nio.channels.Selector;import java.nio.channels.SocketChannel;import java.util.Iterator;import common.ByteBufferStringOperation;import common.ThreadParameterSettings;public class SocketConnectionThread extends Thread {Selector selector;String url;int port;public SocketConnectionThread(String url, int port) {this.url = url;this.port = port;try {selector = Selector.open();} catch (IOException e) {e.printStackTrace();}}private void ProcessKey(SelectionKey key) throws IOException {SocketChannel client = ((SocketChannel) key.channel());if (key.isReadable()) {ByteBuffer in = ByteBuffer.allocate(1024*256);String content = new String();int flag;while(( flag= client.read(in))>0){content += ByteBufferStringOperation.ByteBufferToString(in);in.clear();// 读入}System.out.println("Reade "+flag);if (flag == -1) {// cancel 指的是,将它从selector那解除注册,放入canceled key set 中// 在下一次轮询时被remove// 那为什么之前的在迭代器中 直接remove后,这里又作了一次remove呢?//我懂了:第一次remove是将其移出 触发了操作的set里面//这一次移出,将是真正的移出selectorin = null;key.cancel();client.close();}System.out.println("Client Recieved Msg: "+ ByteBufferStringOperation.ByteBufferToString(in));return;}if (key.isWritable()) {//为了防止无限写if(key.attachment()!=null&&(Integer)key.attachment()==1)return ;String HttpHeader = new String();HttpHeader = "GET / HTTP/1.1/r/n";HttpHeader +="Accept:*/*/r/n";//HttpHeader +="User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)/n";HttpHeader +="Host:www.baidu.com/r/n";HttpHeader +="Connection:Keep-Alive/r/n/r/n";/*ByteBuffer out = ByteBufferStringOperation.StringToByteBuffer("你好,Server");*/ByteBuffer out =ByteBufferStringOperation.StringToByteBuffer(HttpHeader);int writeAmount = client.write(out);// while (out.hasRemaining())//为了防止无限写,附上一个keykey.attach(new Integer(1));//感兴趣的设置为读key.interestOps(SelectionKey.OP_READ);System.err.println("finish writing : " + writeAmount);}// 否则 不做任何处理,继续等待return;}private void Connect() throws Exception {for (int i = 0; i < ThreadParameterSettings.Thread_MAX_SOCKET; i++) {SocketChannel client = SocketChannel.open();client.configureBlocking(false);// client.client.connect(new InetSocketAddress(url, port));// 向选择器注册我们感兴趣的通道,关注:可读,可写/*client.register(selector, SelectionKey.OP_READ| SelectionKey.OP_WRITE);*///先写，再读client.register(selector, SelectionKey.OP_WRITE);// ByteBuffer out = ByteBuffer.allocate(1024 * 256);// 等待连接成功while (!client.finishConnect()) {Thread.sleep(100);}}}@Overridepublic void run() {try {Connect();// 非线程安全性,只能移除 不能增加// 包含的是至少有一个情况被触发的所有key的集合while (true) {if (selector.select() > 0) {Iterator<SelectionKey> iterator = selector.selectedKeys().iterator();SelectionKey key;while (iterator.hasNext()) {key = iterator.next();ProcessKey(key);}}}} catch (Exception e) {// TODO Auto-generated catch blocke.printStackTrace();}}}

 

 

 

package client;import java.io.IOException;import common.ThreadParameterSettings;public class SpiderClient implements Runnable{private SocketConnectionThread[] connectionPool = new SocketConnectionThread[ThreadParameterSettings.MAX_THREAD];@Overridepublic void run() {try {for (int i = 0; i <ThreadParameterSettings.MAX_THREAD; i++) {SocketConnectionThread socketThread = new SocketConnectionThread("www.baidu.com",80);connectionPool[i] = socketThread;socketThread.start();Thread.sleep(500);}while(true){// TODO Some thingThread.sleep(500);}} catch (InterruptedException e) {e.printStackTrace();}}public static void main(String[] args) throws IOException {SpiderClient s = new SpiderClient();new Thread(s).start();}}