流缓冲区的特化版本(适用于通用tcp业务包)

这是c++标准库的流缓冲区接口

http://www.cplusplus.com/reference/streambuf/streambuf/

基本理念是使用游标标识缓冲区的读写点，让缓冲区尽量以惰性扩容和减容来减少性能开支.

一下是我的特化版本的的主体结构

// Ethernet len[46~1500] MTU 20 
// udp head  8 1500 - 20 -  8 = 1472
// tcp head 20 1500 - 20 - 20 = 1460
// we define packet page size to 1024.
#define MM_PACKET_PAGE_SIZE 1024

// streambuf page size.default is 1024.
#define MM_STREAMBUF_PAGE_SIZE 1024


// output sequence (put)
//    +++++-------
//    |    |      |
//    0  pptr   size
///////////////////////
// input  sequence (get)
//    +++++-------
//    |    |      |
//    0  gptr   size

灰色代表已经读取,绿色代表已经写入等待读取,白色代表等待写入.

先说说一页的大小缺省为1024的原因：

1.经过统计，我们接触的90%以上的通用tcp业务包数据基本为int,单包总大小不会超过1k.

2.Ethernet len[46~1500] MTU 20,现代路由器基本都是1500MTU.

包体在缓冲区中通常会面临以下几种状况:

红色代表需要写入的数据超出的部分

第一张图表示数据游标已经写到末尾，超出数据缓冲区时，超出部分小于可用长度，仅做一次数据迁移(有效数据拷贝到缓冲区头，变成第三图)即可

第二张图表示缓冲区超出，总长超过一页(比如两页，闲置了一页)，除了做数据迁移，还会做一次减容，将缓冲区减少为一页

第四张图表示由于特殊情况导致可用缓冲区够用(但是需要做一次数据迁移)，但是有效数据超过一页，就会简单附加一页而不是做数据迁移(数据过长迁移代价变大了)

我的实现是尽量惰性的做减容,尽可能的减少大块数据(大于一页)的数据迁移(比如最后一种情况)。只有在数据超出时才会做一次数据迁移.

扩容和减容策略:

mm_streambuf.h

#ifndef __mm_streambuf_h__#define __mm_streambuf_h__#include "core/mm_prefix.h"#include "core/mm_core.h"#include "core/mm_platform.h"#include "core/mm_types.h"#include <stdio.h>// streambuf page size.default is 1024.#define MM_STREAMBUF_PAGE_SIZE 1024// output sequence (put)//    +++++-------//    |    |      |//    0  pptr   size///////////////////////// input  sequence (get)//    +++++-------//    |    |      |//    0  gptr   sizestruct mm_streambuf{// output sequence (put)size_t pptr;// input  sequence (get)size_t gptr;// max size for this streambuf.default is MM_STREAMBUF_PAGE_SIZE = 1024size_t size;mm_uint8_t* buff;// buffer for real data.};//////////////////////////////////////////////////////////////////////////MM_EXPORT_DLL void mm_streambuf_init(struct mm_streambuf* p);MM_EXPORT_DLL void mm_streambuf_destroy(struct mm_streambuf* p);//////////////////////////////////////////////////////////////////////////// copy q to p.MM_EXPORT_DLL void mm_streambuf_copy(struct mm_streambuf* p, struct mm_streambuf* q);// add max streambuf size.not checking overflow,use it when overflow only.MM_EXPORT_DLL void mm_streambuf_addsize(struct mm_streambuf* p, size_t size);MM_EXPORT_DLL void mm_streambuf_removeget(struct mm_streambuf* p);MM_EXPORT_DLL void mm_streambuf_removeget_size(struct mm_streambuf* p, size_t rsize);MM_EXPORT_DLL void mm_streambuf_clearget(struct mm_streambuf* p);MM_EXPORT_DLL void mm_streambuf_removeput(struct mm_streambuf* p);MM_EXPORT_DLL void mm_streambuf_removeput_size(struct mm_streambuf* p, size_t wsize);MM_EXPORT_DLL void mm_streambuf_clearput(struct mm_streambuf* p);MM_EXPORT_DLL size_t mm_streambuf_getsize(struct mm_streambuf* p);MM_EXPORT_DLL size_t mm_streambuf_putsize(struct mm_streambuf* p);// size for input and output interval.MM_EXPORT_DLL size_t mm_streambuf_size(struct mm_streambuf* p);// set get pointer to new pointer.MM_EXPORT_DLL void mm_streambuf_setg_ptr(struct mm_streambuf* p, size_t new_ptr);// set put pointer to new pointer.MM_EXPORT_DLL void mm_streambuf_setp_ptr(struct mm_streambuf* p, size_t new_ptr);// set get and put pointer to zero.MM_EXPORT_DLL void mm_streambuf_reset(struct mm_streambuf* p);// get get pointer(offset).MM_EXPORT_DLL size_t mm_streambuf_gptr(struct mm_streambuf* p);// get put pointer(offset).MM_EXPORT_DLL size_t mm_streambuf_pptr(struct mm_streambuf* p);//////////////////////////////////////////////////////////////////////////// get data s + offset length is n.and gbump n.MM_EXPORT_DLL size_t mm_streambuf_sgetn(struct mm_streambuf* p, mm_uint8_t* s, size_t o, size_t n);// put data s + offset length is n.and pbump n.MM_EXPORT_DLL size_t mm_streambuf_sputn(struct mm_streambuf* p, mm_uint8_t* s, size_t o, size_t n);// gptr += nMM_EXPORT_DLL void mm_streambuf_gbump(struct mm_streambuf* p, size_t n);// pptr += nMM_EXPORT_DLL void mm_streambuf_pbump(struct mm_streambuf* p, size_t n);//////////////////////////////////////////////////////////////////////////MM_EXPORT_DLL void mm_streambuf_underflow(struct mm_streambuf* p);MM_EXPORT_DLL void mm_streambuf_overflow(struct mm_streambuf* p);//////////////////////////////////////////////////////////////////////////// if p->gptr >= MM_STREAMBUF_PAGE_SIZE we realloc a thin memory.ohter we just remove get.make sure the p->pptr + n <= p->sizeMM_EXPORT_DLL void mm_streambuf_try_decrease(struct mm_streambuf* p, size_t n);// check overflow when need write size n.MM_EXPORT_DLL void mm_streambuf_try_increase(struct mm_streambuf* p, size_t n);// aligned streambuf memory if need sputn n size buffer before.MM_EXPORT_DLL void mm_streambuf_aligned_memory(struct mm_streambuf* p, size_t n);//////////////////////////////////////////////////////////////////////////#include "core/mm_suffix.h"#endif//__mm_streambuf_h__

mm_streambuf.c

#include "core/mm_streambuf.h"#include "core/mm_alloc.h"// realloc size (nsize+1024*1024*n,n<32) will case a malloc and memcpy.MM_EXPORT_DLL void mm_streambuf_init(struct mm_streambuf* p){// setpp->pptr = 0;// setgp->gptr = 0;//p->size = MM_STREAMBUF_PAGE_SIZE;p->buff = (mm_uint8_t*)mm_malloc(p->size);}MM_EXPORT_DLL void mm_streambuf_destroy(struct mm_streambuf* p){// setpp->pptr = 0;// setgp->gptr = 0;//mm_free(p->buff);//p->size = 0;p->buff = NULL;}MM_EXPORT_DLL void mm_streambuf_copy(struct mm_streambuf* p, struct mm_streambuf* q){mm_uint8_t* buff = (mm_uint8_t*)mm_malloc(q->size);mm_memcpy(buff,q->buff,q->size);mm_free(p->buff);p->buff = buff;p->size = q->size;// setpp->pptr = q->pptr;// setgp->gptr = q->gptr;}MM_EXPORT_DLL void mm_streambuf_addsize(struct mm_streambuf* p, size_t size){mm_uint8_t* buff = (mm_uint8_t*)mm_malloc(p->size + size);mm_memcpy(buff,p->buff,p->size);mm_free(p->buff);p->buff = buff;p->size += size;}MM_EXPORT_DLL void mm_streambuf_removeget(struct mm_streambuf* p){size_t rsize = p->gptr;if (0 < rsize){size_t wr = p->pptr;mm_memmove(p->buff, p->buff + rsize, p->pptr - p->gptr);wr -= rsize;// setpp->pptr = wr;// setgp->gptr = 0;}}MM_EXPORT_DLL void mm_streambuf_removeget_size(struct mm_streambuf* p, size_t rsize){size_t rdmax = 0;mm_streambuf_clearget(p);rdmax = p->pptr - p->gptr;if (rdmax < rsize){rsize = rdmax;}if (0 < rsize){size_t wr = p->pptr;mm_memmove(p->buff, p->buff + rsize, p->pptr - p->gptr);wr -= rsize;// setpp->pptr = wr;// setgp->gptr = 0;}}MM_EXPORT_DLL void mm_streambuf_clearget(struct mm_streambuf* p){// setgp->gptr = 0;}MM_EXPORT_DLL void mm_streambuf_removeput(struct mm_streambuf* p){size_t wsize = p->pptr;if (0 < wsize){size_t rd = p->gptr;mm_memmove(p->buff, p->buff + wsize, p->pptr - p->gptr);rd -= wsize;// setpp->pptr = 0;// setgp->gptr = rd;}}MM_EXPORT_DLL void mm_streambuf_removeput_size(struct mm_streambuf* p, size_t wsize){size_t wrmax = 0;mm_streambuf_clearput(p);wrmax = p->pptr - p->gptr;if (wrmax < wsize){wsize = wrmax;}if (0 < wsize){size_t rd = p->gptr;mm_memmove(p->buff, p->buff + wsize, p->pptr - p->gptr);rd -= wsize;// setpp->pptr = 0;// setgp->gptr = rd;}}MM_EXPORT_DLL void mm_streambuf_clearput(struct mm_streambuf* p){// setpp->pptr = 0;}MM_EXPORT_DLL size_t mm_streambuf_getsize(struct mm_streambuf* p){return p->gptr;}MM_EXPORT_DLL size_t mm_streambuf_putsize(struct mm_streambuf* p){return p->pptr;}MM_EXPORT_DLL size_t mm_streambuf_size(struct mm_streambuf* p){return p->pptr - p->gptr;}// set get pointer to new pointer.MM_EXPORT_DLL void mm_streambuf_setg_ptr(struct mm_streambuf* p, size_t new_ptr){// setgp->gptr = new_ptr;}// set put pointer to new pointer.MM_EXPORT_DLL void mm_streambuf_setp_ptr(struct mm_streambuf* p, size_t new_ptr){// setpp->pptr = new_ptr;}MM_EXPORT_DLL void mm_streambuf_reset(struct mm_streambuf* p){// setpp->pptr = 0;// setgp->gptr = 0;}// get get pointer(offset).MM_EXPORT_DLL size_t mm_streambuf_gptr(struct mm_streambuf* p){return p->gptr;}// get put pointer(offset).MM_EXPORT_DLL size_t mm_streambuf_pptr(struct mm_streambuf* p){return p->pptr;}MM_EXPORT_DLL size_t mm_streambuf_sgetn(struct mm_streambuf* p, mm_uint8_t* s, size_t o, size_t n){while( p->gptr + n > p->size ){mm_streambuf_underflow(p);}mm_memcpy(s + o, p->buff + p->gptr, n);p->gptr += n;return n;}MM_EXPORT_DLL size_t mm_streambuf_sputn(struct mm_streambuf* p, mm_uint8_t* s, size_t o, size_t n){while( p->pptr + n > p->size ){mm_streambuf_overflow(p);}mm_memcpy(p->buff + p->pptr, s + o, n);p->pptr += n;return n;}MM_EXPORT_DLL void mm_streambuf_gbump(struct mm_streambuf* p, size_t n){p->gptr += n;}MM_EXPORT_DLL void mm_streambuf_pbump(struct mm_streambuf* p, size_t n){p->pptr += n;}MM_EXPORT_DLL void mm_streambuf_underflow(struct mm_streambuf* p){if(0 < p->gptr){// if gptr have g data.we try remove for free space.mm_streambuf_removeget(p);}else{// if gptr no data we just add page size.mm_streambuf_addsize(p,MM_STREAMBUF_PAGE_SIZE);}}MM_EXPORT_DLL void mm_streambuf_overflow(struct mm_streambuf* p){mm_streambuf_addsize(p,MM_STREAMBUF_PAGE_SIZE);}// if p->gptr >= MM_STREAMBUF_PAGE_SIZE we realloc a thin memory.ohter we just remove get.make sure the p->pptr + n <= p->sizeMM_EXPORT_DLL void mm_streambuf_try_decrease(struct mm_streambuf* p, size_t n){size_t cursz = p->pptr - p->gptr;size_t fresz = p->size - cursz;// free size > n and MM_STREAMBUF_PAGE_SIZE < fresz - n.if ( fresz > n && MM_STREAMBUF_PAGE_SIZE < fresz - n ){// the surplus size for decrease.size_t ssize = fresz - n;// memmove size is <  MM_STREAMBUF_PAGE_SIZE.// decreasemm_uint8_t* buff = NULL;// use memove to remove the already get buffer.size_t wr = p->pptr;p->size -= (ssize / MM_STREAMBUF_PAGE_SIZE) * MM_STREAMBUF_PAGE_SIZE;buff = (mm_uint8_t*)mm_malloc(p->size);mm_memcpy(buff, p->buff + p->gptr, cursz);mm_free(p->buff);p->buff = buff;wr -= p->gptr;// setpp->pptr = wr;// setgp->gptr = 0;}else{// use memove to remove the already get buffer.mm_streambuf_removeget(p);}}// check overflow when need write size n.MM_EXPORT_DLL void mm_streambuf_try_increase(struct mm_streambuf* p, size_t n){if ( p->pptr + n > p->size ){// the increase process is here.// assert( p->pptr + n > p->size && "(p->pptr + n > p->size) is invalid.");// size_t dsz = p->pptr + n - p->size;// assert( 0 < dsz );// size_t asz = dsz / MM_STREAMBUF_PAGE_SIZE;// size_t bsz = dsz % MM_STREAMBUF_PAGE_SIZE;// assert( 0 != bsz );// size_t nsz = ( asz + ( 0 == bsz ? 0 : 1 ) );// assert( 0 < nsz );// mm_streambuf_addsize(p, nsz * MM_STREAMBUF_PAGE_SIZE);// the quick process is here.size_t dsz = p->pptr + n - p->size;size_t asz = dsz / MM_STREAMBUF_PAGE_SIZE;size_t nsz = asz + 1;mm_streambuf_addsize(p, nsz * MM_STREAMBUF_PAGE_SIZE);}}// aligned streambuf memory if need sputn n size buffer before.MM_EXPORT_DLL void mm_streambuf_aligned_memory(struct mm_streambuf* p, size_t n){// if current pptr + n not enough for target n.we make a aligned memory.if ( p->pptr + n > p->size ){size_t cursz = p->pptr - p->gptr;// the free size is enough and memmove size is <  MM_STREAMBUF_PAGE_SIZE.if ( p->size - cursz >= n && MM_STREAMBUF_PAGE_SIZE > cursz){mm_streambuf_try_decrease(p, n);} else{mm_streambuf_try_increase(p, n);}}}

mm_streambuf.java

package org.mm.core;//output sequence (put)//+++++-------//|    |      |//0  pptr   size/////////////////////////input  sequence (get)//+++++-------//|    |      |//0  gptr   sizepublic class mm_streambuf{public static final int MM_STREAMBUF_PAGE_SIZE = 1024;// output sequence (put)public int pptr = 0;// input  sequence (get)public int gptr = 0;// max size for this streambuf.default is MM_STREAMBUF_PAGE_SIZE = 1024public int size = 0;public byte[] buff = null;// buffer for real data.public void init(){this.pptr = 0;this.gptr = 0;this.size = MM_STREAMBUF_PAGE_SIZE;this.buff = new byte[MM_STREAMBUF_PAGE_SIZE];}public void destroy(){this.pptr = 0;this.gptr = 0;this.size = 0;this.buff = null;}// copy q to p.public void copy(mm_streambuf q){byte[] buff = new byte[q.size];System.arraycopy(q.buff,0,buff,0,q.size);this.buff = buff;this.size = q.size;// setpthis.pptr = q.pptr;// setgthis.gptr = q.gptr;}// add max streambuf size.not checking overflow,use it when overflow only.public void addsize(int size){byte[] buff = new byte[this.size + size];System.arraycopy(this.buff,0,buff,0,this.size);this.buff = buff;this.size += size;}public void removeget(){int rsize = this.gptr;if (0 < rsize){int wr = this.pptr;System.arraycopy(this.buff, rsize, this.buff, 0, this.pptr - this.gptr);wr -= rsize;// setpthis.pptr = wr;// setgthis.gptr = 0;}}public void removeget_size( int rsize ){int rdmax = 0;this.clearget();rdmax = this.pptr - this.gptr;if (rdmax < rsize){rsize = rdmax;}if (0 < rsize){int wr = this.pptr;System.arraycopy(this.buff, rsize, this.buff, 0, this.pptr - this.gptr);wr -= rsize;// setpthis.pptr = wr;// setgthis.gptr = 0;}}public void clearget(){// setgthis.gptr = 0;}public void removeput(){int wsize = this.pptr;if (0 < wsize){int rd = this.gptr;System.arraycopy(this.buff, wsize, this.buff, 0, this.pptr - this.gptr);rd -= wsize;// setpthis.pptr = 0;// setgthis.gptr = rd;}}public void removeput_size( int wsize ){int wrmax = 0;this.clearput();wrmax = this.pptr - this.gptr;if (wrmax < wsize){wsize = wrmax;}if (0 < wsize){int rd = this.gptr;System.arraycopy(this.buff, wsize, this.buff, 0, this.pptr - this.gptr);rd -= wsize;// setpthis.pptr = 0;// setgthis.gptr = rd;}}public void clearput(){// setpthis.pptr = 0;}public int getsize(){return this.gptr;}public int putsize(){return this.pptr;}// size for input and output interval.public int size(){return this.pptr - this.gptr;}// set get pointer to new pointer.public void setg_ptr( int new_ptr ){// setgthis.gptr = new_ptr;}// set put pointer to new pointer.public void setp_ptr( int new_ptr ){// setpthis.pptr = new_ptr;}// set get and put pointer to zero.public void reset(){// setpthis.pptr = 0;// setgthis.gptr = 0;}// get get pointer(offset).public int gptr(){return this.gptr;}// get put pointer(offset).public int pptr(){return this.pptr;}//////////////////////////////////////////////////////////////////////////// get data s + offset length is n.and gbump n.public int sgetn( byte[] s, int o, int n ){while( this.gptr + n > this.size ){this.underflow();}System.arraycopy(this.buff, this.gptr, s, o, n);this.gptr += n;return n;}// put data s + offset length is n.and pbump n.public int sputn( byte[] s, int o, int n ){while( this.pptr + n > this.size ){this.overflow();}System.arraycopy(s, o, this.buff, this.pptr, n);this.pptr += n;return n;}// gptr += npublic void gbump( int n ){this.gptr += n;}// pptr += npublic void pbump( int n ){this.pptr += n;}//////////////////////////////////////////////////////////////////////////public void underflow(){if(0 < this.gptr){// if gptr have g data.we try remove for free space.this.removeget();}else{// if gptr no data we just add page size.this.addsize(MM_STREAMBUF_PAGE_SIZE);}}public void overflow(){this.addsize(MM_STREAMBUF_PAGE_SIZE);}// if p->gptr >= MM_STREAMBUF_PAGE_SIZE we realloc a thin memory.ohter we just remove get.make sure the p->pptr + n <= p->sizepublic void try_decrease( int n ){int cursz = this.pptr - this.gptr;int fresz = this.size - cursz;// free size > n and MM_STREAMBUF_PAGE_SIZE < fresz - n.if ( fresz > n && MM_STREAMBUF_PAGE_SIZE < fresz - n ){// the surplus size for decrease.int ssize = fresz - n;// memmove size is <  MM_STREAMBUF_PAGE_SIZE.// decreasebyte[] buff = null;// use memove to remove the already get buffer.int wr = this.pptr;this.size -= (ssize / MM_STREAMBUF_PAGE_SIZE) * MM_STREAMBUF_PAGE_SIZE;buff = new byte[this.size];System.arraycopy(this.buff, this.gptr, buff, 0, cursz);this.buff = buff;wr -= this.gptr;// setpthis.pptr = wr;// setgthis.gptr = 0;}else{// use memove to remove the already get buffer.this.removeget();}}// check overflow when need write size n.public void try_increase( int n ){if ( this.pptr + n > this.size ){// the increase process is here.// assert( p->pptr + n > p->size && "(p->pptr + n > p->size) is invalid.");// size_t dsz = p->pptr + n - p->size;// assert( 0 < dsz );// size_t asz = dsz / MM_STREAMBUF_PAGE_SIZE;// size_t bsz = dsz % MM_STREAMBUF_PAGE_SIZE;// assert( 0 != bsz );// size_t nsz = ( asz + ( 0 == bsz ? 0 : 1 ) );// assert( 0 < nsz );// mm_streambuf_addsize(p, nsz * MM_STREAMBUF_PAGE_SIZE);// the quick process is here.int dsz = this.pptr + n - this.size;int asz = dsz / MM_STREAMBUF_PAGE_SIZE;int nsz = asz + 1;this.addsize( nsz * MM_STREAMBUF_PAGE_SIZE );}}// aligned streambuf memory if need sputn n size buffer before.public void aligned_memory( int n ){// if current pptr + n not enough for target n.we make a aligned memory.if ( this.pptr + n > this.size ){int cursz = this.pptr - this.gptr;// the free size is enough and memmove size is <  MM_STREAMBUF_PAGE_SIZE.if ( this.size - cursz >= n && MM_STREAMBUF_PAGE_SIZE > cursz){this.try_decrease( n );} else{this.try_increase( n );}}}}