BeansDB源码剖析——record.c

//DataRecord与item的不同是，item只保存键值，而record保存键值和value值，但是内存里只存PADDING的大小typedef struct data_record {char *value;union {bool free_value;    // free value or not，改为need_free比较好uint32_t crc;};int32_t tstamp; //时间戳int32_t flag; //record.c开头的那几个const int标志的组合。int32_t version; uint32_t ksz; //key大小uint32_t vsz; //v大小char key[0]; } DataRecord;

/**  Beansdb - A high available distributed key-value storage system:**      http://beansdb.googlecode.com**  Copyright 2010 Douban Inc.  All rights reserved.**  Use and distribution licensed under the BSD license.  See*  the LICENSE file for full text.**  Authors:*      Davies Liu <[e-mail]>**/#include <sys/mman.h>#include <sys/stat.h>#include <fcntl.h>#include <stdio.h>#include <stdlib.h>#include <stdint.h>#include <string.h>#include "record.h"#include "crc32.c"#include "quicklz.h"//#include "fnv1a.h"const int PADDING = 256; //PADDING是为了留出低8位，来记录bucket的下标const int32_t COMPRESS_FLAG = 0x00010000;const int32_t CLIENT_COMPRESS_FLAG = 0x00000010;const float COMPRESS_RATIO_LIMIT = 0.7;//最小的压缩比例const int TRY_COMPRESS_SIZE = 1024 * 10;uint32_t gen_hash(char *buf, int len){uint32_t hash = len * 97;if (len <= 1024){hash += fnv1a(buf, len); //整个}else{hash += fnv1a(buf, 512); //前512个hash *= 97;hash += fnv1a(buf + len - 512, 512); //后512个}return hash;}typedef struct hint_record {uint32_t ksize:8;uint32_t pos:24;int32_t version;uint16_t hash;char name[2]; // allign} HintRecord;const int NAME_IN_RECORD = 2;//|               |                 |//----------------------------------//buf     已写     cur     可写        size//param中存放了多个（HintRecord+key），而HintRecord又是根据Item得到的。struct param {int size;int curr;char* buf;};//将it存入param中void collect_items(Item* it, void* param){//-NAME_IN_RECORD是为了减少HintRecord中name的那两个比特//+1是为了后面空出一个位置放'\0'int length = sizeof(HintRecord) + strlen(it->name) + 1 - NAME_IN_RECORD;struct param *p = (struct param *)param;//不够存，扩大paramif (p->size - p->curr < length) {p->size *= 2;p->buf = (char*)realloc(p->buf, p->size);}//相当于replacement newHintRecord *r = (HintRecord*)(p->buf + p->curr);r->ksize = strlen(it->name);//it->pos的低8位表示file_id，高24位表示在file中的posr->pos = it->pos >> 8;r->version = it->ver;r->hash = it->hash;memcpy(r->name, it->name, r->ksize + 1);p->curr += length;}//将buf中的内容写入到一个临时文件中，最后用这个文件代替path的文件。void write_file(char *buf, int size, const char* path){char tmp[255];sprintf(tmp, "%s.tmp", path);FILE *hf = fopen(tmp, "wb");if (NULL==hf){fprintf(stderr, "open %s failed\n", tmp);return;}//写入size个字符，每个字符的大小为1int n = fwrite(buf, 1, size, hf); fclose(hf);if (n == size) {//删除path所指文件unlink(path);//改变这个已经写入的文件的名字为pathrename(tmp, path);}else{fprintf(stderr, "write to %s failed \n", tmp);}}//将tree中的数据放入到hint文件中，这个tree（其实是bitcast中的cur_tree）会被销毁//1.从tree中收集Item存入一个buf中，然后将treee销毁//2.压缩buf//3.将buf写入到一个hintfile中void build_hint(HTree* tree, const char* hintpath){struct param p;p.size = 1024 * 1024;p.curr = 0;p.buf = malloc(p.size);//1//将tree里的item都搜集到p中//ver<0的也收集了ht_visit(tree, collect_items, &p);ht_destroy(tree);    // 2//如果后缀是.qlz说明数据要经过压缩if (strcmp(hintpath + strlen(hintpath) - 4, ".qlz") == 0) {char* wbuf = malloc(QLZ_SCRATCH_COMPRESS);char* dst = malloc(p.size + 400);//将p中的数据压缩成dst_size个字节存到dst中int dst_size = qlz_compress(p.buf, dst, p.curr, wbuf);free(p.buf);p.curr = dst_size;p.buf = dst;free(wbuf);}//3write_file(p.buf, p.curr, hintpath);free(p.buf);}//扫描hintfile，将其中的HintRecord放入到tree中。//tree -- 实际是BitCask的tree//bucket -- 是这个hintfile在BitCask中的编号//path -- hintfile文件的目录//new_path -- 把hintfile文件中的内容存入这个文件中//1.打开hintfile并使用mmap得到里面的全部内容//2.解压缩//3.依次读取每个HintRecord放入到tree中。void scanHintFile(HTree* tree, int bucket, const char* path, const char* new_path){char *addr;int fd;struct stat sb;size_t length;fd = open(path, O_RDONLY);if (fd == -1) {fprintf(stderr, "open %s failed\n", path);return;     }if (fstat(fd, &sb) == -1 || sb.st_size == 0){close(fd);return ;}//1addr = (char*) mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0);if (addr == MAP_FAILED){fprintf(stderr, "mmap failed %s\n", path);close(fd);return;}//2char *start = addr, *end = addr + sb.st_size;if (strcmp(path + strlen(path) - 4, ".qlz") == 0) {char wbuf[QLZ_SCRATCH_DECOMPRESS];int size = qlz_size_decompressed(addr);start = malloc(size);int vsize = qlz_decompress(addr, start, wbuf);if (vsize < size) {fprintf(stderr, "decompress %s failed: %d < %d, remove it\n", path, vsize, size);unlink(path);exit(1);}end = start + vsize;}//为什么不把这一步放到前面，直接将addr对应的内容拷贝到new_path中？if (new_path != NULL) {if (strcmp(new_path + strlen(new_path) - 4, ".qlz") == 0) {char* wbuf = malloc(QLZ_SCRATCH_COMPRESS);char* dst = malloc(sb.st_size + 400);int dst_size = qlz_compress(start, dst, end - start, wbuf);write_file(dst, dst_size, new_path);free(dst);free(wbuf);} else {write_file(start, end - start, new_path);}}//3char *p = start;while (p < end) {HintRecord *r = (HintRecord*) p;p += sizeof(HintRecord) - NAME_IN_RECORD + r->ksize + 1;if (p > end){fprintf(stderr, "scan %s: unexpected end, need %ld byte\n", path, p - end);break;}uint32_t pos = (r->pos << 8) | (bucket & 0xff);if (strlen(r->name) == r->ksize) {ht_add(tree, r->name, pos, r->hash, r->version);}else{fprintf(stderr, "scan %s: key length not match %d\n", path, r->ksize);}}munmap(addr, sb.st_size);if (start != addr ) free(start);close(fd);}//返回r中的value值char* record_value(DataRecord *r){char *res = r->value;if (res == r->key + r->ksz + 1) {// value was alloced in recordres = malloc(r->vsz);memcpy(res, r->value, r->vsz);}return res;}void free_record(DataRecord *r){if (r == NULL) return;if (r->value != NULL && r->free_value) free(r->value);free(r);}void compress_record(DataRecord *r){int ksz = r->ksz, vsz = r->vsz; int n = sizeof(DataRecord) - sizeof(char*) + ksz + vsz;//比一个PADDING还大，而且没有被压缩过if (n > PADDING && (r->flag & (COMPRESS_FLAG|CLIENT_COMPRESS_FLAG)) == 0) {char *wbuf = malloc(QLZ_SCRATCH_COMPRESS);char *v = malloc(vsz + 400);if (wbuf == NULL || v == NULL) return ;//先尝试压缩一部分，如果没压缩完，就重新压缩//取较小的int try_size = vsz > TRY_COMPRESS_SIZE ? TRY_COMPRESS_SIZE : vsz; int vsize = qlz_compress(r->value, v, try_size, wbuf);//没有压缩完，并且尝试压缩的压缩比例达到了0.7，重新压缩if (try_size < vsz && vsize < try_size * COMPRESS_RATIO_LIMIT){try_size = vsz;vsize = qlz_compress(r->value, v, try_size, wbuf);}free(wbuf);//如果压缩失败，返回if (vsize > try_size * COMPRESS_RATIO_LIMIT || try_size < vsz) {free(v);return;}//压缩成功，更新rif (r->free_value) {free(r->value);}r->value = v;r->free_value = true; //r的value需要freer->vsz = vsize;r->flag |= COMPRESS_FLAG;}}DataRecord* decompress_record(DataRecord *r){if (r->flag & COMPRESS_FLAG) {char scratch[QLZ_SCRATCH_DECOMPRESS];//先验证原数据有没有被破坏int csize = qlz_size_compressed(r->value);if (csize != r->vsz) {fprintf(stderr, "broken compressed data: %d != %d, flag=%x\n", csize, r->vsz, r->flag);goto DECOMP_END;}//解压//解压本应得到的大小int size = qlz_size_decompressed(r->value);char *v = malloc(size);//内存申请不成功也if (v == NULL) {fprintf(stderr, "malloc(%d)\n", size);goto DECOMP_END;}int ret = qlz_decompress(r->value, v, scratch);//解压得到的数据少，发生错误if (ret < size) {fprintf(stderr, "decompress %s failed: %d < %d\n", r->key, ret, size);goto DECOMP_END;}//更新rif (r->free_value) {free(r->value);}r->value = v;r->free_value = true;r->vsz = size;r->flag &= ~COMPRESS_FLAG;}return r;//r是错误的，释放DECOMP_END:free_record(r); return NULL;}DataRecord* decode_record(char* buf, int size){DataRecord *r = (DataRecord *) (buf - sizeof(char*));int ksz = r->ksz, vsz = r->vsz;if (ksz < 0 || ksz > 200 || vsz < 0 || vsz > 100 * 1024 * 1024){fprintf(stderr, "invalid ksz=: %d, vsz=%d\n", ksz, vsz);return NULL;}int need = sizeof(DataRecord) - sizeof(char*) + ksz + vsz;if (size < need) {fprintf(stderr, "not enough data in buffer: %d < %d\n", size, need);return NULL;}// CRC check ?DataRecord *r2 = (DataRecord *) malloc(need + 1 + sizeof(char*));memcpy(r2, r, sizeof(DataRecord) + ksz);r2->key[ksz] = 0; // c str    r2->free_value = false;r2->value = r2->key + ksz + 1;memcpy(r2->value, r->key + ksz, vsz);return decompress_record(r2);}//从f中读取一个DataRecord//1.分步骤读取。//1.1.首先从文件中读一个PADDING出来，这是一个DataRecord所占的最小的文件空间。//1.2.计算读取的内容中是否包含完整的value//2.crc校验//3.解压缩DataRecord* read_record(FILE *f, bool decomp){//1//申请的空间比DataRecord的size大没有关系。DataRecord *r = (DataRecord*) malloc(PADDING + sizeof(char*));r->value = NULL;//1.1if (fread(&r->crc, 1, PADDING, f) != PADDING) {//或者到达f的末尾，或者f为空。fprintf(stderr, "read record faied\n");         goto READ_END;}int ksz = r->ksz, vsz = r->vsz;if (ksz < 0 || ksz > 200 || vsz < 0 || vsz > 100 * 1024 * 1024){fprintf(stderr, "invalid ksz=: %d, vsz=%d\n", ksz, vsz);goto READ_END;}uint32_t crc_old = r->crc;//1.2//计算PADDING的数据中除了DataRecord和它的key以外，还有多少数据。//sizeof(char*)是DataRecord最后的key[0]int read_size = PADDING - (sizeof(DataRecord) - sizeof(char*)) - ksz;if (vsz < read_size) {//value只存在于刚才读取的PADDING里r->value = r->key + ksz + 1; //key的最后一个字节是结束符'\0'，所以加1r->free_value = false;//后移一个字节，腾出空间给key的0memmove(r->value, r->key + ksz, vsz);//注意如果包含完整的value，那么读取的这个PADDING里也没有其它DataRecord的内容了。//因为是按照PADDING对齐的。}else{//刚才的PADDING没有读完，在f中还有残留r->value = malloc(vsz);r->free_value = true;//先把可以读的读到memcpy(r->value, r->key + ksz, read_size);int need = vsz - read_size;int ret = 0;//然后再从文件中读if (need > 0 && need != (ret=fread(r->value + read_size, 1, need, f))) {r->key[ksz] = 0; // c str    fprintf(stderr, "read record %s faied: %d < %d @%ld\n", r->key, ret, need, ftell(f)); goto READ_END;}}r->key[ksz] = 0; // c str//2uint32_t crc = crc32(0, (char*)(&r->tstamp), sizeof(DataRecord) - sizeof(char*) - sizeof(uint32_t) + ksz);crc = crc32(crc, r->value, vsz);if (crc != crc_old){fprintf(stderr, "%s @%ld crc32 check failed %d != %d\n", r->key, ftell(f), crc, r->crc);goto READ_END;}//3if (decomp) {r = decompress_record(r);}return r;READ_END:free_record(r);return NULL; }//encode与compress的不同是，encode是整个的记录，这包括crc，而compress只是K、Vchar* encode_record(DataRecord *r, int *size){compress_record(r);int m, n;int ksz = r->ksz, vsz = r->vsz;int hs = sizeof(char*); // over headerm = n = sizeof(DataRecord) - hs + ksz + vsz;//凑成PADDING的整数倍，这样，m的低八位就全为0了if (n % PADDING != 0) {m += PADDING - (n % PADDING);}char *buf = malloc(m);DataRecord *data = (DataRecord*)(buf - hs);memcpy(&data->crc, &r->crc, sizeof(DataRecord)-hs);memcpy(data->key, r->key, ksz);memcpy(data->key + ksz, r->value, vsz);data->crc = crc32(0, (char*)&data->tstamp, n - sizeof(uint32_t));*size = m;    return buf;}//向文件f中写记录r,f已经定位int write_record(FILE *f, DataRecord *r) {int size;char *data = encode_record(r, &size);if (fwrite(data, 1, size, f) < size){fprintf(stderr, "write %d byte failed\n", size);free(data);return -1;}free(data);return 0;}//遍历DataFile中的DataRecord加入到tree中。//注意这个函数的调用情境，是在bc_open时，发现对应hintfile不存在后才调用的。//bc_open是datafile决定tree(因为tree一开始是不存在的)，//而optimize是tree决定datafile(因为tree中的数据是最新的)//1.准备工作：打开datafile，新建一个htree来记录hint//2.依次读取DataRecord，加入到tree中。//3.新建hintfile文件。void scanDataFile(HTree* tree, int bucket, const char* path, const char* hintpath){if (bucket < 0 || bucket > 255) return;//1FILE *df = fopen(path, "rb");if (NULL==df){fprintf(stderr, "open %s failed\n", path);return;}fprintf(stderr, "scan datafile %s \n", path);//datafile对应的treeHTree *cur_tree = ht_new(0);fseek(df, 0, SEEK_END);uint32_t total = ftell(df);fseek(df, 0, SEEK_SET);uint32_t pos = 0;//2while (pos < total) {DataRecord *r = read_record(df, true);if (r != NULL) {uint16_t hash = gen_hash(r->value, r->vsz);//datafile决定tree//pos是Item->pos的前24位，bucket是后8位if (r->version > 0){ht_add(tree, r->key, pos | bucket, hash, r->version);            }else{ht_remove(tree, r->key);}ht_add(cur_tree, r->key, pos | bucket, hash, r->version);free_record(r);}//datafile文件是以PADDING个字节对齐的pos = ftell(df);if (pos % PADDING != 0){int left = PADDING - (pos % PADDING);fseek(df, left, SEEK_CUR);pos += left;}}fclose(df);//3build_hint(cur_tree, hintpath);}//只考察befor之前的recordvoid scanDataFileBefore(HTree* tree, int bucket, const char* path, time_t before){if (bucket < 0 || bucket > 255) return;FILE *df = fopen(path, "rb");if (NULL == df){fprintf(stderr, "open %s failed\n", path);return;}fprintf(stderr, "scan datafile %s before %ld\n", path, before);fseek(df, 0, SEEK_END);uint32_t total = ftell(df);fseek(df, 0, SEEK_SET);uint32_t pos = 0;while (pos < total) {DataRecord *r = read_record(df, true);if (r != NULL) {//这个记录是在时间戳之后才有的if (r->tstamp >= before ){break;}if (r->version > 0){uint16_t hash = gen_hash(r->value, r->vsz);ht_add(tree, r->key, pos | bucket, hash, r->version);            }else{ht_remove(tree, r->key);}free_record(r);}pos = ftell(df);if (pos % PADDING != 0){int left = PADDING - (pos % PADDING);fseek(df, left, SEEK_CUR);pos += left;}}fclose(df);}//计算删除掉的记录//从path对应的hint文件中，逐一扫描HintRecord，如果发现HintRecord跟tree中的key对应的//Item不符，或者tree中不存在，或者tree中的ver小于0，那么deleted++//total记录hint文件中总的HintRecord的数目//1.打开path(hint)处的文件,读取内容并解压，存入到一个buf中//2.从buf中依次得到HintRecord//3.比较这些record在tree中是否被删除了(ver<0或者tree中不存在)或者被移动到了其它的文件static int count_deleted_record(HTree* tree, int bucket, const char* path, int *total){char *addr;int fd;struct stat sb;size_t length;*total = 0;//1fd = open(path, O_RDONLY);if (fd == -1) {fprintf(stderr, "open %s failed\n", path);return 0; }if (fstat(fd, &sb) == -1 || sb.st_size == 0){close(fd);return 0;}addr = (char*) mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0);if (addr == MAP_FAILED){fprintf(stderr, "mmap failed %s\n", path);close(fd);return 0;}//解压char *start = addr, *end = addr + sb.st_size;if (strcmp(path + strlen(path) - 4, ".qlz") == 0) {char wbuf[QLZ_SCRATCH_DECOMPRESS];int size = qlz_size_decompressed(addr);start = malloc(size);int vsize = qlz_decompress(addr, start, wbuf);if (vsize < size) {fprintf(stderr, "decompress %s failed: %d < %d, remove it\n", path, vsize, size);unlink(path);return 0;}end = start + vsize;}char *p = start;int deleted = 0;while (p < end) {HintRecord *r = (HintRecord*) p;p += sizeof(HintRecord) - NAME_IN_RECORD + r->ksize + 1;if (p > end){fprintf(stderr, "scan %s: unexpected end, need %ld byte\n", path, p - end);break;}(*total) ++;Item *it = ht_get(tree, r->name);//关于it->pos != ((r->pos << 8) | bucket)：//如果一个record被删除了，然后相同的key又被插入，这样两个datafile中就会有//相同的key对应的data，但是bc->tree中是只有一个的，可以据此消除重复if (it == NULL || it->pos != ((r->pos << 8) | bucket) || it->ver <= 0) {deleted ++;}if (it) free(it);}munmap(addr, sb.st_size);if (start != addr) free(start);close(fd);return deleted;}//优化，通过hintpath的统计记录，来决定是否优化data文件//将有效record对应的item保存至一棵新建的树中，也就是用来进行hint的tree//1.估算是否值得优化,如果是，打开一个临时文件进行写入//2.扫面datafile中的每个DataRecord，看看它//a.在tree中不存在//b.改变了位置——或者不在这个文件中，或者在文件中的其它位置//c.ver < 0//如果以上条件都不满足，才能写进新的文件中//3.修改临时文件名，完成优化。HTree* optimizeDataFile(HTree* tree, int bucket, const char* path, const char* hintpath, int limit) {//1int all = 0;//hintpath的文件中保存的是老数据。需要跟tree里的新数据比较。int deleted = count_deleted_record(tree, bucket, hintpath, &all);//只有删除的record占到总record的十分之一，才进行优化if (deleted <= all * 0.1 && deleted <= limit) {fprintf(stderr, "only %d records deleted in %d, skip %s\n", deleted, all, path);return NULL;}FILE *df = fopen(path, "rb");if (NULL==df){fprintf(stderr, "open %s failed\n", path);return NULL;}char tmp[255];sprintf(tmp, "%s.tmp", path);FILE *new_df = fopen(tmp, "wb");if (NULL==new_df){fprintf(stderr, "open %s failed\n", tmp);fclose(df);return NULL;}//1HTree *cur_tree = ht_new(0);fseek(df, 0, SEEK_END);uint32_t total = ftell(df);fseek(df, 0, SEEK_SET);uint32_t pos = 0;deleted = 0;while (pos < total) {DataRecord *r = read_record(df, false);if (r != NULL) {Item *it = ht_get(tree, r->key);//这个item是在这个datafile中的//与scanDataFIle相对应，这里是tree决定datafileif (it && it->pos  == (pos | bucket) && it->ver > 0) {r->version = it->ver;uint32_t new_pos = ftell(new_df);uint16_t hash = it->hash;//数据在datafile中的pos改变了。ht_add(cur_tree, r->key, new_pos | bucket, hash, it->ver);if (write_record(new_df, r) != 0) {ht_destroy(cur_tree);fclose(df);fclose(new_df);return NULL;}}else{deleted ++;}if (it) free(it);free_record(r);}//对齐pos = ftell(df);if (pos % PADDING != 0){int left = PADDING - (pos % PADDING);fseek(df, left, SEEK_CUR);pos += left;}}uint32_t deleted_bytes = ftell(df) - ftell(new_df);fclose(df);fclose(new_df);//3unlink(hintpath);unlink(path);rename(tmp, path);fprintf(stderr, "optimize %s complete, %d records deleted, %d bytes came back\n", path, deleted, deleted_bytes);return cur_tree;}//对datafile中的record进行遍历。void visit_record(const char* path, RecordVisitor visitor, void *arg1, void *arg2, bool decomp){FILE *df = fopen(path, "rb");if (NULL==df){fprintf(stderr, "open %s failed\n", path);return;}fprintf(stderr, "scan datafile %s \n", path);fseek(df, 0, SEEK_END);uint32_t total = ftell(df);fseek(df, 0, SEEK_SET);uint32_t pos = 0;while (pos < total) {DataRecord *r = read_record(df, decomp);if (r != NULL) {bool cont = visitor(r, arg1, arg2);if (cont) break;}pos = ftell(df);if (pos % PADDING != 0){int left = PADDING - (pos % PADDING);fseek(df, left, SEEK_CUR);pos += left;}}fclose(df);}