MongoDB笔记二

这里有三个文件组成，一个是测试文件，另外两个是对MongoDB操作的封装

测试文件

#include "MongoDBWrapper.h"#include "sperformance.h"#include <iostream>using namespace std;#include <boost/algorithm/string/split.hpp>#include <boost/algorithm/string.hpp>kagula::CMongoDBWrapper& mongoDB = kagula::CMongoDBWrapper::Instance();//测试一天内插入多次。//测试不同日期插入。void testInsert(){std::string cable_id("00d34f28-8bb9-4f22-a032-b46468d2176a");const int type_id = 2;std::string channel_id("c5e5ef1d-b31a-443b-a044-d70c64ac3354");int pos = 0;float fInterval = .0f;float value = .1f;mongoDB.insertOrUpdate(cable_id, type_id, channel_id, pos, fInterval, value);//100条记录，corei3-2120,用了23s+，虽然console output拖了后腿，这个更新速度也是无法接受的。//决定新建个方法，专门进行批处理更新。kagula::PerformanceTest pt;pt.start();for (int i = 0; i < 100;i++){value += .01f;mongoDB.insertOrUpdate(cable_id, type_id, channel_id, pos, fInterval, value);}pt.stop();std::cout << "100 records update elapsed time:" << pt.toString() << std::endl << std::endl;}void testInsertBatch(){std::string cable_id("00d34f28-8bb9-4f22-a032-b46468d2176a");const int type_id = 2;std::string channel_id("c5e5ef1d-b31a-443b-a044-d70c64ac3354");int pos = 0;float fInterval = .0f;float value = .2f;std::vector<double> vecInsert;for (int i = 0; i < 10 * 1000; i++){vecInsert.push_back(i + value);}cout << "testInsertBatch 10k  test begin." << endl;kagula::PerformanceTest pt;pt.start();mongoDB.insertOrUpdate_batch(cable_id, type_id, channel_id, fInterval, vecInsert);pt.stop();//10k条记录，corei3-2120,新建记录用了1,760ms  - 3,655ms//10k条记录，corei3-2120,更新记录用了304,432mscout << "testInsertBatch 10k  elapsed time:" << pt.toString() << endl;}void testSplit(){string strBuffer;char buf[32] = { 0 };kagula::PerformanceTest pt;cout << "准备测试数据" << endl;for (int i = 0; i < 100*1000; i++){sprintf_s(buf, "%d,", i);strBuffer += buf;}cout << "测试100k split性能!" << endl;pt.start();std::vector<std::string> vecRec;boost::split(vecRec, strBuffer, boost::is_any_of(","));pt.stop();//100k个点，corei3-2120  用了26,228ms+cout << "elapsed time:" << pt.toString();}void testLargeArrayCopy(){std::vector<double> vecSrc,vecDest;cout << "void testLargeArrayCopy()  准备测试数据" << endl;unsigned nCount = 100 * 1000;for (int i = 0; i < nCount;i++){vecSrc.push_back(i);}kagula::PerformanceTest pt;pt.start();vecDest.resize(vecSrc.size());for (int i = 0; i < nCount; i++){vecDest[i] = vecSrc[i];}pt.stop();//100k个点，corei3-2120  用了29ms+cout << "elapsed time:" << pt.toString() << endl;}//测试查询指定时间位置的[cable+channel]void testQuery(){kagula::PerformanceTest pt;pt.start();std::string cable_id("00d34f28-8bb9-4f22-a032-b46468d2176a");std::string channel_id("c5e5ef1d-b31a-443b-a044-d70c64ac3354");int pos = 0;std::string outJSON;mongoDB.query(cable_id, channel_id, pos, outJSON);pt.stop();cout << "Query测试：" << pt.toString() << std::endl;cout << endl << "outJSON=>" << outJSON << endl;}int main(int argc, char *argv[]){if (mongoDB.initDB() != 0){cout << "Initialize mongoDB failed!" << endl;return -1;}//testSplit();//testLargeArrayCopy();//测试新建或更新document(记录)//testInsert();testInsertBatch();//测试返回查询结果//testQuery();cout << "input any key to close." << endl;cin.get();return 0;}

用到的头文件

#ifndef _CMongoDBWrapper_H_#define _CMongoDBWrapper_H_#pragma once#include <string>#include <vector>/*Title: MongoDB AccessAuthor: kagulaDate: 2017-09-06Environment:[1]VS2013 Update5[2]mongo-c-driver-1.7.0*/namespace kagula{class CMongoDBWrapper{public:static CMongoDBWrapper& Instance(){static CMongoDBWrapper singleton;return singleton;}~CMongoDBWrapper() { releaseDB(); }int initDB();  //初始化不一定成功int insertOrUpdate(const std::string &cable_id,//cable guidconst int &type_id,const std::string &channel_id,const int &pos,//cable中的位置，以0为base索引。const float &fInterval,//对应SQL Server nksys_serial_datas表space_interval字段const float &value);//该点的值//批处理插入更新方法int insertOrUpdate_batch(const std::string &cable_id,//cable guidconst int &type_id,const std::string &channel_id,const float &fInterval,//对应SQL Server nksys_serial_datas表space_interval字段const std::vector<double> &vecData);//一条线缆采样到的所有数据//查询历史采样信息int query(const std::string &cable_id,const std::string &channel_id,const int &pos,std::string &outJSON);private:CMongoDBWrapper() :m_nError(0){}CMongoDBWrapper(const CMongoDBWrapper&);           // Prevent copy-constructionCMongoDBWrapper& operator=(const CMongoDBWrapper&);// Prevent assignmentvoid releaseDB();void setError(int nError) { m_nError = nError; }bool isOK() { return m_nError == 0; }int m_nError;};}#endif

用到的实现文件

#include "MongoDBWrapper.h"#include <mongoc.h>#include<time.h>const char *g_uri_str = "mongodb://localhost:27017";const char *g_collectionName = "sampleCollection";mongoc_client_t *g_client;mongoc_database_t *g_database;mongoc_collection_t *g_collection;/*功能描述：[1]每天会新建一条记录用来保存当天的所有设备采样。[2]如果有n个采样设备m个位置，那每天会产生n*m条记录。例如: n=10,m+1ok，则每天会产生100k条记录，一年36,500K = 36.5M条记录。每天最大采样次数24*60=1440次=1.44k，即最多每分钟采样一次。调用BSON要注意的地方：[1]若Element的数据类型不是字符串例如非字符串类型index array, [11,12.3]使用bson_as_canonical_extended_json输出json字符串会变成下面的形式[ { "$numberInt" : "11" }, { "$numberDouble" : "12.300000000000001" } ] }]也就是说它的key是数据类型的描述。字符串类型index array,[ "a", "b", "c" ]"arrayOfValue" : [ "a", "b", "c" ] }]上面的规则也适合BSON_APPEND_INT32的调用情形。期望  "intkey":11   会变成  "intkey":{ "$numberInt" : "11" }可以使用bson_as_json代替bson_as_canonical_extended_json解决这个问题。[2]BSON_APPEND_UNDEFINED  指的是key为undefined，值为bool类型的一个节点。[3]array不能重复，比如[a,b,c]可以，但是向array插入[a,b,b]数组后会变成[a,b]。[4]document相当于关系型数据库的record或objectcollection这里被看作table。array类似C++里的Set容器，因为，里面的值不允许重复。[5]据baidu,每个document有16M限制。[6]查询条件是放到filter参数里去的。Debug常用命令[1]查看数据有没有加到collection.use kagula;db.sampleCollection.find().pretty()db.sampleCollection.find({"date":"2017-09-05","pt_pos":0}).pretty()[2]删除数据db.sampleCollection.remove({'date':'2017-09-04'})涉及的知识点[1]查询document是否存在.[2]docuemnt添加.[3]往查询结果的array里添加document.[4]查询指定"cable+channel+pos"的所有历史document，并按照时间排序.[5]批量插入和更新Collection的结构{"date": "2017-09-04","cable_id": "00d34f28-8bb9-4f22-a032-b46468d2176a","channel_id": "c5e5ef1d-b31a-443b-a044-d70c64ac3354","pt_pos": 0,"pt_interval": 0,"arrayOfSample": [{"sample_time": "15:29:33","sample_value": "0.10"},{"sample_time": "15:32:38","sample_value": "0.10"},{"sample_time": "15:40:46","sample_value": "0.20"}]}参考文档[1]《MongoDB C Driver 1.7.0》http://mongoc.org/libmongoc/current/mongoc_collection_find_with_opts.html[2]《libbson 1.7.0》http://mongoc.org/libbson/current/tutorial.html[3]《Documents》https://docs.mongodb.com/manual/core/document/#document-dot-notationhttps://docs.mongodb.com/manual/reference/operator/update/set/*///#define _DEBUG_MONGODB_#ifdef _DEBUG_MONGODB_#include <iostream>using namespace std;#endif // DEBUGnamespace kagula{void print_bt(const char* prompt, bson_t* bt){#ifdef _DEBUG_MONGODB_if (prompt!=nullptr && bt!=nullptr){char *strJSON = bson_as_canonical_extended_json(bt, NULL);if (strJSON==nullptr){return;}cout << prompt << endl;cout << strJSON << endl;bson_free(strJSON);}#endif // _DEBUG_MONGODB_}int CMongoDBWrapper::initDB(){mongoc_init();g_client = mongoc_client_new(g_uri_str);mongoc_client_set_appname(g_client, "connect-example");g_database = mongoc_client_get_database(g_client, "kagula");g_collection = mongoc_client_get_collection(g_client, "kagula", g_collectionName);return 0;}void CMongoDBWrapper::releaseDB(){mongoc_collection_destroy(g_collection);mongoc_database_destroy(g_database);mongoc_client_destroy(g_client);mongoc_cleanup();}int CMongoDBWrapper::insertOrUpdate(const std::string &cable_id,//cable guidconst int &type_id,const std::string &channel_id,const int &pos,//cable中的位置，以0为base索引。const float &fInterval,//对应SQL Server nksys_serial_datas表space_interval字段const float &value)//该点的值{if (!isOK()){return -1;}//std::string YYYY_MM_DD;std::string HHmmSS;time_t timer;time(&timer);tm t_tm;localtime_s(&t_tm, &timer);char buf[16] = { 0 };YYYY_MM_DD = [t_tm, &buf]()->std::string{memset(buf, 0, sizeof(buf));const unsigned YYYY = t_tm.tm_year + 1900;const unsigned MM = t_tm.tm_mon + 1;const unsigned DD = t_tm.tm_mday;sprintf_s(buf, "%04d-%02d-%02d", YYYY, MM, DD);return buf;}();HHmmSS = [t_tm, &buf]()->std::string{memset(buf, 0, sizeof(buf));const unsigned HH = t_tm.tm_hour;const unsigned mm = t_tm.tm_min;const unsigned SS = t_tm.tm_sec;sprintf_s(buf, "%02d:%02d:%02d", HH, mm, SS);return buf;}();//Query if exist by [date + cable_id + channel_id + pos] keybson_t *bt_query, *bt_insert;bt_query = bson_new();BSON_APPEND_UTF8(bt_query, "date", YYYY_MM_DD.c_str());BSON_APPEND_UTF8(bt_query, "cable_id", cable_id.c_str());BSON_APPEND_UTF8(bt_query, "channel_id", channel_id.c_str());BSON_APPEND_INT32(bt_query, "pt_pos", pos);#ifdef _DEBUG_MONGODB_{char *strJSON = bson_as_canonical_extended_json(bt_query, NULL);cout << "filter is [" << strJSON << "]" << endl;bson_free(strJSON);}#endifmongoc_cursor_t *cursor = mongoc_collection_find_with_opts(g_collection, bt_query, NULL, NULL);const bson_t *bt_result;bson_error_t error;if (mongoc_cursor_next(cursor, &bt_result)){//if exist,append new node.print_bt("found document:", const_cast<bson_t*>(bt_result));#ifdef _DEBUG_MONGODB_bson_iter_t iterArrayOfTime;bson_iter_t childArrayOfTime;if (bson_iter_init_find(&iterArrayOfTime, bt_result, "arrayOfSample") &&BSON_ITER_HOLDS_ARRAY(&iterArrayOfTime) && bson_iter_recurse(&iterArrayOfTime, &childArrayOfTime)) {//found arrayOfTimewhile (bson_iter_next(&childArrayOfTime)){/*cout << "key=" << bson_iter_key(&childArrayOfTime) <<",value=" << bson_iter_utf8(&childArrayOfTime, nullptr) << endl;*/cout << "key=" << bson_iter_key(&childArrayOfTime) << endl;}//while}//if#endif{//add field:value to the array of time.beginbson_t *addToSet;mongoc_find_and_modify_opts_t *opts;bson_t reply;char strValue[16] = { 0 };sprintf_s(strValue, "%.2f", value);addToSet = BCON_NEW("$addToSet", "{", "arrayOfSample", "{", "sample_time",HHmmSS.c_str(), "sample_value",strValue, "}", "}");//输出json string，不要带key(field)，可以使用下面的statement。//addToSet = BCON_NEW("$addToSet", "{", "arrayOfTime", HHmmSS.c_str(), "}");opts = mongoc_find_and_modify_opts_new();mongoc_find_and_modify_opts_set_update(opts, addToSet);bool success = mongoc_collection_find_and_modify_with_opts(g_collection, bt_query, opts, &reply, &error);if (success) {#ifdef _DEBUG_MONGODB_char *str;str = bson_as_canonical_extended_json(&reply, NULL);printf("%s\n", str);bson_free(str);#endif}else {#ifdef _DEBUG_MONGODB_fprintf(stderr, "Got error: \"%s\" on line %d\n", error.message, __LINE__);#else#endif}bson_destroy(&reply);bson_destroy(addToSet);mongoc_find_and_modify_opts_destroy(opts);}//add field:value to the array of time.end}else{//if not exist, insert new documentbt_insert = bson_new();BSON_APPEND_UTF8(bt_insert, "date", YYYY_MM_DD.c_str());BSON_APPEND_UTF8(bt_insert, "cable_id", cable_id.c_str());BSON_APPEND_UTF8(bt_insert, "channel_id", channel_id.c_str());//只要不是字符串类型，输出某个Element的JSON字符串，都会变类似这样的样子"pt_pos" : { "$numberInt" : "0" }BSON_APPEND_INT32(bt_insert, "pt_pos", pos);BSON_APPEND_DOUBLE(bt_insert, "pt_interval", fInterval);/*//add array的另一种方法:bson_t *bt_arrayOfTime = bson_new();BSON_APPEND_ARRAY_BEGIN(bt_insert, "arrayOfTime", bt_arrayOfTime);//以0为索引的element，用bson_array_as_json (&bson, NULL);就会转成["e1 value","e2 value"]这样的字符串BSON_APPEND_UTF8(bt_arrayOfTime, "0", HHmmSS.c_str());//key必须填 0 1 2...依次类推。bson_append_array_end(bt_insert, bt_arrayOfTime);BSON_APPEND_ARRAY(bt_insert, "arrayOfTime", bt_arrayOfTime);*///这是已知，把subdocument插入到array中的唯一办法，试过往array里面insert netsted-sub-document的方法，但是都没成功。char buf_json[64] = { 0 };sprintf_s(buf_json, "[{\"sample_time\":\"%s\",\"sample_value\":\"%.2f\"}]", HHmmSS.c_str(), value);bson_t *bt_arrayOfSample = bson_new_from_json((const uint8_t *)buf_json,strlen(buf_json), &error);BSON_APPEND_ARRAY(bt_insert, "arrayOfSample", bt_arrayOfSample);print_bt("found document:",bt_insert);if (!mongoc_collection_insert(g_collection, MONGOC_INSERT_NONE, bt_insert, NULL, &error)) {setError(1);#ifdef _DEBUG_MONGODB_fprintf(stderr, "%s\n", error.message);#endif}bson_destroy(bt_arrayOfSample);//bson_destroy(bt_arrayOfTime);//bson_destroy(bt_arrayOfValue);bson_destroy(bt_insert);}//if not exist document will not cause below error statementif (mongoc_cursor_error(cursor, &error)) {setError(1);#ifdef _DEBUG_MONGODB_fprintf(stderr, "Auth error: %s\n", error.message);#endifgoto _CLEANUP;}_CLEANUP:mongoc_cursor_destroy(cursor);bson_destroy(bt_query);return 0;}//批处理插入更新方法int CMongoDBWrapper::insertOrUpdate_batch(const std::string &cable_id,//cable guidconst int &type_id,const std::string &channel_id,const float &fInterval,//对应SQL Server nksys_serial_datas表space_interval字段const std::vector<double> &vecData)//一条线缆采样到的所有数据{if (!isOK())return -1;//std::string YYYY_MM_DD;std::string HHmmSS;time_t timer;time(&timer);tm t_tm;localtime_s(&t_tm, &timer);char buf[16] = { 0 };YYYY_MM_DD = [t_tm, &buf]()->std::string{memset(buf, 0, sizeof(buf));const unsigned YYYY = t_tm.tm_year + 1900;const unsigned MM = t_tm.tm_mon + 1;const unsigned DD = t_tm.tm_mday;sprintf_s(buf, "%04d-%02d-%02d", YYYY, MM, DD);return buf;}();HHmmSS = [t_tm, &buf]()->std::string{memset(buf, 0, sizeof(buf));const unsigned HH = t_tm.tm_hour;const unsigned mm = t_tm.tm_min;const unsigned SS = t_tm.tm_sec;sprintf_s(buf, "%02d:%02d:%02d", HH, mm, SS);return buf;}();//Query if exist by [date + cable_id + channel_id + pos] keybson_t *bt_query, *bt_insert;bt_query = bson_new();BSON_APPEND_UTF8(bt_query, "date", YYYY_MM_DD.c_str());BSON_APPEND_UTF8(bt_query, "cable_id", cable_id.c_str());BSON_APPEND_UTF8(bt_query, "channel_id", channel_id.c_str());BSON_APPEND_INT32(bt_query, "pt_pos", 0);mongoc_cursor_t *cursor = mongoc_collection_find_with_opts(g_collection, bt_query, NULL, NULL);const bson_t *bt_result;bson_error_t error;if (mongoc_cursor_next(cursor, &bt_result)){//位置0的记录有，就假定当天这条线缆所有位置的信息都有//接下去进行批量更新操作mongoc_bulk_operation_t *bulk = mongoc_collection_create_bulk_operation(g_collection, true, NULL);bson_error_t error;for (int i = 0; i < vecData.size();i++){bson_t *query = bson_new(); ;BSON_APPEND_UTF8(query, "date", YYYY_MM_DD.c_str());BSON_APPEND_UTF8(query, "cable_id", cable_id.c_str());BSON_APPEND_UTF8(query, "channel_id", channel_id.c_str());BSON_APPEND_INT32(query, "pt_pos", i);bson_t *bt_push;char strValue[16] = { 0 };sprintf_s(strValue, "%.2f", vecData[i]);//10k条记录，corei3-2120,更新记录用了304,432ms//addToSet = BCON_NEW("$addToSet", "{", "arrayOfSample", "{", "sample_time", HHmmSS.c_str(), "sample_value", strValue, "}", "}");//10k条记录，corei3-2120,更新记录用了211,764msbt_push = BCON_NEW("$push", "{", "arrayOfSample", "{", "sample_time", HHmmSS.c_str(), "sample_value", strValue, "}", "}");//bson_t *opts;//opts = BCON_NEW ("upsert", BCON_BOOL (true));//若不存在则添加mongoc_bulk_operation_update_many_with_opts(bulk, query, bt_push, NULL, &error);bson_destroy(query);bson_destroy(bt_push);}//forbson_t reply;bool ret = mongoc_bulk_operation_execute(bulk, &reply, &error);#ifdef _DEBUG_MONGODB_char *str = bson_as_canonical_extended_json(&reply, NULL);printf("%s\n", str);bson_free(str);#endifif (!ret) {setError(1);#ifdef _DEBUG_MONGODB_printf("Error: %s\n", error.message);#endif}bson_destroy(&reply);mongoc_bulk_operation_destroy(bulk);}else{//当天这条cable位置0的记录不存在，就假定所有位置信息不存在//进行批量添加记录的操作。mongoc_write_concern_t *wc;mongoc_bulk_operation_t *bulk;wc = mongoc_write_concern_new();mongoc_write_concern_set_w(wc, 0);bson_t reply;bulk = mongoc_collection_create_bulk_operation(g_collection, true, wc);for (int i = 0; i < vecData.size();i++){bt_insert = bson_new();BSON_APPEND_UTF8(bt_insert, "date", YYYY_MM_DD.c_str());BSON_APPEND_UTF8(bt_insert, "cable_id", cable_id.c_str());BSON_APPEND_UTF8(bt_insert, "channel_id", channel_id.c_str());BSON_APPEND_INT32(bt_insert, "pt_pos", i);BSON_APPEND_DOUBLE(bt_insert, "pt_interval", fInterval);char buf_json[64] = { 0 };sprintf_s(buf_json, "[{\"sample_time\":\"%s\",\"sample_value\":\"%.2f\"}]", HHmmSS.c_str(), vecData[i]);bson_t *bt_arrayOfSample = bson_new_from_json((const uint8_t *)buf_json,strlen(buf_json), &error);BSON_APPEND_ARRAY(bt_insert, "arrayOfSample", bt_arrayOfSample);mongoc_bulk_operation_insert(bulk, bt_insert);bson_destroy(bt_insert);bson_destroy(bt_arrayOfSample);}if (!mongoc_bulk_operation_execute(bulk, &reply, &error)) {setError(1);}bson_destroy(&reply);mongoc_bulk_operation_destroy(bulk);mongoc_write_concern_destroy(wc);}//if not exist document will not cause below error statementif (mongoc_cursor_error(cursor, &error)) {setError(1);#ifdef _DEBUG_MONGODB_fprintf(stderr, "Auth error: %s\n", error.message);#endifgoto _CLEANUP;}_CLEANUP:mongoc_cursor_destroy(cursor);bson_destroy(bt_query);return 0;}int CMongoDBWrapper::query(const std::string &cable_id,const std::string &channel_id,const int &pos,std::string &outJSON){if (!isOK()){return -1;}bson_t *bt_filter;bt_filter = bson_new();BSON_APPEND_UTF8(bt_filter, "cable_id", cable_id.c_str());BSON_APPEND_UTF8(bt_filter, "channel_id", channel_id.c_str());BSON_APPEND_INT32(bt_filter, "pt_pos", pos);bson_t *opts;/* order by "date" descending */opts = BCON_NEW("sort", "{", "date", BCON_INT32(-1), "}");mongoc_cursor_t *cursor = mongoc_collection_find_with_opts(g_collection, bt_filter, opts, NULL);const bson_t *bt_result;//通过mongoc_cursor_next可以把结果集中的所有数据取出来//这里只要取一条就够了。if (mongoc_cursor_next(cursor, &bt_result)){//char *str = bson_as_canonical_extended_json(bt_result, NULL);char *str = bson_as_json(bt_result, NULL);outJSON = str;bson_free(str);}else{outJSON.clear();}mongoc_cursor_destroy(cursor);bson_destroy(bt_filter);bson_destroy(opts);return 0;}//function}//namespace