node csv文件流读取
来源：互联网 发布：sql数据库开发 编辑：程序博客网 时间：2024/04/29 03:18
csv文件流读取，可以应对大文件,数据截断发送，不会出现内存不足的情况function readInAppEventReports(user,date,filePath, callback) { var reports = []; var google_facebook_reports=[]; var other_reports=[]; var item_other = {}; var item_fg={}; var dbFlag = true; var getCount = 0; var tCount = 0; //api获取数据条数; var otherCount=0; var facebook_google_Count=0; var other_reports_send_config=5000; var google_facebook_reports_send_config=5000; current_site={ id : filePath.split('=')[1].split('&')[0] }; async.waterfall([ function (cb) { readStream = fs.createReadStream(filePath); console.log('---test', filePath); readStream.pipe(csv()) .on('data', function (data) { // readStream.pause(); //数据封装，打包 var media_source=data['Media Source']; if (media_source==="googleadwords_int" || media_source==="Facebook Ads") { item_fg=packItemForInAppEventReportFacebookGoogle(data); if (item_fg.err) { console.log('item_fg.err',item_fg.err); return callback(null); } else { eventDataFilterFacebookGoogle(item_fg.value,google_facebook_reports); } } else { item_other = packItemForInAppEventReport(data); if (item_other.err) { console.log('item_other.err', item_other.err); return callback(null); } else { //数据重复过滤 eventDataFilter(item_other.value, other_reports); } } tCount++; //发送其余数据 async.waterfall([ function (fn) { if (other_reports.length >= other_reports_send_config && dbFlag) { console.log('on data... send before other_reports.length=',other_reports.length); dbFlag = false; readStream.pause(); var temOther = other_reports.slice(0, other_reports_send_config); Dbapi.sendMsg({ json: { actionid: 8220, noat: true, events: temOther } }, function (err, result) { console.log('other_reports Dbapi sendMsg', 8220, err, result); getCount += temOther.length; otherCount+=temOther.length; console.log('on data other_reports tCount', tCount, 'getCount', getCount,'otherCount',otherCount); other_reports.splice(0, temOther.length); console.log('on data... after other_reports.length=',other_reports.length); dbFlag = true; readStream.resume(); fn(err); }, 2); } else { fn(null); } }, function (fn) { //发送facebook，google数据 // console.log('google_facebook_reports.length=',google_facebook_reports.length); if (google_facebook_reports.length >= google_facebook_reports_send_config && dbFlag) { console.log('on data ...send before google_facebook_reports.length=',google_facebook_reports.length); dbFlag = false; readStream.pause(); //发送其余数据 var temp_google_facebook_ = google_facebook_reports.slice(0, google_facebook_reports_send_config); Dbapi.sendEvent({ json: { actionid: 1001, noat: true, events: temp_google_facebook_ } }, function (err, result) { console.log('google_facebook_reports Dbapi sendMsg', 1001, err, result); getCount += temp_google_facebook_.length; facebook_google_Count+=temp_google_facebook_.length; console.log('on data google_facebook_reports tCount', tCount, 'getCount', getCount,'facebook_google_Count',facebook_google_Count); google_facebook_reports.splice(0, temp_google_facebook_.length); console.log('on data... send after google_facebook_reports.length=',google_facebook_reports.length); dbFlag = true; readStream.resume(); fn(err); }, 2); } else { fn(null); } } ],function (err) { if (err) console.log('readInAppEventReports on(data) ',err); if (tCount>=API_MAX) { reports.push(data); } }); }) .on('end', function () { var otherMsgSend=function (buff,cb) { Dbapi.sendMsg({ json: { actionid: 8220, noat: true, events: buff } }, function (err, result) { console.log('other_reports Dbapi sendMsg', 8220, err, result); getCount += buff.length; otherCount+=buff.length; console.log('other_reports *** tCount', tCount, 'getCount', getCount,'otherCount',otherCount); cb(err); }, 2); }; var fgMsgSend=function (buff,cb) { Dbapi.sendEvent({ json: { actionid: 1001, noat: true, events: buff } }, function (err, result) { console.log('google_facebook_reports Dbapi sendMsg', 1001, err, result); getCount += buff.length; facebook_google_Count+=buff.length; console.log('google_facebook_reports *** tCount', tCount, 'getCount', getCount,'facebook_google_Count',facebook_google_Count); cb(err); }, 2); }; async.waterfall([ function (fn) { console.log('on end read other_reports'); console.log('on end send before other_reports.length =',other_reports.length ); if (other_reports.length !== 0) { var flag=true; async.whilst( function () { return flag; }, function (fn_cb) { console.log('on end send..other_reports.length =',other_reports.length ); if (other_reports.length > other_reports_send_config){ flag=true; var buff=other_reports.splice(0,other_reports_send_config); otherMsgSend(buff,function (err) { fn_cb(err); }); } else { flag=false; return fn_cb(null) } }, function (err) { if (err) console.log(err); otherMsgSend(other_reports,function (error) { fn(error); }); }); } else { fn(null) } }, function (fn) { console.log('on end read google_facebook_reports'); console.log('on end send before google_facebook_reports.length=',google_facebook_reports.length); if (google_facebook_reports.length !==0) { var flag=true; async.whilst( function () { return flag; }, function (fn_cb) { console.log('on end send...google_facebook_reports.length=',google_facebook_reports.length); if (google_facebook_reports.length > google_facebook_reports_send_config) { flag=true; var buff=google_facebook_reports.splice(0,google_facebook_reports_send_config); fgMsgSend(buff,function (err) { fn_cb(err); }); } else { flag=false; return fn_cb(null); } }, function (err) { if (err) console.log(err); fgMsgSend(google_facebook_reports,function (error) { fn(error); }); }); } else { fn(null); } } ],function (err) { if (err) console.log('readInAppEventReports on(end) ',err); cb(null); }); }); }, function (cb) { // fs.unlink(filePath, function (err) { // cb(err); // }); cb(null); } ], function (err) { if (err) console.log(err); async.waterfall([ function (fn) { console.log('tCount=',tCount,'API_MAX=',API_MAX); if (tCount >= API_MAX - 20000) { console.log('readInAppEventReports API_MAX - 20000 Monitor ...'); var apiErr = Monitor.errFactory(); apiErr.err = '数据不完整'; apiErr.funct = readInstallsReport.toString(); apiErr.param = filePath; apiErr.level = 10; Monitor.send(apiErr); } fn(null); }, function (fn) { console.log('tCount=',tCount,'API_MAX=',API_MAX); if (tCount >= API_MAX) { console.log('readInAppEventReports data >',API_MAX/1000,'K'); // console.log('reports.length=',reports.length); // console.log(JSON.stringify(reports[0])); // var str='2017/7/10 23:59:59'; var detail_date=[]; detail_date=reports[0]['Event Time'].substr(10,6).split(':'); detail_date[0]=detail_date[0].replace(" ",""); if (parseInt(detail_date[1])<59) { detail_date[1]=parseInt(detail_date[1])+1; } if (parseInt(detail_date[1])===59) { detail_date[0]=parseInt(detail_date[0])+1; detail_date[1]='00'; } console.log('detail_date:',detail_date); var param={ detail_date:detail_date, website:current_site }; setTimeout(function () { inAppEventReport(user, date, param, function (err) { console.log('inAppEventReport end err', err); Monitor.send('read event report'); fn(err); }); },3000); } else if (tCount===6 || other_reports.length===0 || google_facebook_reports.length===0) { console.log('当前时间',new Date().toLocaleDateString(),'待获取应用日期 date=',date,' website= ',current_site.id,'应用超过请求次数'); fn(null); } else { console.log('当前时间',new Date().toLocaleDateString(),'待获取应用日期 date=',date,' website= ',current_site.id,'该天没有数据'); fn(null); } } ],function (err) { callback(err); }); });}集合遍历，eachLimit里面的方法会并行 async.eachLimit(users[user].websites, users[user].websites.length, function (website, cb) { console.log('inAppEventReports user=',user,'date=',date,'website=',website); loadInAppEventReports(user, website, date, function (err, path) { if (path) filesPath = filesPath.concat(path); console.log('loadInAppEventReports end', website.id, 'err', err, new Date().toString()); cb(null); }) }, function (err) { cba(err, filesPath); });可以将limit设置为1 使其串行 async.eachLimit(dateList.slice(0,6), 1, function (dateItem, callback) { console.log('waterfall-start', dateItem); async.waterfall([ function (cb) { mainFunction(dateItem, dateItem, function (err) { cb(null); }); } ], function (err) { callback(null); }); }, function (err) { console.error('history end err', err); });阅读全文
0 0
- node csv文件流读取
- node.js 读取csv文件数据处理导入mongodb 数据库
- node文件读取流
- 如何读取csv文件
- CSV文件直接读取
- C#读取CSV文件
- java读取CSV文件
- ajax 读取 .csv 文件
- Java读取CSV文件
- Java 读取 CSV 文件
- 读取.csv文件数据
- csv文件读取
- 读取/输出 CSV文件
- java读取csv文件
- c#读取csv文件
- php读取csv文件
- Python-读取csv文件
- DataTable 读取CSV文件
- java统计list不同元素，满足n个返回
- 52.Scala中路径依赖代码实战详解
- 二叉树的路径和-LintCode
- PHP创建迅雷、快车、旋风链接
- Hrbust 2319 Number Game【思维+优先队列】
- node csv文件流读取
- iOS Podfile文件常用库
- 53.Scala中结构类型实战详解
- Codeforces 730 C Bulmart [bfs+贪心]
- Hibernate 配置 MySQL dialect 解决由MySQL版本导致的问题
- 缓解数据倾斜的方案浅谈
- 手把手教你使用 axis1.4 生成客户端代码
- Java SE8 Lambda 基础入门---为什么需要lambda
- ubuntu常用命令
