Nodejs实现自动清理elasticsearch过期索引(日志清理)--学习笔记
来源：互联网 发布：电音入门耳机 知乎 编辑：程序博客网 时间：2024/06/06 20:41
本人是C#开发者，写起Nodejs估计还保留着浓浓的C#风格吧。
elasticsearch简单说一下，索引一般推荐以日期为单位。没啥原因大家的经验而已。
elasticsearch相关资料：https://www.elastic.co/guide/en/elasticsearch/client/index.html
https://github.com/elastic/elasticsearch
主要使用的API https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/api-reference-2-4.html#api-fieldstats-2-4
indices.delete 删除索引
indices.get 获取索引信息create 建立索引并插入信息，如存在则追加信息。
这里我们看的是2.4的，5.X的应该也没啥大问题。这些都是相对简单的。
配置文件AppConfig.js
clearhistory:
indexPrefix：索引前缀，分割符为下划线（"_"）。日期格式之间为"-"。索引格式为：xxx_xxx_2017-02-17
month：保留几个月
day:保留几天
AppConfig.js源码
var AppConfig = { debug: false, elasticSearch: { host: ['http://192.168.151.190:9200/'],//ES服务器IP地址 sniffOnStart: true, sniffInterval: 300000 }, clearhistory: [{ indexPrefix: 'xxx_xxx_', month: 0, day: 6 }, { indexPrefix: 'xxx_frontend_', month: 0, day: 1 }, { indexPrefix: 'xxx_', month: 0, day: 1 }], log4js: { appenders: [{ type: 'dateFile', absolute: true, filename: './logs/', maxLogSize: 1024 * 1024, backup: 3, pattern: 'yyyy-MM-dd.txt', alwaysIncludePattern: true, category: 'normal' }], replaceConsole: false }};module.exports = AppConfig;ClearHistoryES.js
主要函数
ClearHistoryES源码:
//清除ES日志var ClearHistoryES = function ClearHistoryES(client,appConfig,logs) { /* curl -XGET http://localhost:9200/_all * curl 方式*/ /* var request = require('request'); var $ = require('jquery')(require("jsdom").jsdom().defaultView); request.get(appConfig.elasticSearch.host[0] + "/_all", function (err, res, body) { console.log("[_all:]" + body); var json = $.parseJSON(body); console.log(json); var allIndex = getAllIndex(body); var allDelIndex = getAllDelIndex(allIndex); if(allDelIndex==''){ //console.log("清理完毕：") delIndex(allDelIndex); }else { delIndex(allDelIndex); } }); */ client.indices.get({index:'_all'},function (error,response) { //console.log("gel_allindex:"+ JSON.stringify(response)); if (error != undefined) { console.log("[获取全部索引异常：]" + JSON.stringify(error)); logs.error(JSON.stringify(error)); //logs.info(data); } else { var allIndex = getAllIndex(response); var allDelIndex = getAllDelIndex(allIndex); console.log("allDelIndex："+allDelIndex); if(allDelIndex==''){ console.log("没有需要清理的数据！"); }else { delIndex(allDelIndex); } } }); //获取ES的index function getAllIndex(indexInfo) { //var json = $.parseJSON(indexInfo); var json = indexInfo;//JSON.parse(indexInfo); //console.log(json); var arry = []; for (var key in json) { arry.push(key); } //console.log(arry); return arry; } function getAllDelIndex(allIndex) { var result = []; for (var index in allIndex) { var value = allIndex[index]; if (isDelIndex(value)) { result.push(value); } } return result; function isDelIndex(value) { //var delIndex = ''; var dateNow = getDateNow(); for (var index in appConfig.clearhistory) { var prefix = value.substring(0, value.lastIndexOf("_") + 1); var dateValue = value.substring(value.lastIndexOf("_") + 1, value.length); //console.log(prefix); //对比索引前缀是否符合自动清理规则 if (appConfig.clearhistory[index].indexPrefix == prefix) { var deadline = dateNow; //console.log(dateValue); //console.log("dateNow:" + dateNow); //要求保留的月时长 if (appConfig.clearhistory[index].month > 0) { deadline = addMonths(deadline, -appConfig.clearhistory[index].month); //console.log("月：" + deadline); } //要求保留的天数 if (appConfig.clearhistory[index].day > 0) { deadline = addDays(deadline, -appConfig.clearhistory[index].day);//加保留的天数 //console.log("天：" + deadline); } /* if (isDateLimit(formatStringToDate(dateValue), deadline)) {//日期是否超限 delIndex += value + ","; console.log("日期是否超限:"+value); console.log("delIndex:"+delIndex); }*/ return isDateLimit(formatStringToDate(dateValue), deadline); break; } } //console.log("delIndex:"+delIndex); //return delIndex == '' ? '' : delIndex.substring(0, delIndex.length - 1); //字符串格式化为日期 function formatStringToDate(valueDate) { //console.log("formatStringToDate:" + valueDate.replace(/-/g, "/")); return new Date(Date.parse(valueDate.replace(/-/g, "/"))); } function getDateNow() { var dateNow = new Date(); var year = dateNow.getFullYear(); var month = dateNow.getMonth(); var day = dateNow.getDate(); return new Date(year, month, day); //return year + "-" + month + "-" + day; //return new Date("yyyy-MM-dd"); } //添加月份 function addMonths(date, month) { var dateNow = date; var year = dateNow.getFullYear(); var month = dateNow.getMonth() + month; var day = dateNow.getDate(); if (parseInt(month / 12) > 0) { year += parseInt(month / 12) } //console.log("year:" + year + " month:" + month + " day:" + day); return new Date(year, month, day); } //添加日 function addDays(date, days) { return new Date(date.setDate(date.getDate() + days)); } //是否超过时限，超过返回true，没有是false function isDateLimit(indexDate, limitDate) { //console.log("【indexDate:】" + indexDate + " 【limitDate:】" + limitDate); //console.log(indexDate <= limitDate ? true : false); return indexDate <= limitDate ? true : false; } } } function delIndex(allDelIndex){ console.log(allDelIndex); //删除多个client.indices.delete({index: [' // xxx_2017-02-16','xxx_2017-02-15']})； //"{index: ['xxx_2017-02-16','xxx_2017-02-15']}"; client.indices.delete({index: allDelIndex}, function (error, response) { //console.log("client.indices.delete:"+error); if (error != undefined) { console.log("[清理异常]" + error); logs.error(error); }else{ console.log("[清理完毕]" + allDelIndex); } }); /* //使用$ curl -XDELETE 'http://localhost:9200/twitter,fgfg,ghjg/' //allDelIndex ="['xxx_2017-02-16,xxx_sdk_2017-02-16']"; request.delete(appConfig.elasticSearch.host[0] + allDelIndex, function (err, res, body) { console.log("清理情况：" + body); });*/ }}module.exports = ClearHistoryES;ESLogsServer.js
ESLogsServer源码
/** * Created by gongzy on 2016/7/12. */var appConfig =require('../AppConfig.js');var log4js = require('log4js');log4js.configure(appConfig.log4js);var logs = log4js.getLogger('normal');var express = require("express");var app = express();var elasticsearch = require('elasticsearch');var client = elasticsearch.Client(appConfig.elasticSearch);var heartbeatServicEe = require('./HeartBeatService');var hbs = heartbeatService.getInstance(app);//var request = require('request');//var $ = require('jquery')(require("jsdom").jsdom().defaultView);var clearHistoryES = require('./ClearHistoryES');init();function init() { timerHealthStart(1000 * 60 * 5);//5分钟检测一次 timerClearHistoryStart(1000 * 60 * 60 * 24);//一天运行一次 acceptLogData();//；将日志记录到ES及文本中。双份日志。 app.listen(8000); getClientHealth();//先执行一次ClientHealth检查 clearHistoryES(client, appConfig, logs);//先执行一次历史数据清除}function acceptLogData(){ app.post('/ES', function(req, res) { var content = ''; req.on('data', function (data) { content += data; }); req.on('end', function () { var data = JSON.parse(content); logs.info(data); try { if(appConfig.debug){ console.log(data); } esCreateFiles(data); } catch (error){ console.log("[error]:"+error.name+error.message); hbs.state =false; logs.error(error); res.end("false"); } res.writeHead(200, { "Content-Type": "text/plain;charset=utf-8" }); res.end("true"); }); });}var esCreateFiles=function(data) { client.create(data , function (error, response) { //console.log("client.create.error:"+error); if (error != undefined) { console.log("[异常]" + error); logs.error(error); logs.info(data); } });};function timerHealthStart(millisecond) { var timerToken = setInterval(function () { getClientHealth(); }, millisecond );}function timerClearHistoryStart(millisecond) { var timerToken = setInterval(function () { getClientHealth(); }, millisecond );}//获取集群状态function getClientHealth(){ client.cluster.health(function (err, response) { if (err) { hbs.state = false ; console.error(err.message); } else { if(response.status=='red'){ hbs.state = false; }else{ hbs.state = true; } if(appConfig.debug){ console.dir(response); } } });}HeartBeatService.JS详见：http://blog.csdn.net/gzy11/article/details/54972798
源码下载地址：http://download.csdn.net/detail/gzy11/9757117
0 0
- Nodejs实现自动清理elasticsearch过期索引(日志清理)--学习笔记
- 清理Elasticsearch中过期的日志索引
- elasticsearch索引自动清理
- elasticsearch索引自动清理
- elasticsearch索引自动清理
- elasticsearch索引自动清理
- Linux_df_du_find_清理过期日志
- linux 清理过期日志
- 清理elasticsearch的索引
- 使用groovy脚本自动清理过多Elasticsearch索引
- zookeeper3.4.6配置实现自动清理日志
- zookeeper3.4.6配置实现自动清理日志
- 自动清理数据库日志
- zookeeper自动清理日志
- IIS日志自动清理
- Linux自动清理日志
- SQL SERVER日志文件清理 学习笔记
- 清理MySQL数据库的过期binlog日志
- 2017峰会：项目管理与高级项目管理（深圳站）
- java实现spark
- Jquery 3.1.1 Syntax error, unrecognized expression: #
- 设置silverlight启用剪贴板的功能
- Android框架之Picasso和Glide的使用及区别
- Nodejs实现自动清理elasticsearch过期索引(日志清理)--学习笔记
- Linux学习笔记之Linux常用命令行
- myeclipse导入项目后项目上有粗体红色感叹号
- 给 Android 开发者的 RxJava 详解
- Android框架之volley的使用
- 前端跨域知识总结
- LintCode Interleaving Positive and Negative Numbers
- ajax请求到参数太大获取参数失败问题
- MyBatis 配置sql语句输出
