Nodejs实现自动清理elasticsearch过期索引(日志清理)--学习笔记

本人是C#开发者，写起Nodejs估计还保留着浓浓的C#风格吧。
elasticsearch简单说一下，索引一般推荐以日期为单位。没啥原因大家的经验而已。
elasticsearch相关资料：https://www.elastic.co/guide/en/elasticsearch/client/index.html
                                       https://github.com/elastic/elasticsearch 主要使用的API https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/api-reference-2-4.html#api-fieldstats-2-4

indices.delete 删除索引

indices.get 获取索引信息

create 建立索引并插入信息，如存在则追加信息。

这里我们看的是2.4的，5.X的应该也没啥大问题。这些都是相对简单的。
配置文件AppConfig.js
clearhistory:
      indexPrefix：索引前缀，分割符为下划线（"_"）。日期格式之间为"-"。索引格式为：xxx_xxx_2017-02-17
      month：保留几个月
      day:保留几天
AppConfig.js源码

var AppConfig = {
debug: false,
elasticSearch: {
host: ['http://192.168.151.190:9200/'],//ES服务器IP地址
sniffOnStart: true,
sniffInterval: 300000
},
clearhistory: [{
indexPrefix: 'xxx_xxx_',
month: 0,
day: 6
}, {
indexPrefix: 'xxx_frontend_',
month: 0,
day: 1
}, {
indexPrefix: 'xxx_',
month: 0,
day: 1
}],
log4js: {
appenders: [{
type: 'dateFile',
absolute: true,
filename: './logs/',
maxLogSize: 1024 * 1024,
backup: 3,
pattern: 'yyyy-MM-dd.txt',
alwaysIncludePattern: true,
category: 'normal'
}],
replaceConsole: false
}
};
module.exports = AppConfig;

ClearHistoryES.js
主要函数



ClearHistoryES源码:

//清除ES日志
var ClearHistoryES = function ClearHistoryES(client,appConfig,logs) {
/* curl -XGET http://localhost:9200/_all 
* curl 方式*/
/*
var request = require('request');
var $ = require('jquery')(require("jsdom").jsdom().defaultView);
request.get(appConfig.elasticSearch.host[0] + "/_all", function (err, res, body) {
console.log("[_all:]" + body);
var json = $.parseJSON(body);
console.log(json);
var allIndex = getAllIndex(body);
var allDelIndex = getAllDelIndex(allIndex);

if(allDelIndex==''){
//console.log("清理完毕：")
delIndex(allDelIndex);
}else {
delIndex(allDelIndex);
}
});
*/
client.indices.get({index:'_all'},function (error,response) {
//console.log("gel_allindex:"+ JSON.stringify(response));
if (error != undefined) {
console.log("[获取全部索引异常：]" + JSON.stringify(error));
logs.error(JSON.stringify(error));
//logs.info(data);
}
else {
var allIndex = getAllIndex(response);
var allDelIndex = getAllDelIndex(allIndex);
console.log("allDelIndex："+allDelIndex);
if(allDelIndex==''){
console.log("没有需要清理的数据！");
}else {
delIndex(allDelIndex);
}
}
});
//获取ES的index
function getAllIndex(indexInfo) {
//var json = $.parseJSON(indexInfo);
var json = indexInfo;//JSON.parse(indexInfo);
//console.log(json);
var arry = [];
for (var key in json) {
arry.push(key);
}
//console.log(arry);
return arry;
}

function getAllDelIndex(allIndex) {
var result = [];
for (var index in allIndex) {
var value = allIndex[index];
if (isDelIndex(value)) {
result.push(value);
}
}
return result;
function isDelIndex(value) {
//var delIndex = '';
var dateNow = getDateNow();
for (var index in appConfig.clearhistory) {
var prefix = value.substring(0, value.lastIndexOf("_") + 1);
var dateValue = value.substring(value.lastIndexOf("_") + 1, value.length);
//console.log(prefix);
//对比索引前缀是否符合自动清理规则
if (appConfig.clearhistory[index].indexPrefix == prefix) {
var deadline = dateNow;
//console.log(dateValue);
//console.log("dateNow:" + dateNow);
//要求保留的月时长
if (appConfig.clearhistory[index].month > 0) {
deadline = addMonths(deadline, -appConfig.clearhistory[index].month);
//console.log("月：" + deadline);
}
//要求保留的天数
if (appConfig.clearhistory[index].day > 0) {
deadline = addDays(deadline, -appConfig.clearhistory[index].day);//加保留的天数
//console.log("天：" + deadline);
}
/*
if (isDateLimit(formatStringToDate(dateValue), deadline)) {//日期是否超限
delIndex += value + ",";
console.log("日期是否超限:"+value);
console.log("delIndex:"+delIndex);
}*/
return isDateLimit(formatStringToDate(dateValue), deadline);
break;
}
}
//console.log("delIndex:"+delIndex);
//return delIndex == '' ? '' : delIndex.substring(0, delIndex.length - 1);
//字符串格式化为日期
function formatStringToDate(valueDate) {
//console.log("formatStringToDate:" + valueDate.replace(/-/g, "/"));
return new Date(Date.parse(valueDate.replace(/-/g, "/")));
}

function getDateNow() {
var dateNow = new Date();
var year = dateNow.getFullYear();
var month = dateNow.getMonth();
var day = dateNow.getDate();
return new Date(year, month, day);
//return year + "-" + month + "-" + day;
//return new Date("yyyy-MM-dd");
}

//添加月份
function addMonths(date, month) {
var dateNow = date;
var year = dateNow.getFullYear();
var month = dateNow.getMonth() + month;
var day = dateNow.getDate();
if (parseInt(month / 12) > 0) {
year += parseInt(month / 12)
}
//console.log("year:" + year + " month:" + month + " day:" + day);
return new Date(year, month, day);

}

//添加日
function addDays(date, days) {
return new Date(date.setDate(date.getDate() + days));
}

//是否超过时限，超过返回true，没有是false
function isDateLimit(indexDate, limitDate) {
//console.log("【indexDate:】" + indexDate + " 【limitDate:】" + limitDate);
//console.log(indexDate <= limitDate ? true : false);
return indexDate <= limitDate ? true : false;
}
}
}

function delIndex(allDelIndex){
console.log(allDelIndex);
//删除多个client.indices.delete({index: ['
// xxx_2017-02-16','xxx_2017-02-15']})；
//"{index: ['xxx_2017-02-16','xxx_2017-02-15']}";
client.indices.delete({index: allDelIndex}, function (error, response) {
//console.log("client.indices.delete:"+error);
if (error != undefined) {
console.log("[清理异常]" + error);
logs.error(error);
}else{
console.log("[清理完毕]" + allDelIndex);
}

});
/*
//使用$ curl -XDELETE 'http://localhost:9200/twitter,fgfg,ghjg/'
//allDelIndex ="['xxx_2017-02-16,xxx_sdk_2017-02-16']";
request.delete(appConfig.elasticSearch.host[0] + allDelIndex, function (err, res, body) {
console.log("清理情况：" + body);
});*/
}
}
module.exports = ClearHistoryES;

ESLogsServer.js

ESLogsServer源码

/**
* Created by gongzy on 2016/7/12.
*/
var appConfig =require('../AppConfig.js');
var log4js = require('log4js');
log4js.configure(appConfig.log4js);
var logs = log4js.getLogger('normal');
var express = require("express");
var app = express();
var elasticsearch = require('elasticsearch');
var client = elasticsearch.Client(appConfig.elasticSearch);
var heartbeatServicEe = require('./HeartBeatService');
var hbs = heartbeatService.getInstance(app);
//var request = require('request');
//var $ = require('jquery')(require("jsdom").jsdom().defaultView);
var clearHistoryES =  require('./ClearHistoryES');
init();
function  init() {

timerHealthStart(1000 * 60 * 5);//5分钟检测一次
timerClearHistoryStart(1000 * 60 * 60 * 24);//一天运行一次
acceptLogData();//；将日志记录到ES及文本中。双份日志。
app.listen(8000);
getClientHealth();//先执行一次ClientHealth检查
clearHistoryES(client, appConfig, logs);//先执行一次历史数据清除
}
function  acceptLogData(){
app.post('/ES', function(req, res) {
var content = '';
req.on('data', function (data) {
content += data;
});
req.on('end', function () {
var data = JSON.parse(content);
logs.info(data);
try {
if(appConfig.debug){
console.log(data);
}
esCreateFiles(data);
}
catch (error){
console.log("[error]:"+error.name+error.message);
hbs.state =false;
logs.error(error);
res.end("false");
}
res.writeHead(200, {
"Content-Type": "text/plain;charset=utf-8"
});
res.end("true");
});
});
}

var esCreateFiles=function(data) {
client.create(data
, function (error, response) {
//console.log("client.create.error:"+error);
if (error != undefined) {
console.log("[异常]" + error);
logs.error(error);
logs.info(data);
}
});
};

function  timerHealthStart(millisecond) {
var timerToken = setInterval(function () {
getClientHealth();
},
millisecond
);
}
function timerClearHistoryStart(millisecond) {
var timerToken = setInterval(function () {
getClientHealth();
},
millisecond
);
}
//获取集群状态
function getClientHealth(){
client.cluster.health(function (err, response) {
if (err) {
hbs.state = false ;
console.error(err.message);
} else {
if(response.status=='red'){
hbs.state = false;
}else{
hbs.state = true;
}
if(appConfig.debug){
console.dir(response);
}
}
});
}

HeartBeatService.JS

详见：http://blog.csdn.net/gzy11/article/details/54972798
源码下载地址：http://download.csdn.net/detail/gzy11/9757117