JAVA定时器每日删除ES过期索引

JAVA定时器每日删除ES过期索引

背景：在Linux服务器上，设置一个后台进程，每天0点10分自动删除ELK中Elasticsearch中超过三个月的旧索引，使用Quartz实现。

废话少说，直接上代码！！！

目录：

1、源码：

2、调用方法：

1、源码：

（1）QuartzCli.java

package com.remoa.quartzCli;import org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import org.quartz.*;import org.quartz.impl.StdSchedulerFactory;/** * This class is used to use Quartz * @author remoa * @since 2017.09.18 */public class QuartzCli{    //日志记录    private static final Log LOG = LogFactory.getLog(QuartzCli.class);    //后台流程每天0点10分开始跑    //public static final String INDEX_DELETE_CRON = "0 10 0 * * ?";    //后台流程每10秒开始跑    private static final String INDEX_DELETE_CRON = "*/10 * * * * ?";    //定义Job名字    private static final String JOB_NAME = "Index_quartz_cli";    /**     * 启动定时任务     */    public void start(){        System.out.println("Start quartz mode: " + INDEX_DELETE_CRON);        LOG.info("Start quartz mode: " + INDEX_DELETE_CRON);        //SchedulerFactory：提供用于获取调度程序实例的客户端可用句柄的机制        SchedulerFactory schedulerFactory = new StdSchedulerFactory();        //Scheduler：调度程序，接口，代表一个独立运行容器，调度程序维护JobDetail和触发器的注册表，一旦注册，调度程序负责执行作业。        Scheduler scheduler = null;        try {            //设置调度程序的实现类            scheduler = schedulerFactory.getScheduler();        } catch (SchedulerException e) {            LOG.error(e.getMessage(), e);        }        //实例化JobDetail调度任务，JobDetail存放作业的状态，JobDetail对象储存作业的侦听器、群组、数据映射、描述以及作业的其它属性。        //newJob方法设置需要执行的Job任务类名        JobBuilder jobBuilder = JobBuilder.newJob(MyJob.class);        //withIdentity设置JobDetail的name和group        jobBuilder.withIdentity(JOB_NAME, "remoaGroup");        //JobDetail是一个接口，必须通过JobBuilder的build方法实例化JobDetail        JobDetail job = jobBuilder.build();        //实例化Trigger触发器，实现对作业的调度。        CronTrigger cronTrigger = (CronTrigger)TriggerBuilder.newTrigger()                .withIdentity("Cron_Trigger" + System.currentTimeMillis(), "remoaGroup")                .withSchedule(CronScheduleBuilder.cronSchedule(INDEX_DELETE_CRON))                .startNow()                .build();        //JobDataMap是java map的具体实现，并添加了一些便利的方法用于存储与读取原生类型数据，可以为Job实例提供属性/配置        job.getJobDataMap()                .put("executor", this);        try {            //为调度程序设置job和trigger            scheduler.scheduleJob(job, cronTrigger);            //调用start开始调度            scheduler.start();        } catch (Exception e) {            LOG.error(e.getMessage(), e);        }    }    public static void main(String[] args) {        QuartzCli quartzCli = new QuartzCli();        quartzCli.start();    }}

（2）MyJob.java

package com.remoa.quartzCli;import com.remoa.index.IndexProcess;import org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import org.quartz.Job;import org.quartz.JobExecutionContext;import org.quartz.JobExecutionException;import java.io.IOException;import java.io.InputStream;import java.text.SimpleDateFormat;import java.util.Date;import java.util.List;import java.util.Properties;/** * This class is used to define job * @author remoa * @since 2017.09.18 */public class MyJob implements Job {    private static final Log LOG = LogFactory.getLog(MyJob.class);    //在Quartz中，所有的任务都必须实现Job接口，Job中只有一个execute(JobExecutionContext jobExecutionContext)实现方法，具体业务实现需要写在这个方法里，这个方法将会在触发器满足调度条件时触发。    @Override    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");        System.out.println(sdf.format(new Date()) + "------------- start job");        LOG.info(sdf.format(new Date()) + "------------- start job");        InputStream input = MyJob.class.getClassLoader().getResourceAsStream("indexDelete.properties");        Properties properties = new Properties();        try {            properties.load(input);        } catch (IOException e) {            LOG.error(e.getMessage(), e);        }        IndexProcess indexProcess = null;        try {            indexProcess = new IndexProcess(properties.getProperty("keystorePath"), properties.getProperty("truststorePath"), properties.getProperty("keypassword"), properties.getProperty("trustpassword"), properties.getProperty("elasticsearchHost"), properties.getProperty("elasticsearchClusterName"));        } catch (Throwable throwable) {            LOG.error(throwable.getMessage());            throwable.printStackTrace();        }        System.out.println("get index-------------");        List<String> list = indexProcess.getIndex();        System.out.println("Total number of lists :" + list.size());        LOG.info("Total number of lists :" + list.size());        String keyword = indexProcess.getKeyword();        indexProcess.deleteOldIndex(list, keyword);    }}

（3）IndexProcess.java

package com.remoa.index;import com.floragunn.searchguard.ssl.SearchGuardSSLPlugin;import com.floragunn.searchguard.ssl.util.SSLConfigConstants;import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;import org.elasticsearch.client.AdminClient;import org.elasticsearch.client.transport.TransportClient;import org.elasticsearch.common.settings.Settings;import org.elasticsearch.common.transport.InetSocketTransportAddress;import org.elasticsearch.transport.client.PreBuiltTransportClient;import java.net.InetAddress;import java.util.ArrayList;import java.util.Calendar;import java.util.Iterator;import java.util.List;/** * This class is used to delete old index more than 100 days in elasticsearch. * @author remoa * @since 2017.09.18 */public class IndexProcess {    private TransportClient client;    private AdminClient adminClient;    public static final int DATE_NUMBER = -90;    /**     * 构造方法，提供连接Elasticsearch的相关参数并使用传输客户端TransportClient连接到Elasticsearch集群     * @param keystorePath KeyStore签名证书位置     * @param truststorePath Truststore签名证书位置     * @param keypwd KeyStore签名证书密码     * @param trustpwd Truststore签名证书密码     * @param esHost Elasticsearch集群服务器     * @param esClusterName Elasticsearch集群名称     * @throws Throwable 异常不单独处理了，直接在方法中抛出     */    public IndexProcess(String keystorePath, String truststorePath, String keypwd, String trustpwd, String            esHost, String esClusterName) throws Throwable{        Settings settings = Settings.builder()                //ES集群名称                .put("cluster.name", esClusterName)                .put("searchguard.ssl.transport.enabled", true)                //私钥证书位置                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_KEYSTORE_FILEPATH, keystorePath)                //公钥证书位置                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_TRUSTSTORE_FILEPATH, truststorePath)                //keystore一般保存我们的私钥，用来加密解密或者为别人做签名。                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_KEYSTORE_PASSWORD, keypwd)                //truststore里存放的是只包含公钥的数字证书。                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_TRUSTSTORE_PASSWORD, trustpwd)                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION, false)                .put("path.home", ".")                //使得客户端去嗅探整个集群的状态，把集群中其它机器的IP地址加到客户端中，设置为true则不用手动设置集群里所有机器的IP连接到客户端，它会自动帮助添加，并自动发现新加入集群的机器。                .put("client.transport.sniff", true)                .build();        client = new PreBuiltTransportClient(settings, SearchGuardSSLPlugin.class)                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esHost), 9301))                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esHost), 9300));        adminClient = client.admin();    }    /**     * 根据索引名删除索引     * @param index 索引名     */    public void deleteIndex(String index){        adminClient.indices().                delete(new DeleteIndexRequest(index)).                actionGet();    }    /**     * 得到全部索引     * @return 返回List封装的全部索引的名字     */    public List<String> getIndex(){        List<String> list = new ArrayList<String>();        String [] indices = adminClient.indices()                .prepareGetIndex()                .setFeatures()                .get()                .getIndices();        for(int i = 0; i < indices.length; i++){            list.add(indices[i]);        }        return list;    }    /**     * 得到过滤的关键字，即xxxx.xx.xx(年月日字符串)，这里设置为90天     * @return 过滤关键字     */    public String getKeyword(){        Calendar calendar = Calendar.getInstance();        calendar.add(Calendar.DATE, DATE_NUMBER);        String year = String.valueOf(calendar.get(Calendar.YEAR));        String month = String.valueOf(calendar.get(Calendar.MONTH) + 1);        String day = String.valueOf(calendar.get(Calendar.DATE));        if(month.length() == 1){            month = "0" + month;        }        if(day.length() == 1){            day = "0" + day;        }        String keyword = year + "." + month + "." + day;        return keyword;    }    /**     * 删除所有的超期的旧索引     * @param list 索引列表     * @param keyword 过滤关键字     */    public void deleteOldIndex(List<String> list, String keyword){        int count = 0;        for(Iterator<String> iter = list.iterator(); iter.hasNext(); ){            String str = iter.next();            //索引名称字段长度比10小的话就不作为删除对象了            if(str.length() > 10){                String dateStr = str.substring(str.length() - 10 ,str.length());//取出索引名称字段的结尾字段                //索引名称字段最后不是以日期结尾的话也不作为删除对象                String patternStr = "^[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}$";                if(dateStr.matches(patternStr)){                    if(str.substring(str.length() - 10 ,str.length()).compareTo(keyword) < 0){                        System.out.println("index： " + str + " is deleted");                        //this.deleteIndex(str);                        count++;                    }                }            }        }        System.out.println("The total delete index is: " + count);    }}

（4）log4j2.xml

<?xml version="1.0" encoding="UTF-8" ?><!-- monitorInterval：自动检测修改配置文件和重新配置本身，设置间隔秒数--><Configuration stauts="WARN" monitorInterval="60">    <!-- Appender附加器对象，负责输出日志信息到不同的地方，如数据库，文件，控制台等等-->    <Appenders>        <Console name="Console" target="SYSTEM_ERR">            <!--                输出日志的格式                %-5p：用空格右垫，当事件级别的名称少于五个字符                %F：用于输出被发出日志记录请求的文件名，带.java后缀及包名称                %L：用于输出从被发出日志记录请求的行号                %m：输出记录事件的消息内容                %x：用于与产生该日志事件的线程相关联输出的NDC                %n：输出行分隔符或文字                %t：用于输出生成的日志记录事件的线程的名称，比如在main方法中，将输出main            -->            <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss} [%-5p] --> %F(%L): %m %x %n" />        </Console>        <!-- 每次大小超过size，则size大小的日志会自动存入按年份-月份建立的文件夹下面并进行压缩，作为存档。 -->        <RollingFile name="RollingFileInfo" fileName="/root/remoa/logs/deleteIndex.log" filePattern="/root/remoa/logs/$${date:yyyy-MM}/deleteIndex-%d{yyyy-MM-dd}-%i.log">            <!-- 只输出level及以上级别的事件，其它的直接拒绝 -->            <ThresholdFilter level="warn" onMatch="ACCEPT" onMismatch="DENY" />            <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss,SSS} [%C{2}] [%p %L] [%t] %m [ %x] %n" />            <Policies>                <TimeBasedTriggerPolicy/>                <SizeBasedTriggerPolicy size="100MB" />            </Policies>            <!-- 设置默认同一个文件夹下文件数目，默认为7个，这里设置为10个 -->            <DefaultRolloverStrategy max="10" />        </RollingFile>    </Appenders>    <!-- 定义记录器对象，Logger对象负责捕获日志信息及它们存储在一个空间的层次结构 -->    <Loggers>        <!-- 全局的级别为info，可通过具体appender中的ThresholdFilter进行设置别的级别 -->        <Root level="info">            <!-- 只有引入了logger并引入的appender，appender才会生效 -->            <AppenderRef ref="Console" />            <AppenderRef ref="RollingFileInfo" />        </Root>    </Loggers></Configuration>

（5）pom.xml

<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0"         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">    <modelVersion>4.0.0</modelVersion>    <groupId>com.remoa</groupId>    <artifactId>indexTest</artifactId>    <version>1.0-SNAPSHOT</version>    <build>        <finalName>indexTest</finalName>        <!-- 默认源代码和资源文件目录配置 -->        <sourceDirectory>src/main/java </sourceDirectory>        <testSourceDirectory>src/test/java</testSourceDirectory>        <resources>            <resource>                <directory>src/main/resources</directory>            </resource>        </resources>        <testResources>            <testResource>                <directory>src/test/resources</directory>            </testResource>        </testResources>        <plugins>            <!-- 编译插件，处理maven项目管理因为版本不一致导致编译不通过的问题 -->            <plugin>                <artifactId>maven-compiler-plugin</artifactId>                <version>3.3</version>                <configuration>                    <!-- 源代码使用的开发版本 -->                    <source>1.8</source>                    <!-- 需要生成的目标class文件的编译版本 -->                    <target>1.8</target>                </configuration>            </plugin>            <plugin>                <!-- 将依赖的某个jar包内部的类或者资源include/exclude掉-->                <!-- 通过maven-shade-plugin插件生成一个uber-jar，它包含所有的依赖jar包-->                <groupId>org.apache.maven.plugins</groupId>                <artifactId>maven-shade-plugin</artifactId>                <version>2.4.3</version>                <executions>                    <execution>                        <phase>package</phase>                        <goals>                            <goal>shade</goal>                        </goals>                        <configuration>                            <transformers>                                <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">                                    <resource>reference.conf</resource>                                </transformer>                            </transformers>                            <filters>                                <filter>                                    <artifact>*:*</artifact>                                    <excludes>                                        <exclude>META-INF/*.SF</exclude>                                        <exclude>META-INF/*.DSA</exclude>                                        <exclude>META-INF/*.RSA</exclude>                                    </excludes>                                </filter>                            </filters>                            <shadedArtifactAttached>true</shadedArtifactAttached>                            <shadedClassifierName>jar-with-dependencies</shadedClassifierName>                        </configuration>                    </execution>                </executions>            </plugin>        </plugins>    </build>    <dependencies>        <!-- https://mvnrepository.com/artifact/org.quartz-scheduler/quartz -->        <dependency>            <groupId>org.quartz-scheduler</groupId>            <artifactId>quartz</artifactId>            <version>2.2.1</version>        </dependency>        <dependency>            <groupId>com.floragunn</groupId>            <artifactId>search-guard-5</artifactId>            <version>5.2.2-15</version>        </dependency>        <!-- https://mvnrepository.com/artifact/com.floragunn/search-guard-ssl -->        <dependency>            <groupId>com.floragunn</groupId>            <artifactId>search-guard-ssl</artifactId>            <version>5.2.2-22</version>        </dependency>        <!-- https://mvnrepository.com/artifact/log4j/log4j -->        <dependency>            <groupId>org.elasticsearch.client</groupId>            <artifactId>transport</artifactId>            <version>5.2.2</version>        </dependency>        <dependency>            <groupId>org.apache.logging.log4j</groupId>            <artifactId>log4j-api</artifactId>            <version>2.7</version>        </dependency>        <dependency>            <groupId>org.apache.logging.log4j</groupId>            <artifactId>log4j-core</artifactId>            <version>2.7</version>        </dependency>        <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->        <dependency>            <groupId>org.slf4j</groupId>            <artifactId>slf4j-log4j12</artifactId>            <version>1.6.6</version>        </dependency>    </dependencies></project>

2、调用方法：

使用mvn进行打包部署后，进入target目录，输入nohup java -cp /root/remoa/indexTest/target/indexTest-1.0-SNAPSHOT-jar-with-dependencies.jar com.remoa.quartzCli.QuartzCli &将其设置为后台进程即可。