java 操作 elasticsearch 杂记

elasticsearch  是基于 lucene 的搜索引擎,我们在项目中 当做数据库使用,具有 查询快等 优点,下面是我在 项目 中 用到的 方法,以防忘记!

service 层  封装 QueryBuilder/FilterBuilder 语句,相当于 sql

代码如下:

package com.venustech.tsoc.cupid.botnet.service;import java.util.Collections;import java.util.Comparator;import java.util.HashMap;import java.util.LinkedList;import java.util.Map;import org.elasticsearch.index.query.QueryBuilder;import org.elasticsearch.index.query.QueryBuilders;import com.venustech.tsoc.cupid.commons.mvc.Pager;import org.nutz.ioc.loader.annotation.IocBean;import com.venustech.tsoc.cupid.botnet.util.EsUtiOfBotnet;import com.venustech.tsoc.cupid.botnet.util.SortUtil;import com.venustech.tsoc.cupid.security.bean.BotNet;import com.venustech.tsoc.cupid.customer.bean.Botnet;;@IocBeanpublic class BotnetEsService {/** *  获取 首页的江苏Ck 个数 * @return */public Map<String,Object> getStatisticNum(){EsUtiOfBotnetesUti= EsUtiOfBotnet.getInstance();QueryBuilder queryBuilder1=QueryBuilders.wildcardQuery("bigArea", "*电信");QueryBuilder queryBuilderCkType=QueryBuilders.termQuery("type", "3");QueryBuilder queryBuilderCCType=QueryBuilders.termQuery("type", "1");QueryBuilder queryBuilderUrlType=QueryBuilders.termQuery("type", "2");QueryBuilder queryBuilderCC=QueryBuilders.boolQuery().must(queryBuilder1).must(queryBuilderCCType);QueryBuilder queryBuilderCk=QueryBuilders.boolQuery().must(queryBuilder1).must(queryBuilderCkType);QueryBuilder queryBuilderUrl=QueryBuilders.boolQuery().must(queryBuilderUrlType);longnumberCC=esUti.getNumber(queryBuilderCC);longnumberCk=esUti.getNumber(queryBuilderCk);longnumberUrl=esUti.getNumber(queryBuilderUrl);HashMap<String,Object> hashMap = new HashMap<String,Object>();hashMap.put("cc", numberCC);hashMap.put("ck", numberCk);hashMap.put("url", numberUrl);return hashMap;}/** *  * @param isBig  是否显示省份还是省份之下 * @param isMap  是不是地图数据 还是 饼形图数据 * @param bigArea  所要查询的地区 * @param state   状态 1：疑似  2：确认 * @param type   1：CC   3：肉鸡  * @return * 地图和饼形图都要 排序 */public LinkedList<BotNet>  getChartAndMap(boolean isBig,boolean isMap,String bigArea,String state,String type,String ccIp){EsUtiOfBotnetesUti= EsUtiOfBotnet.getInstance();SortUtil sortUtil = SortUtil.getInstance();LinkedList<BotNet> Data = new LinkedList<BotNet>();if(bigArea==null) return new LinkedList<BotNet>();String index="botnet";//地图 ： 小地图的 情况就有一种，就是首页展示的右侧江苏地图，大地图的,其他的都是 大地图if(isMap){LinkedList<BotNet> mapData=new LinkedList<BotNet>();if(isBig){// 僵木蠕模块 大地图  疑似与 确认全取QueryBuilder queryBuilderChina=QueryBuilders.wildcardQuery("bigArea", "*电信");QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);Data = esUti.getMapData(QueryBuilders.boolQuery().must(queryBuilderChina).must(queryBuilderType), "bigArea",false);}else{// 首页地图QueryBuilder queryBuilderChina=QueryBuilders.termQuery("bigArea", "江苏电信");QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type); mapData = esUti.getMapOrChartsData(index,QueryBuilders.boolQuery().must(queryBuilderChina).must(queryBuilderType), "smallArea"); Data=sortUtil.sort2(sortUtil.getJiangSuCityInfo(mapData));}}// 饼形图： else{LinkedList<BotNet> chartsData=new LinkedList<BotNet>();if(isBig){// 省城，电信，运营商，其他。。。if(bigArea.equals("运营商")){QueryBuilder queryBuilderArea=QueryBuilders.termQuery("bigArea", null);QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);QueryBuilder queryBuilderState=QueryBuilders.termQuery("state", state);if(ccIp!=null && !ccIp.equals("")){QueryBuilder queryBuilderCCip=QueryBuilders.termQuery("ccIp", ccIp);chartsData=esUti.getMapOrChartsData("botnetinfo",QueryBuilders.boolQuery().mustNot(queryBuilderArea).must(queryBuilderCCip).must(queryBuilderState), "bigArea");}else{chartsData=esUti.getMapOrChartsData(index,QueryBuilders.boolQuery().mustNot(queryBuilderArea).must(queryBuilderState).must(queryBuilderType), "bigArea");}}else{QueryBuilder queryBuilderArea=QueryBuilders.wildcardQuery("bigArea", "*"+bigArea+"*");QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);QueryBuilder queryBuilderState=QueryBuilders.termQuery("state", state);if(ccIp!=null && !ccIp.equals("")){QueryBuilder queryBuilderCCip=QueryBuilders.termQuery("ccIp", ccIp);chartsData=esUti.getMapOrChartsData("botnetinfo",QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderCCip).must(queryBuilderState), "bigArea");}else{chartsData=esUti.getMapOrChartsData(index,QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderState).must(queryBuilderType), "bigArea");}}}else{// 小地区QueryBuilder queryBuilderArea=QueryBuilders.termQuery("bigArea", bigArea);QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);QueryBuilder queryBuilderState=QueryBuilders.termQuery("state", state);if(ccIp!=null && !ccIp.equals("")){QueryBuilder queryBuilderCCip=QueryBuilders.termQuery("ccIp", ccIp);chartsData=esUti.getMapOrChartsData("botnetinfo",QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderCCip).must(queryBuilderState), "area");}else{chartsData=esUti.getMapOrChartsData(index,QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderState).must(queryBuilderType), "smallArea");}}Data=sortUtil.sort(chartsData); // 降序 排列     Collections.sort( Data, new Comparator<BotNet>(){//list     @Override     public int compare(BotNet b1, BotNet b2) {     if(b1.getNu()>b2.getNu())return -1;     else if(b1.getNu()<b2.getNu())return 1;     else return 0;     }     });}return Data;}/** * 根据区域bigArea(全国电信,江苏电信,重庆电信等来显示肉鸡的地理分布),仅限定 电信 * 不区分确认与疑似,当为全国时 显示 bigArea ,当时某个省份时,则显示 二级地理信息  即为small,此方法可 * 继续扩展 * @param Area   * type : 1-CC ;3-Ck * @return */public LinkedList<BotNet>  getMapDataByArea(String area,String type){EsUtiOfBotnetesUti= EsUtiOfBotnet.getInstance();SortUtil sortUtil = SortUtil.getInstance();LinkedList<BotNet> Data = new LinkedList<BotNet>();if(area==null) return new LinkedList<BotNet>();String field="smallArea";QueryBuilder queryBuilderArea=null;if(area.equals("全国电信")){field="bigArea";// 僵木蠕模块 大地图  疑似与 确认全取 queryBuilderArea=QueryBuilders.wildcardQuery("bigArea", "*电信");}else{ queryBuilderArea=QueryBuilders.termQuery("bigArea", area);}QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);Data = esUti.getMapData(QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderType), field,true);// 降序 排列 Collections.sort( Data, new Comparator<BotNet>(){//list @Override public int compare(BotNet b1, BotNet b2) { if(b1.getNu()>b2.getNu())return -1; else if(b1.getNu()<b2.getNu())return 1; else return 0; } });return Data;}/** * 分页查询 * @param queryBuilder * @param pager * @return */public Pager<Botnet> paging(QueryBuilder queryBuilder,Pager<Botnet> pager){EsUtiOfBotnetesUti= EsUtiOfBotnet.getInstance();return esUti.getList(queryBuilder, pager);}/** * 增加僵木蠕到es */public void addBotnet(Botnet bot){EsUtiOfBotnet.getInstance().insertBotnetInfo(bot);}/** * 删除 僵木蠕方法  * @param long[] lids */public void deleteBotnet(long[] lids,long type){EsUtiOfBotnet.getInstance().deleteInfoByLid(lids,type,"true");}/** * 由 key  获取 botnet 信息，返回一个 * @param lid * @return */public Botnet find(String key,String value){return EsUtiOfBotnet.getInstance().getBotnetInfoById(key,value);}/** * 检验是否存在 * @param args */public boolean checkInfo(String key,String value){boolean flag=true;Botnet botnet = EsUtiOfBotnet.getInstance().getBotnetInfoById(key,value);if(botnet!=null){flag=false;}return flag;}/** * 更新数据 * @param bot */public void updateBotnet(Botnet bot){EsUtiOfBotnet.getInstance().editBotnetBykey("lid", bot.getLid()+"", bot);}/** * 同步更新botnetinfo数据(肉鸡且状态改变) * @param bot */public void updateBotnetInfo(Botnet bot){if(bot.getType()==3){EsUtiOfBotnet.getInstance().updateBotnetMaping(bot.getIp(), bot);}}public static void main(String[] args) {BotnetEsService botnetEsService = new BotnetEsService();//botnetEsService.getStatisticNum();}}

具体执行类 如下:

package com.venustech.tsoc.cupid.botnet.util;import java.text.SimpleDateFormat;import java.util.ArrayList;import java.util.Date;import java.util.HashMap;import java.util.Iterator;import java.util.LinkedList;import java.util.List;import java.util.Map;import java.util.Random;import org.apache.log4j.Logger;import org.elasticsearch.ElasticsearchException;import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest;import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;import org.elasticsearch.action.bulk.BulkRequestBuilder;import org.elasticsearch.action.delete.DeleteRequest;import org.elasticsearch.action.search.SearchRequestBuilder;import org.elasticsearch.action.search.SearchResponse;import org.elasticsearch.action.search.SearchType;import org.elasticsearch.client.transport.TransportClient;import org.elasticsearch.common.settings.ImmutableSettings;import org.elasticsearch.common.settings.Settings;import org.elasticsearch.common.transport.InetSocketTransportAddress;import org.elasticsearch.common.unit.TimeValue;import org.elasticsearch.index.query.BoolQueryBuilder;import org.elasticsearch.index.query.FilterBuilder;import org.elasticsearch.index.query.FilterBuilders;import org.elasticsearch.index.query.QueryBuilder;import org.elasticsearch.index.query.QueryBuilders;import org.elasticsearch.search.SearchHit;import org.elasticsearch.search.SearchHits;import org.elasticsearch.search.aggregations.Aggregation;import org.elasticsearch.search.aggregations.AggregationBuilder;import org.elasticsearch.search.aggregations.AggregationBuilders;import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;import org.elasticsearch.search.sort.SortOrder;import com.venustech.tsoc.cupid.botnet.analysis.cache.BotnetCache;import com.venustech.tsoc.cupid.botnet.analysis.cache.IpLibraryCache;import com.venustech.tsoc.cupid.commons.mvc.Pager;import com.venustech.tsoc.cupid.customer.bean.Botnet;import com.venustech.tsoc.cupid.customer.bean.ccBotnetInfo;import com.venustech.tsoc.cupid.knowledgebase.botnet.data.BotnetBean;import com.venustech.tsoc.cupid.origins.port.IspInfo.cache.ISPCache;import com.venustech.tsoc.cupid.security.bean.BotNet;import com.venustech.tsoc.cupid.util.es.ESClient;import com.venustech.tsoc.cupid.util.es.ESClient.Type;import com.venustech.tsoc.cupid.util.es.ESClientFactory;/** * es 上面的 知识库 操作工具类 * @author me * botnet */public class EsUtiOfBotnet { private Logger logger = Logger.getLogger(this.getClass()); private EsUtiOfBotnet(){}private static EsUtiOfBotnet esUtil=null;public static EsUtiOfBotnet getInstance(){if (esUtil==null){esUtil=new EsUtiOfBotnet();}return esUtil;}public static  HashMap<String,Boolean> checkCCIPMap=new HashMap<String,Boolean>();/** * 是否为调试 */private static boolean Debug = false;// 获取 client//ESClient cupidClient = ESClientFactory.me().getDefaultDelegateClient();private static ESClient buildClient(){Settings settings = ImmutableSettings.settingsBuilder()//cupid-es-zhangli.put("cluster.name","cupid-cdn-index")//cupid-cdn-index->  132; cupid-es ->90.put("client.transport.sniff", true).build();@SuppressWarnings("resource")// 192.168.56.49TransportClient client1 = new TransportClient(settings)//192.168.19.98  192.168.19.90 132.224.196.232.addTransportAddress(new InetSocketTransportAddress("192.168.56.49", 8300)); ESClient client = new ESClient(client1, Type.cdn);return client;}//cupidClient = ESClientFactory.me().getDefaultDelegateClient();// 获取 首页江苏肉鸡的 个数public  ESClient getClient(){ESClient cupidClient = null;try {cupidClient = ESClientFactory.me().getDefaultDelegateClient();} catch (Exception e) {}return (!Debug)?cupidClient:buildClient();}/** * 首页 展示的 江苏 肉鸡与CC 个数 * @param queryBuilder * @return */public long getNumber(QueryBuilder queryBuilder){long number=0;ESClient client = this.getClient();/*Settings settings = ImmutableSettings.settingsBuilder()//cupid-es-zhangli.put("cluster.name","cupid-cdn-index").put("client.transport.sniff", true).build();@SuppressWarnings("resource")TransportClient client1 = new TransportClient(settings)//192.168.19.98.addTransportAddress(new InetSocketTransportAddress("132.224.196.232", 8300)); ESClient client = new ESClient(client1, Type.cdn);*/boolean existsIndex = isExistsIndex("botnet");if(existsIndex){number= client.prepareCount("botnet")//索引//.setTypes("botnet")// 类型.setQuery(queryBuilder)// ES 默认查询是10 条,想要改变的话  可以在此处 设置 size setSize(100).execute().actionGet().getCount();}return number;}/** * 僵木蠕模块 地图数据 * @param queryBuilder * @return */public LinkedList<BotNet> getMapOrChartsData(String index,QueryBuilder queryBuilder,String field){LinkedList<BotNet> mapData = new LinkedList<BotNet>();ESClient client = this.getClient();boolean existsIndex = isExistsIndex("botnet");if(existsIndex){//number= delegateClient.prepareCount("botnet")//索引 SearchRequestBuilder srb=client.prepareSearch(index);// srb.setTypes(index); srb.setSearchType(SearchType.COUNT);AggregationBuilder<?> aggregation = AggregationBuilders.terms("na").field(field).size(Integer.MAX_VALUE);srb.setQuery(queryBuilder);srb.addAggregation(aggregation);SearchResponse respose=srb.execute().actionGet();//System.out.println("result:"+respose);Map<String, Aggregation> asMap = respose.getAggregations().asMap();StringTerms  result =(StringTerms)asMap.get("na");Iterator<Bucket> iterator = result.getBuckets().iterator();while(iterator.hasNext()){ Bucket gradeBucket = iterator.next(); BotNet botNet = new BotNet(); String key = gradeBucket.getKey(); Long count = gradeBucket.getDocCount(); botNet.setNa(key); botNet.setNu(count.intValue()); mapData.add(botNet);}}return mapData;}/** * 分页查询 * @param queryBuilderm */public Pager<Botnet>  getList(QueryBuilder queryBuilder,Pager<Botnet> pager){ESClient client = this.getClient();List<Botnet> list = new ArrayList<Botnet>();long number=0;int pageFrom=(pager.getPage()-1)*pager.getPageSize();// 总页boolean existsIndex = isExistsIndex("botnet");if(existsIndex){// 数据SearchResponse respose = client.prepareSearch("botnet")//索引//.setTypes("botnet")// 类型.setQuery(queryBuilder)// ES 默认查询是10 条,想要改变的话  可以在此处 设置 size setSize(100).addSort("createTime", SortOrder.DESC)// 将序.setFrom(pageFrom).setSize(pager.getPageSize()).execute().actionGet();SearchHits hits = respose.getHits();// 命中数number=hits.getTotalHits();SearchHit[] searchHits = hits.getHits();if(searchHits.length>0){for (SearchHit searchHit : searchHits) {Map<String, Object> source = searchHit.getSource();Botnet bo = new Botnet();long lid=(long)source.get("lid");long type=Long.parseLong(source.get("type").toString());long state=Long.parseLong(source.get("state").toString());long source1=Long.parseLong(source.get("source").toString());long create=Long.parseLong(source.get("createTime").toString());long update=Long.parseLong(source.get("updateTime").toString());long port =Long.parseLong(source.get("port").toString());if(update==0){update=create;}String ip=source.get("ip")==null ?"":source.get("ip").toString();String url=source.get("url")==null ?"":source.get("url").toString();String note=source.get("note")==null ?"":source.get("note").toString();String createTime=getStrTime(create);String updateTime=getStrTime(update);bo.setIp(ip);bo.setUrl(url);bo.setLid(lid);bo.setType(type);bo.setSource(source1);bo.setNote(note);bo.setState(state);bo.setAction(createTime);// 创建时间bo.setProt(updateTime);// 更新时间bo.setPort(port);list.add(bo);}}}pager.setRecords(number);pager.setData(list);return pager;}/** * 初始化 botnet  到缓存 */public void getAllBotnetbyType(String type1,int size){//BotnetCache.botnets.put(key, value)//QueryBuilders.logger.info("初始化 --- botnet-"+type1+"-加载到缓存开始");long start=System.currentTimeMillis();ESClient client = this.getClient();boolean existsIndex = isExistsIndex("botnet");if(existsIndex){// 数据QueryBuilder queryBuilder=null;QueryBuilder queryBuilder1=QueryBuilders.boolQuery().must(QueryBuilders.termQuery("type", type1));if(type1.equals("3")){//QueryBuilder queryBuilder2 = QueryBuilders.rangeQuery("createTime").from(System.currentTimeMillis()-(60*1000*60*24*5)).to(System.currentTimeMillis());queryBuilder = QueryBuilders.boolQuery().must(queryBuilder1);//.must(queryBuilder2)}else{queryBuilder = QueryBuilders.boolQuery().must(queryBuilder1);}SearchResponse respose = client.prepareSearch("botnet")//索引.setTypes("botnet")// 类型.setQuery(queryBuilder).setSize(size).execute().actionGet();SearchHits hits = respose.getHits();// 命中数logger.info("botnet-type-"+type1+" -个数："+hits.getTotalHits());SearchHit[] searchHits = hits.getHits();if(searchHits.length>0){for (SearchHit searchHit : searchHits) {Map<String, Object> source = searchHit.getSource();BotnetBean bo = new BotnetBean();long type=Long.parseLong(source.get("type").toString());long state=Long.parseLong(source.get("state").toString());String ip=source.get("ip")==null ?"":source.get("ip").toString();String url=source.get("url")==null ?"":source.get("url").toString();bo.setIp(ip);bo.setUrl(url);bo.setType(Integer.parseInt(type+""));bo.setState(Integer.parseInt(state+""));// 加入 初始化map// urlif(type1.equals("2")){BotnetCache.botnets.put(url, bo);}// cc  + ckelse{BotnetCache.botnets.put(ip, bo);}}} logger.info("botnet 加载进条数 ："+(searchHits.length) +"");logger.info("botnet 加载进 缓存结束，共耗时："+(System.currentTimeMillis()-start) +"毫秒");}   }/** * 初始化 botnet ck -- 到缓存  每次 24000 */public void getAllBotnetCkData(int size){logger.info("开始加载肉鸡到内存----start botnet--2");int allSize=0; long start=System.currentTimeMillis();ESClient client = this.getClient();//boolean existsIndex = isExistsIndex("botnet");if(existsIndex){SearchResponse repose = client.prepareSearch("botnet")//.setTypes("botnet").setQuery(QueryBuilders.termQuery("type", 3)).setScroll(new TimeValue(600000)).setSize(2000).setSearchType(SearchType.SCAN).execute().actionGet();String scrollId = repose.getScrollId();try {while(true){long start2=System.currentTimeMillis();SearchResponse repose2 = client.prepareSearchScroll(scrollId).setScroll(new TimeValue(size*1000)).execute().actionGet();SearchHits hits = repose2.getHits();if(hits.getHits().length==0){break;}SearchHit[] hits2 = hits.getHits();allSize+=hits2.length;//System.out.println("size:"+hits2.length);for (SearchHit searchHit : hits2) {Map<String, Object> source = searchHit.getSource();BotnetBean bo = new BotnetBean();long type=Long.parseLong(source.get("type").toString());long state=Long.parseLong(source.get("state").toString());String ip=source.get("ip")==null ?"":source.get("ip").toString();String url=source.get("url")==null ?"":source.get("url").toString();bo.setIp(ip);bo.setUrl(url);bo.setType(Integer.parseInt(type+""));bo.setState(Integer.parseInt(state+""));BotnetCache.botnets.put(ip, bo);}//System.out.println("耗时："+(System.currentTimeMillis()-start2));}logger.info("botnet-肉鸡 导入缓存 结束..... 共耗时："+(System.currentTimeMillis()-start) +"毫秒 ，总条数" +allSize +"条");BotnetCache.overFlag=true;} catch (NumberFormatException e) {// TODO Auto-generated catch block//e.printStackTrace();} catch (ElasticsearchException e) {// TODO Auto-generated catch block//e.printStackTrace();}}else{/* * 若是此时,botnet 不存在,则设置 默认读取完成 * */BotnetCache.overFlag=true;}}/** * 验证 ip/url 是否在 botnet 中  true ： 含有 * @param key * @param value * @return */public boolean checkBotnetkey_in(String key,String value,String type,String stateFlag){logger.info(" --- botnet_检索开始 判断库中是否含有此Key:"+value);long start=System.currentTimeMillis();ESClient client = this.getClient();//long number = 1;boolean existsIndex = isExistsIndex("botnet");if(existsIndex){BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();queryBuilder.must(QueryBuilders.termQuery("type", type)).must(QueryBuilders.termQuery(key, value));if(stateFlag!=null && !stateFlag.equals("")){long state=stateFlag.equals("true")?2:1;queryBuilder.must(QueryBuilders.termQuery("state", state));}SearchResponse repose = client.prepareSearch("botnet")//索引//.setTypes("botnet")// 类型.setQuery(queryBuilder)// ES 默认查询是10 条,想要改变的话  可以在此处 设置 size setSize(100).setSize(1).execute().actionGet(); number=repose.getHits().getTotalHits();if(number>0 && type.equals("3")){int state=0;if(!stateFlag.equals("")){state=stateFlag.equals("true")?2:1;}BotnetCache.botnets.put(value, new BotnetBean(value,"",state,3));}logger.info("botnet 检索结束，共耗时："+(System.currentTimeMillis()-start) +"毫秒");}return number>0 ? true:false;}/** * 僵木蠕模块 大地图数据 * @param queryBuilder * @param isAll  true:合并 疑似与确认   : false : 分开聚合 * @return */public LinkedList<BotNet> getMapData(QueryBuilder queryBuilder,String field,boolean isAll){LinkedList<BotNet> mapData = new LinkedList<BotNet>();ESClient client = this.getClient();boolean existsIndex = isExistsIndex("botnet");if(existsIndex){SearchRequestBuilder srb=client.prepareSearch("botnet");//srb.setTypes("botnet");srb.setSearchType(SearchType.COUNT);srb.setQuery(queryBuilder);AggregationBuilder<?> aggregationField = AggregationBuilders.terms("na").field(field).size(Integer.MAX_VALUE);if(isAll==false){AggregationBuilder<?> aggregationState = AggregationBuilders.terms("state").field("state").size(Integer.MAX_VALUE);aggregationField.subAggregation(aggregationState);}srb.addAggregation(aggregationField);SearchResponse respose=srb.execute().actionGet();Map<String, Aggregation> asMap = respose.getAggregations().asMap();StringTerms  result =(StringTerms)asMap.get("na");Iterator<Bucket> iterator = result.getBuckets().iterator();// 合并疑似与确认if(isAll){while(iterator.hasNext()){BotNet botNet = new BotNet();Bucket fieldBucket = iterator.next();String key = fieldBucket.getKey();Long botnetNum=fieldBucket.getDocCount();botNet.setNa(key);//区域botNet.setNu(botnetNum.intValue());mapData.add(botNet);}}// 分开聚合else{while(iterator.hasNext()){Bucket gradeBucket = iterator.next();String key = gradeBucket.getKey();LongTerms  result2 =(LongTerms)gradeBucket.getAggregations().asMap().get("state");Iterator<Bucket> iterator2 = result2.getBuckets().iterator();while(iterator2.hasNext()){Bucket next = iterator2.next();BotNet botNet = new BotNet();int keyState = Integer.parseInt(next.getKey()) ;Long count = next.getDocCount();botNet.setState(keyState);botNet.setNa(key);botNet.setNu(count.intValue());mapData.add(botNet);}}}}return mapData;}/** * 增加一个 僵木蠕信息 * @param hitsArray * @return */public void insertBotnetInfo(Botnet  bo){//boolean existsIndex = isExistsIndex("botnet");//if(existsIndex){ESClient client = this.getClient();if(client != null){long lid=new Date().getTime()*10000+new Random().nextInt(10000);bo.setLid(lid);bo.setIsLive(1);String  jsonData=SortUtil.toJson(bo);client.prepareIndex("botnet", "botnet").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData).setId(lid+"").execute().actionGet();}//}}/** * 批量 添加 到 es  botnet * @param botnets */public void insertBotnets(List<BotnetBean> botnets){//boolean existsIndex = isExistsIndex("botnet");//if(existsIndex){if(botnets ==null || botnets.size()==0) return;// 类型转化List<Botnet> botnet_esList = convertTypeList(botnets);//开启批量插入ESClient client = this.getClient();if(client != null){BulkRequestBuilder bulkRequest = client.prepareBulk();for (Botnet bo: botnet_esList) {String  jsonData=SortUtil.toJson(bo);bulkRequest.add(client.prepareIndex("botnet", "botnet").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData));}bulkRequest.execute().actionGet();logger.info("批量插入(botnet)数据："+botnet_esList.size());}//}}/** * 批量 添加 到 es  botnetinfo 关联表, * 首次添加 还没有 ccIp * @param botnets * @param isQuartzDelete  是否为定时清理调度任务后的 insert 到 bontnet */public void insertBotnetInfos(List<BotnetBean> botnets, boolean isQuartzDelete ){//boolean existsIndex = isExistsIndex("botnetinfo");//if(existsIndex){if(botnets ==null || botnets.size()==0) return;// 类型转化List<ccBotnetInfo>botnet_esList = null;if(isQuartzDelete==false){botnet_esList=convertTypeList2(botnets,"frist");}else{botnet_esList=convertTypeList3(botnets,"frist");}//开启批量插入ESClient client = this.getClient();if(client != null){BulkRequestBuilder bulkRequest = client.prepareBulk();for (ccBotnetInfo bo: botnet_esList) {checkCCIPMap.put(bo.getCkIp(), true);String  jsonData=SortUtil.toJson(bo);bulkRequest.add(client.prepareIndex("botnetinfo", "botnetinfo").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData));}bulkRequest.execute().actionGet();logger.info("批量插入(botnetinfo-ck)数据："+botnet_esList.size());}//}}/** * update * @param botnets */public void updateBotnetInfos(List<ccBotnetInfo> botnets){boolean existsIndex = isExistsIndex("botnetinfo");if(existsIndex){if(botnets ==null || botnets.size()==0) return;//开启批量插入ESClient client = this.getClient();BulkRequestBuilder bulkRequest = client.prepareBulk();for (ccBotnetInfo bo: botnets) {String  jsonData=SortUtil.toJson(bo);bulkRequest.add(client.prepareIndex("botnetinfo", "botnetinfo").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData));}bulkRequest.execute().actionGet();logger.info("首次批量update(botnetinfo-ck)数据："+botnets.size());}}/** * 验证关联表(botnetinfo)中 是否 含义ckIp +ccIp * @return */public boolean  isContains(String ckIp,String ccIp){long number = 1;boolean existsIndex = isExistsIndex("botnetinfo");if(existsIndex){ESClient client = this.getClient();BoolQueryBuilder query = QueryBuilders.boolQuery();QueryBuilder queryBuilderCkIp=QueryBuilders.termQuery("ckIp", ckIp);if(!ccIp.equals("afterDeleteccIp")){QueryBuilder queryBuilderCcIp=QueryBuilders.termQuery("ccIp", ccIp);//QueryBuilder queryBuilderPort=QueryBuilders.termQuery("port", port);query.must(queryBuilderCkIp).must(queryBuilderCcIp);}else{query.must(queryBuilderCkIp);} number = client.prepareCount("botnetinfo")//.setTypes("botnetinfo").setQuery(query).execute().actionGet().getCount();}return number>0?true:false;}/** * 增加 映射关系  ， ccIp 储存在 note 属性中 */public void addMapping(List<BotnetBean> botnets){if(botnets ==null || botnets.size()==0) return;// 此时的 状态是 已经验证过 botnet ,含有此 ckIp ccIp// 所以此时 需要 看是否 要更新 或者 添加一条 映射 关系，// 先判断是否还有 空着 ccIp 的  ckIp//1 .有的话 先 更新 含有 ccIp 2. 没有空着的 则 新增加一条// 开启 批量添加// 转化 类型boolean existsIndex = isExistsIndex("botnetinfo");if(existsIndex){List<ccBotnetInfo> bot_info = convertTypeList2(botnets,"");ESClient client = this.getClient();BulkRequestBuilder bulkRequest = client.prepareBulk();for (ccBotnetInfo b : bot_info) {if(checkCCIPMap.get(b.getCkIp())!=null && checkCCIPMap.get(b.getCkIp())==true){long lid=ccIpisNull(b.getCkIp());// 更新if(lid!=1){// 先删除在 增加deleteBotnetInfoByLid(lid);checkCCIPMap.put(b.getCkIp(), false);}}// 添加String  jsonData=SortUtil.toJson(b);bulkRequest.add(client.prepareIndex("botnetinfo", "botnetinfo").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData));}if(bot_info.size()>0){bulkRequest.execute().actionGet();}logger.info("批量插入(botnetinfo--mapping)数据："+bot_info.size());}}/** * 先判断是否还有 空着 ccIp 的  ckIp * @return */public  long ccIpisNull(String ckIp){long lid=1;ESClient client = this.getClient();boolean existsIndex = isExistsIndex("botnetinfo");if(existsIndex){SearchResponse respose = client.prepareSearch("botnetinfo").setTypes("botnetinfo").setQuery(QueryBuilders.termQuery("ckIp", ckIp)).addSort("createTime", SortOrder.ASC).execute().actionGet();SearchHits hits = respose.getHits();SearchHit[] hits2 = hits.getHits();if(hits2.length>0){for (SearchHit searchHit : hits2) {Map<String, Object> source = searchHit.getSource();if(source.get("ccIp")==null || source.get("ccIp").equals("")){lid=(long)source.get("lid");break;}}}}return lid;}/** * 增加到 关联表 中  * @param bo */public ccBotnetInfo insertBotnet_info(Botnet  bo){long lid=new Date().getTime()*10000+new Random().nextInt(10000);ccBotnetInfo ccBotnetInfo = new ccBotnetInfo();ccBotnetInfo.setArea(bo.getSmallArea());//ccBotnetInfo.setCcIp(bo.get);ccBotnetInfo.setCkIp(bo.getIp());ccBotnetInfo.setLid(lid);//ccBotnetInfo.setccBotnetInfo.setCreateTime(System.currentTimeMillis());return ccBotnetInfo;}/** * 更新 或增加  * @param botnets */public void insertOrUpdateBotnetInfos(List<BotnetBean> botnets){}/** * 判断 botnetinfo 中  含有 ckIp 的 是否 */ /**    * 类型转换  过度    * @param bean    * @return    */   public List<Botnet> convertTypeList(List<BotnetBean> botnets){      ArrayList<Botnet> botnet_es = new ArrayList<Botnet>();   for (BotnetBean bean : botnets) {      Botnet botnet = new Botnet();   long lid=new Date().getTime()*10000+new Random().nextInt(10000);   botnet.setLid(lid);   botnet.setType(bean.getType());   botnet.setBigArea(bean.getBigArea());   botnet.setSmallArea(bean.getSmallArea());   botnet.setIp( bean.getIp());   botnet.setUrl(bean.getUrl());   botnet.setSource(bean.getSource());   botnet.setFirstCreateTime(System.currentTimeMillis());   botnet.setCreateTime(System.currentTimeMillis());   botnet.setState(bean.getState());   botnet.setNote(bean.getNote());   botnet.setIsLive(1);   botnet.setPort(bean.getPort());   botnet.setUpdateTime(System.currentTimeMillis());   if(botnet.getIp().equals("")){   logger.info("将要插入的数据-URL:"+bean.getUrl() );   }else{   logger.info("将要插入的数据为-IP:"+ bean.getIp());   }   logger.info(" 备注:"+bean.getNote());   botnet_es.add(botnet);}   return botnet_es;   }   /**    * 类型转换  过度   ccBotnetInfo 添加到 botnetinfo    *     * 里面 还没有 添加 ccIp    * @param bean    * @return    */   public List<ccBotnetInfo> convertTypeList2(List<BotnetBean> botnets,String flag){      ArrayList<ccBotnetInfo> botnet_es = new ArrayList<ccBotnetInfo>();   for (BotnetBean bean : botnets) {      ccBotnetInfo botnet = new ccBotnetInfo();   long lid=new Date().getTime()*10000+new Random().nextInt(10000);   botnet.setLid(lid);   botnet.setBigArea(bean.getBigArea());   botnet.setArea(bean.getSmallArea());   botnet.setCkIp( bean.getIp());   botnet.setCreateTime(System.currentTimeMillis());   if(!flag.equals("frist")){   botnet.setCcIp(bean.getNote());   }   botnet.setState(bean.getState());   botnet.setPort(bean.getPort());   botnet.setIsLive(1);   botnet_es.add(botnet);   }   return botnet_es;   }   /**    * 类型转换  过度   ccBotnetInfo 添加到 botnetinfo    *     * 里面 还没有 添加 ccIp    * @param bean    * @return    */   public List<ccBotnetInfo> convertTypeList3(List<BotnetBean> botnets,String flag){      ArrayList<ccBotnetInfo> botnet_es = new ArrayList<ccBotnetInfo>();   for (BotnetBean bean : botnets) {      ccBotnetInfo botnet = new ccBotnetInfo();   long lid=new Date().getTime()*10000+new Random().nextInt(10000);   botnet.setLid(lid);   botnet.setBigArea(bean.getBigArea());   botnet.setArea(bean.getSmallArea());   botnet.setCkIp( bean.getIp());   botnet.setCreateTime(bean.getCreateTime());   if(!flag.equals("frist")){   botnet.setCcIp(bean.getNote());   }   botnet.setState(bean.getState());   botnet.setPort(bean.getPort());   botnet.setIsLive(1);   botnet_es.add(botnet);   }   return botnet_es;   }      /** * 改变知识库信息 * @param  * @return */public void updateBotnetInfo(Botnet bo){boolean existsIndex = isExistsIndex("botnet");if(existsIndex){ESClient client = this.getClient();long[] lid=new long[1];lid[0]=bo.getLid();deleteInfoByLid(lid,bo.getType(),"false");bo.setUpdateTime(System.currentTimeMillis());String  jsonData=SortUtil.toJson(bo);client.prepareIndex("botnet", "botnet").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData).execute().actionGet();}//efge}/** *  * @param key  表示键  比如 lid  ip url等 具有唯一性的 * @param value  键的值 * @param botnet  要更新的信息储存在 其中 */public void editBotnetBykey(String key, String value, Botnet bean){ESClient client = this.getClient();boolean existsIndex = isExistsIndex("botnet");if(existsIndex){QueryBuilder queryBuilderKey=QueryBuilders.termQuery(key, value);QueryBuilder queryBuilder=QueryBuilders.boolQuery().must(queryBuilderKey);SearchResponse repose = client.prepareSearch("botnet").setTypes("botnet").setQuery(queryBuilder).setSize(Integer.MAX_VALUE).execute().actionGet();SearchHits hits = repose.getHits();SearchHit[] hits2 = hits.getHits();long oldState = bean.getReliability();if(hits2.length>0){// 获取文档 IDString docId = hits2[0].getId();//更新 updatebean.setUpdateTime(System.currentTimeMillis());bean.setReliability(0);String  jsonData=SortUtil.toJson(bean);// 追加到 es 上client.prepareIndex("botnet", "botnet").setId(docId).setSource(jsonData).execute().actionGet();}// 更新关联表的条件: 1: 为肉鸡ip ; 2: 状态值发生变化 state(oldState  放入了reliability )if(bean.getType() == 3 && bean.getState() != oldState){updateBotnetMaping(bean.getIp(), bean);}}}/** * 更新关联信息 * @param ip */public void updateBotnetMaping(String ckIp,Botnet bo){boolean existsIndex = isExistsIndex("botnetinfo");if(existsIndex){ESClient client = this.getClient();@SuppressWarnings("deprecation")QueryBuilder queryBuilder=QueryBuilders.termQuery("ckIp", ckIp);SearchResponse respose = client.prepareSearch("botnetinfo")//.setTypes("botnetinfo").setQuery(queryBuilder).setSize(Integer.MAX_VALUE).execute().actionGet();SearchHits hits = respose.getHits();SearchHit[] hits2 = hits.getHits();long state_new=bo.getState();if(hits2.length>0){for (SearchHit searchHit : hits2) {ccBotnetInfo mapping = new  ccBotnetInfo();Map<String, Object> source = searchHit.getSource();String docId = searchHit.getId();long lid=Long.parseLong(source.get("lid").toString());String ccIp=source.get("ccIp")==null ?"":source.get("ccIp").toString();//  String ck_Ip=bo.getIp();String note=source.get("note")==null ?"":source.get("note").toString();String countryCode=source.get("countryCode")==null ?"":source.get("countryCode").toString();long createTime=Long.parseLong(source.get("createTime").toString());long port=Long.parseLong(source.get("port").toString());mapping.setLid(lid);mapping.setBigArea(bo.getBigArea());mapping.setArea(bo.getSmallArea());mapping.setNote(note);mapping.setCreateTime(createTime);mapping.setIsLive(bo.getIsLive());mapping.setPort(port);mapping.setState(state_new);mapping.setCountryCode(countryCode);mapping.setCcIp(ccIp);mapping.setCkIp(bo.getIp());String  jsonData=SortUtil.toJson(mapping);// 追加到 es 上client.prepareIndex("botnetinfo", "botnetinfo").setId(docId).setSource(jsonData).execute().actionGet();}}}}/** * 删除 僵木蠕信息  long [lids] * @param lids * @param type * @param isDelete  true: 表示连 关联表 botnetinfo  中的 数据也删: false :表示只删除botnet,不删 botnetinfo */public void deleteInfoByLid(long[] lids,long type,String isDelete){if(lids.length<=0)  return;boolean existsIndex = isExistsIndex("botnet");if(existsIndex){ESClient client = this.getClient();@SuppressWarnings("deprecation")QueryBuilder queryBuilder=QueryBuilders.inQuery("lid", lids);SearchResponse respose = client.prepareSearch("botnet").setTypes("botnet").setQuery(queryBuilder).setSize(Integer.MAX_VALUE).execute().actionGet();SearchHits hits = respose.getHits();SearchHit[] hits2 = hits.getHits();int arrSize=Integer.parseInt(hits.getTotalHits()+"");// 储存要删除的ip  ckArrayList<String> cKipList = new ArrayList<String>();ArrayList<String> ccIpList = new ArrayList<String>();// ccboolean flag= type==3?true:false;BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);int i=0;for (SearchHit searchHit : hits2) {// 1. 删除 botnetDeleteRequest deleteRequest = new DeleteRequest("botnet","botnet",searchHit.getId());bulkRefresh.add(deleteRequest);Map<String, Object> obj = searchHit.getSource();// 2. 为删除botnetinfo 做出准备，记录ipsif(type!=2){String ip= obj.get("ip")==null ?"":obj.get("ip").toString();if(flag){cKipList.add(ip);}if(type==1){ccIpList.add(ip);}BotnetCache.botnets.remove(ip);}// 去除url 内存if(type==2){String url= obj.get("url")==null ?"":obj.get("url").toString();BotnetCache.botnets.remove(url); }i++;}bulkRefresh.execute().actionGet();/*for (String ip : ips) {System.out.println("要删除的ip:"+ip);// 去内存BotnetCache.botnets.remove(ip);}*/String[] array=new String[cKipList.size()];String[] ckips = cKipList.toArray(array);String[] array2=new String[ccIpList.size()];String[] ccips = ccIpList.toArray(array2);if(ckips.length>0 && flag && isDelete.equals("true")){deleteBotnetInfo("ckIp",ckips);}if(ccips.length>0){deleteBotnetInfo("ccIp",ccips);}}}/** * inQuery() size<=1024  filter  则是释放的 * String field  : ckIp /ccIp * @param ips */public void deleteBotnetInfo(String field,String[] ips){if(ips==null || ips.length<=0) return;ESClient client = this.getClient();boolean existsIndex = isExistsIndex("botnetinfo");if(existsIndex){//QueryBuilder queryBuilder=QueryBuilders.inQuery("ckIp", ips);@SuppressWarnings("deprecation")FilterBuilder inFilter = FilterBuilders.inFilter(field, ips);SearchResponse respose = client.prepareSearch("botnetinfo").setTypes("botnetinfo").setPostFilter(inFilter).setSize(Integer.MAX_VALUE).execute().actionGet();SearchHits hits = respose.getHits();SearchHit[] hits2 = hits.getHits();BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);// 储存要删除的ip  ckArrayList<String> cKipList = new ArrayList<String>();for (SearchHit searchHit : hits2) {// 2. 为删除botnetinfo 做出准备，记录ipsDeleteRequest deleteRequest = new DeleteRequest("botnetinfo","botnetinfo",searchHit.getId());bulkRefresh.add(deleteRequest);// 如果删除的是CC 应该判断 把Ck的全部删完，若是全部删完则因该空部位即ccip=""if(field.equals("ccIp")){// 记录要删除的 ckIpMap<String, Object> obj = searchHit.getSource();String ckIp= obj.get("ckIp")==null ?"":obj.get("ckIp").toString();if(ckIp!=null && !ckIp.equals("")){if(!cKipList.contains(ckIp)){cKipList.add(ckIp);}}}}if(hits2.length>0){bulkRefresh.execute().actionGet();}// 判断 info 中 ck--size String[] array=new String[cKipList.size()];String[] ckips = cKipList.toArray(array);if(ckips!=null && ckips.length >0){try {//Thread.sleep(3000);afterDeleteCC(ckips);} catch (Exception e) {// TODO Auto-generated catch blocke.printStackTrace();}//afterDeleteCC(ckips);}// inQuery() 里面若为 数组的话，是有限制的 大小为 1024，所以改为 filter}}/** * 判断botnetinfo 中 ckIp的size, */public void afterDeleteCC(String[] ckips){List<BotnetBean> BotnetInfos =new ArrayList<BotnetBean>();for (String ckIp : ckips) {Botnet oldInfo = getBotnetInfoById("ip",ckIp);// 检验botnet  根还存在if( oldInfo!=null){boolean flag = isContains(ckIp,"afterDeleteccIp");//检验botnetinfo if(flag==true ){// 表明 ckip  还存在，不用补位}else{// 现在 需要补位  List<BotnetBean> botnetsBotnetBean bin =new BotnetBean();bin.setIp(ckIp);// 同事设定了  ipLong/*if (bin.getIpLong() > 0) {bin.setArea(ISPCache.getInstance().getArea(bin.getIpLong()));}*/bin.setBigArea(oldInfo.getBigArea());bin.setSmallArea(oldInfo.getSmallArea());bin.setState(Integer.parseInt(oldInfo.getState()+""));// 设置 createTimebin.setCreateTime(oldInfo.getCreateTime());BotnetInfos.add(bin);}}}if(BotnetInfos.size()>0){insertBotnetInfos(BotnetInfos,true);}}public void deleteBotnetInfoByLids(long[] lids){if(lids==null || lids.length<=0) return;boolean existsIndex = isExistsIndex("botnetinfo");if(existsIndex){ESClient client = this.getClient();@SuppressWarnings("deprecation")QueryBuilder queryBuilder=QueryBuilders.inQuery("lid", lids);SearchResponse respose = client.prepareSearch("botnetinfo")//.setTypes("botnetinfo").setQuery(queryBuilder).setSize(Integer.MAX_VALUE).execute().actionGet();SearchHits hits = respose.getHits();SearchHit[] hits2 = hits.getHits();BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);for (SearchHit searchHit : hits2) {DeleteRequest deleteRequest = new DeleteRequest("botnetinfo","botnetinfo",searchHit.getId());bulkRefresh.add(deleteRequest);}if(hits2.length>0){bulkRefresh.execute().actionGet();}}}/** *  * @param lid */public void deleteBotnetInfoByLid(long lid){ESClient client = this.getClient();boolean existsIndex = isExistsIndex("botnetinfo");if(existsIndex){@SuppressWarnings("deprecation")QueryBuilder queryBuilder=QueryBuilders.termQuery("lid", lid);SearchResponse respose = client.prepareSearch("botnetinfo")//.setTypes("botnetinfo").setQuery(queryBuilder).setSize(Integer.MAX_VALUE).execute().actionGet();SearchHits hits = respose.getHits();SearchHit[] hits2 = hits.getHits();BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);for (SearchHit searchHit : hits2) {DeleteRequest deleteRequest = new DeleteRequest("botnetinfo","botnetinfo",searchHit.getId());bulkRefresh.add(deleteRequest);}bulkRefresh.execute().actionGet();}}/** * 由lid 获取  * @param lid * @return */public Botnet  getBotnetInfoById(String key,String value){Botnet botnet = null;ESClient client = this.getClient();QueryBuilder queryBuilder1=QueryBuilders.termQuery(key, value);//QueryBuilder queryBuilder=QueryBuilders.boolQuery().must(queryBuilder1);boolean existsIndex = isExistsIndex("botnet");if(existsIndex){SearchResponse respose = client.prepareSearch("botnet").setTypes("botnet").setQuery(queryBuilder1).execute().actionGet();SearchHits hits = respose.getHits();SearchHit[] hitsArr = hits.getHits();if(hitsArr.length>0){Map<String, Object> source = hitsArr[0].getSource();//System.out.println("id:"+hitsArr[0].getId());Botnet bo = new Botnet();long lid2=(long)source.get("lid");long type=Long.parseLong(source.get("type").toString());long state=Long.parseLong(source.get("state").toString());long source1=Long.parseLong(source.get("source").toString());long createTime=Long.parseLong(source.get("createTime").toString());long firstCreateTime=Long.parseLong(source.get("firstCreateTime").toString());long create=createTime;long update=Long.parseLong(source.get("updateTime").toString());if(update==0){update=create;}long port=Long.parseLong(source.get("port").toString());String ip=source.get("ip")==null?"":source.get("ip").toString();String url=source.get("url")==null?"":source.get("url").toString();String note=source.get("note")==null?"":source.get("note").toString();String bigArea=source.get("bigArea")==null?"":source.get("bigArea").toString();String smallArea=source.get("smallArea")==null?"":source.get("smallArea").toString();bo.setIp(ip);bo.setUrl(url);bo.setLid(lid2);bo.setType(type);bo.setSource(source1);bo.setPort(port);bo.setNote(note);bo.setBigArea(bigArea);bo.setSmallArea(smallArea);bo.setState(state);bo.setCreateTime(createTime);bo.setFirstCreateTime(firstCreateTime);bo.setAction(getStrTime(createTime));// 创建时间bo.setProt(getStrTime(update));// 更新时间botnet=bo;}}return botnet;}/** * 定时清理库中的 过期的 僵木蠕-肉鸡-信息 */public int deleteBotnetTimerTask(String field,long deleteInterval,String isOldData){ESClient client = this.getClient();int num = 0;boolean existsIndex = isExistsIndex("botnet");if(existsIndex){// 小于 正常期的 都是即将要删除掉的  包括 肉鸡 与 cc+URL//QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", "3");long nowTime=System.currentTimeMillis();long cycileTime=nowTime-deleteInterval;QueryBuilder queryBuilderTime=QueryBuilders.rangeQuery(field).lt(cycileTime);BoolQueryBuilder queryBuilder=QueryBuilders.boolQuery();queryBuilder.must(queryBuilderTime);// 老数据 中 updateTime 都是 0,所以 依 createTime 为准QueryBuilder queryBuilderCreateTime=QueryBuilders.termQuery("updateTime", "0");if(isOldData.equals("oldData")){queryBuilder.must(queryBuilderCreateTime);}else{queryBuilder.mustNot(queryBuilderCreateTime);}// 1.选出你要 删除的 条目 ，获取 ipSearchResponse repose = client.prepareSearch("botnet").setTypes("botnet").setQuery(queryBuilder).setSize(Integer.MAX_VALUE).execute().actionGet();SearchHits hits = repose.getHits();SearchHit[] hits2 = hits.getHits();// 开启 批量删除BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);// 储存要删除的ip  ckArrayList<String> ipList = new ArrayList<String>();ArrayList<String> ccIpList = new ArrayList<String>();// ccif(hits2.length>0){for (SearchHit searchHit : hits2) {DeleteRequest deleteRequest = new DeleteRequest("botnet", "botnet", searchHit.getId());bulkRefresh.add(deleteRequest);// get IPMap<String, Object> obj = searchHit.getSource();long type=obj.get("type")==null?0:Integer.parseInt(obj.get("type").toString());if(type==2){String url=obj.get("url")==null?"":obj.get("url").toString();BotnetCache.botnets.remove(url);}else{String ip=obj.get("ip")==null?"":obj.get("ip").toString();// 提高效率 去除 type=1 的情况if(type==3){ipList.add(ip);}if(type==1){ccIpList.add(ip);}// 清除 缓存 信息BotnetCache.botnets.remove(ip);}}// 删除 botnet 中的  不合理 信息bulkRefresh.execute().actionGet();}String[] array=new String[ipList.size()];String[] ips = ipList.toArray(array);String[] array2=new String[ccIpList.size()];String[] ccips = ccIpList.toArray(array2);// 同步清理  botnetinfo  ckIpif(ips.length>0){deleteBotnetInfo("ckIp",ips);}// ccIpif(ccips.length>0){deleteBotnetInfo("ccIp",ccips);}num = ips.length;}return num;}/** * 监测更新 ,update-->updateTime *  * key： ip/url */public void monitorUpdateInfo(String key,String value){ESClient client = this.getClient();// 小于 正常期的 都是即将要删除掉的boolean existsIndex = isExistsIndex("botnet");if(existsIndex){QueryBuilder queryBuilder=null;/*if(!key.equals("url")){queryBuilder=QueryBuilders.termQuery(key, value);}else{queryBuilder=QueryBuilders.queryString(value).field(key);}*/queryBuilder=QueryBuilders.termQuery(key, value);SearchResponse repose = client.prepareSearch("botnet")//.setTypes("botnet").setQuery(queryBuilder).setSize(Integer.MAX_VALUE).execute().actionGet();SearchHits hits = repose.getHits();SearchHit[] hits2 = hits.getHits();if(hits2.length>0){// 获取文档 IDString docId = hits2[0].getId();Map<String, Object> source = hits2[0].getSource();Botnet bo = new Botnet();long lid2=(long)source.get("lid");long type=Long.parseLong(source.get("type").toString());long ipLong=Long.parseLong(source.get("ipLong").toString());long port=Long.parseLong(source.get("port").toString());long isLive=Long.parseLong(source.get("isLive").toString());long reliability=Long.parseLong(source.get("reliability").toString());long state=Long.parseLong(source.get("state").toString());long source1=Long.parseLong(source.get("source").toString());long createTime=Long.parseLong(source.get("createTime").toString());long firstCreateTime=Long.parseLong(source.get("firstCreateTime").toString());String ip=source.get("ip")==null?"":source.get("ip").toString();String url=source.get("url")==null?"":source.get("url").toString();String note=source.get("note")==null?"":source.get("note").toString();String bigArea=source.get("bigArea")==null?"":source.get("bigArea").toString();String smallArea=source.get("smallArea")==null?"":source.get("smallArea").toString();bo.setIp(ip);bo.setUrl(url);bo.setLid(lid2);bo.setType(type);bo.setSource(source1);bo.setNote(note);bo.setBigArea(bigArea);bo.setState(state);bo.setCreateTime(createTime);bo.setFirstCreateTime(firstCreateTime);bo.setIsLive(isLive);bo.setSmallArea(smallArea);bo.setReliability(reliability);bo.setIpLong(ipLong);bo.setPort(port);//更新 updatebo.setUpdateTime(System.currentTimeMillis());String  jsonData=SortUtil.toJson(bo);// 追加到 es 上client.prepareIndex("botnet", "botnet").setId(docId).setSource(jsonData).execute().actionGet();}}}/** * update-->updateTime * key： ip */public void UpdateCCInfo(String key,String value,BotnetBean bean){ESClient client = this.getClient();// 小于 正常期的 都是即将要删除掉的boolean existsIndex = isExistsIndex("botnet");if(existsIndex){QueryBuilder queryBuilderKey=QueryBuilders.termQuery(key, value);QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", "1");QueryBuilder queryBuilder=QueryBuilders.boolQuery().must(queryBuilderType).must(queryBuilderKey);SearchResponse repose = client.prepareSearch("botnet").setTypes("botnet").setQuery(queryBuilder).setSize(Integer.MAX_VALUE).execute().actionGet();SearchHits hits = repose.getHits();SearchHit[] hits2 = hits.getHits();if(hits2.length>0){// 获取文档 IDString docId = hits2[0].getId();Map<String, Object> source = hits2[0].getSource();Botnet bo = new Botnet();long lid2=(long)source.get("lid");long type=Long.parseLong(source.get("type").toString());long ipLong=Long.parseLong(source.get("ipLong").toString());long port=Long.parseLong(source.get("port").toString());long isLive=Long.parseLong(source.get("isLive").toString());long reliability=Long.parseLong(source.get("reliability").toString());long state=bean.getState();long source1=bean.getSource();long createTime=Long.parseLong(source.get("createTime").toString());long firstCreateTime=Long.parseLong(source.get("firstCreateTime").toString());String ip=source.get("ip")==null?"":source.get("ip").toString();String url=source.get("url")==null?"":source.get("url").toString();String note=bean.getNote();String bigArea=source.get("bigArea")==null?"":source.get("bigArea").toString();String smallArea=source.get("smallArea")==null?"":source.get("smallArea").toString();bo.setIp(ip);bo.setUrl(url);bo.setLid(lid2);bo.setType(type);bo.setSource(source1);bo.setNote(note);bo.setBigArea(bigArea);bo.setState(state);bo.setCreateTime(createTime);bo.setFirstCreateTime(firstCreateTime);bo.setIsLive(isLive);bo.setSmallArea(smallArea);bo.setReliability(reliability);bo.setIpLong(ipLong);bo.setPort(port);//更新 updatebo.setUpdateTime(System.currentTimeMillis());String  jsonData=SortUtil.toJson(bo);// 追加到 es 上client.prepareIndex("botnet", "botnet").setId(docId).setSource(jsonData).execute().actionGet();}}}/** * 时间格式化 * @return */public String getStrTime(long time){SimpleDateFormat sdf= new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");    Date date = new Date(time);    String format = sdf.format(date);        return format;}/*public List<BotNet> getList(SearchHit[] hitsArray){List<BotNet> botList=new LinkedList<BotNet>();for (SearchHit searchHit : hitsArray) {BotNet botNet = new BotNet();Map<String, Object> source = searchHit.getSource();//botNet.setNa(source.get(""));}return botList;}*//**     * 判断指定的索引名是否存在     * @param indexName 索引名     * @return  存在：true; 不存在：false;     */    public boolean isExistsIndex(String indexName){    boolean isExist = false;     // 获取 client    ESClient client = this.getClient();    if(client != null){        try {IndicesExistsResponse  response = client.admin().indices().exists( new IndicesExistsRequest().indices(new String[]{indexName})).actionGet();isExist = response.isExists();} catch (ElasticsearchException e) {logger.info("ES 发生异常:"+e);e.printStackTrace();isExist = false;}    }        return isExist;}    /** * 判断指定的索引的类型是否存在 * @param indexName 索引名 * @param indexType 索引类型 * @return  存在：true; 不存在：false; */public boolean isExistsType(String indexName,String indexType){//ESClient client = this.getClient();    TypesExistsResponse  response =             getClient().admin().indices()            .typesExists(new TypesExistsRequest(new String[]{indexName}, indexType)            ).actionGet();    return response.isExists();}}