脚本index.sh——控制频率往es写数据

脚本技术要点：

1、切割文件

2、http接口写es

#!/bin/bash
es_server=127.0.0.1

#for i in `ls esCrash.data*`
for i in `ls *data* |grep -v "\-D\-"`
do 
    line_num=$(wc -l $i |awk '{print $1}')


    if [ $line_num -eq 0 ];then continue;fi


    if [ $line_num -le 5000 ];then
       cp $i ${i}-D-1
       continue
    else
        begin=1
        end=5000
        count=`expr $line_num / 5000 + 1`
        for j in `seq 1 $count`
        do
            if sed -n "${begin},${end}p" ${i} |tail -n 1 |grep -q _index;then
                end=`expr ${end} - 1`
            fi
            sed -n  "${begin},${end}p" $i >${i}-D-$j
            begin=`expr $end + 1`
            end=`expr 5000 \* $j + 5000`
        done
    fi
done

for i in `ls *-D-* |grep -v 'index.sh'`
do
    #echo >>${i}
    date
    echo "write $i to es $es_server ......"
    curl -s -XPOST ${es_server}:9200/_bulk --data-binary "@$i";
    echo
    sleep 3
done

rm -rf *data*