Edit

ELK

http://kibana.logstash.es/content/logstash/

ElasticSearch

python API

python ES
es = Elasticsearch(['http://esadmin:111111@192.168.1.61:9200'])

python bulk update

n = 0    while True:        actions = []        if n < num:            for js in jsfile:                n += 1                m += 1                continue        for js in jsfile:            m += 1            dic = json.loads(js)            id_cnki = dic['_id']['$oid']            id_wanfang = dic['repeatInfo']['id']            action = {                "_op_type":"update",  # 这里注明 update                "_index": "cnki02",                "_type": "doc",                "_id": id_cnki,                "doc": {'repeatInfo': {'db': 'wanfang',                                       'id': id_wanfang}} #doc表明 更新部分字段，没有doc，是替换            }                   if len(actions) == 500:                break        try:            helpers.bulk(es, actions)            m += 500        except:            logger.exception("Exception Logged")        if len(actions) == 0:            return

python scroll

# Initialize the scroll  page = es.search(  index = 'yourIndex',  doc_type = 'yourType',  scroll = '2m',  search_type = 'scan',  size = 1000,  body = {    # Your query's body    })  sid = page['_scroll_id']  scroll_size = page['hits']['total']  # Start scrolling  while (scroll_size > 0):    print "Scrolling..."    page = es.scroll(scroll_id = sid, scroll = '2m')    # Update the scroll ID    sid = page['_scroll_id']    # Get the number of results that we returned in the last scroll    scroll_size = len(page['hits']['hits'])    print "scroll size: " + str(scroll_size)    # Do something with the obtained page

批量导入

curl -s -XPOST localhost:9200/_bulk --data-binary "@requests"
curl -s -XPOST localhost:9200/_bulk --data-binary "@newbimt_0730_mod_cnki_test.json"

安装head

./bin/plugin install mobz/elasticsearch-head

安装分词插件ik（需要maven）

安装maven

wget http://mirrors.tuna.tsinghua.edu.cn/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
tar -zxf apache-maven-3.3.9-bin.tar.gz
mv apache-maven-3.3.9 /usr/local/
ln -s /usr/local/apache-maven-3.3.9 /usr/local/maven

配置环境变量

vim /etc/profile
export M2_HOME=/usr/local/maven
export PATH=M2_HOME/bin

使新配置的环境变量生效

source /etc/profile

安装分词插件ik

cd plugins
wget https://github.com/medcl/elasticsearch-analysis-ik/archive/v1.9.3.zip
unzip v1.9.3.zip
cd elasticsearch-analysis-ik-1.9.3/
mvn clean
mvn compile
mvn package
cd ..
mkdir ik
cp elasticsearch-analysis-ik-1.9.3/target/releases/elasticsearch-analysis-ik-1.9.3.zip ik/
cd ik
unzip elasticsearch-analysis-ik-1.9.3.zip

安装shield插件

安装许可授权

./bin/plugin install elasticsearch/license/latest

安装shield

./bin/plugin install elasticsearch/shield/latest
/etc/init.d/elasticsearch restart
./bin/shield/esusers useradd es_admin -r admin