Elastic Stack 5.1 部署

Elastic Stack安装步骤

• 1、Elasticsearch
  X-Pack for Elasticsearch
• 2、Kibana
  X-Pack for Kibana
• 3、Logstash
• 4、Beats
• 5、Elasticsearch Hadoop
• 6、redis

参考文档

https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html

1、安装Elasticsearch & X-pack（yum安装）

X-Pack是一个Elastic Stack的扩展，将安全，警报，监视，报告和图形功能包含在一个易于安装的软件包中。x-Pack安装之后有一个超级用户elastic ，其默认的密码是changeme，拥有对所有索引和数据的控制权，可以使用该用户创建和修改其他用户。通过kibana的web界面进行用户和用户组的管理，也可以使用shell 终端进行管理。

[root@tzgsqvelk01 tzg]# rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch[root@tzgsqvelk01 tzg]# cat > /etc/yum.repos.d/elasticsearch.repo <<-EOF[elasticsearch-5.x]name=Elasticsearch repository for 5.x packagesbaseurl=https://artifacts.elastic.co/packages/5.x/yumgpgcheck=1gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearchenabled=1autorefresh=1type=rpm-mdEOF[root@tzgsqvelk01 tzg]# yum install elasticsearch -y[root@tzgsqvelk01 tzg]# /bin/systemctl daemon-reload[root@tzgsqvelk01 tzg]# /bin/systemctl enable elasticsearch.serviceCreated symlink from /etc/systemd/system/multi-user.target.wants/elasticsearch.service to /usr/lib/systemd/system/elasticsearch.service.[root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service[root@tzgsqvelk01 tzg]# systemctl stop elasticsearch.service[root@tzgsqvelk01 tzg]# grep -v "^#" /etc/elasticsearch/elasticsearch.ymlnode.name: tzg-elk01path.data: /tzgData/elk/elasticsearch/datapath.logs: /opt/elk/elasticsearch/logsnetwork.host: 0.0.0.0[root@tzgsqvelk01 tzg]# mkdir -p /tzgData/elk/elasticsearch/data[root@tzgsqvelk01 tzg]# mkdir -p /opt/elk/elasticsearch/logs[root@tzgsqvelk01 tzg]# chown -R elasticsearch.elasticsearch /tzgData/elk/[root@tzgsqvelk01 tzg]# chown -R elasticsearch.elasticsearch /opt/elk/[root@tzgsqvelk01 tzg]#[root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service[root@tzgsqvelk01 tzg]#[root@tzgsqvelk01 tzg]#[root@tzgsqvelk01 tzg]# curl http://localhost:9200{  "name" : "tzg-elk01",  "cluster_name" : "elasticsearch",  "cluster_uuid" : "K8bEym6OSpm9o1Y4CJrrEg",  "version" : {    "number" : "5.1.1",    "build_hash" : "5395e21",    "build_date" : "2016-12-06T12:36:15.409Z",    "build_snapshot" : false,    "lucene_version" : "6.3.0"  },  "tagline" : "You Know, for Search"}[root@tzgsqvelk01 tzg]# systemctl stop elasticsearch.service[root@tzgsqvelk01 tzg]# /usr/share/elasticsearch/bin/elasticsearch-plugin install x-pack[root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service[root@tzgsqvelk01 tzg]# curl -XPUT -u elastic 'localhost:9200/_xpack/security/user/elastic/_password' -d '{"password" : "123456" }'Enter host password for user 'elastic':{}[root@tzgsqvelk01 tzg]#[root@tzgsqvelk01 tzg]# systemctl stop elasticsearch.service[root@tzgsqvelk01 tzg]# /usr/share/elasticsearch/bin/elasticsearch-plugin install x-pack[root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service

kibana & x-pack安装

[root@tzgsqvelk01 tzg]# cat > /etc/yum.repos.d/kibana.repo <<-EOF[kibana-5.x]name=Kibana repository for 5.x packagesbaseurl=https://artifacts.elastic.co/packages/5.x/yumgpgcheck=1gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearchenabled=1autorefresh=1type=rpm-mdEOF[root@tzgsqvelk01 tzg]#[root@tzgsqvelk01 tzg]#[root@tzgsqvelk01 tzg]# yum install kibana -y[root@tzgsqvelk01 tzg]# grep "^[a-z]" /etc/kibana/kibana.ymlserver.host: "172.16.5.37"elasticsearch.username: "elastic"elasticsearch.password: "123456"[root@tzgsqvelk01 tzg]# systemctl stop kibana.service[root@tzgsqvelk01 tzg]# /usr/share/kibana/bin/kibana-plugin install file:///home/x-pack-5.1.1.zip[root@tzgsqvelk01 tzg]# systemctl start kibana.service[root@tzgsqvelk01 tzg]# curl -XPUT -u elastic 'localhost:9200/_xpack/security/user/kibana/_password' -d '{  "password" : "123456"}'Enter host password for user 'elastic':

logstash 安装

[root@tzgsqvelk01 tzg]# cat > /etc/yum.repos.d/logstash.repo <<-EOF[logstash-5.x]name=Elastic repository for 5.x packagesbaseurl=https://artifacts.elastic.co/packages/5.x/yumgpgcheck=1gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearchenabled=1autorefresh=1type=rpm-mdEOF[root@tzgsqvelk01 tzg]# yum install logstash -y[root@tzgsqvelk01 tzg]# /usr/share/logstash/bin/logstash -e 'input { stdin{} } output { elasticsearch { hosts => "172.16.5.37:9200" user => elastic  password => 123456 } } 'hahahhaWARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaultsCould not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs to consoleThe stdin plugin is now waiting for input:10:47:39.397 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>["http://~hidden~:~hidden~@172.16.5.37:9200"]}}10:47:39.400 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Running health check to see if an Elasticsearch connection is working {:url=>#<URI::HTTP:0x673e5894 URL:http://~hidden~:~hidden~@172.16.5.37:9200>, :healthcheck_path=>"/"}10:47:39.617 [[main]-pipeline-manager] WARN  logstash.outputs.elasticsearch - Restored connection to ES instance {:url=>#<URI::HTTP:0x673e5894 URL:http://~hidden~:~hidden~@172.16.5.37:9200>}10:47:39.617 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Using mapping template from {:path=>nil}10:47:39.658 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>50001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"_all"=>{"enabled"=>true, "norms"=>false}, "dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword"}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date", "include_in_all"=>false}, "@version"=>{"type"=>"keyword", "include_in_all"=>false}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}10:47:39.772 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Installing elasticsearch template to _template/logstash10:47:39.815 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["172.16.5.37:9200"]}10:47:39.817 [[main]-pipeline-manager] INFO  logstash.pipeline - Starting pipeline {"id"=>"main", "pipeline.workers"=>8, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>1000}10:47:39.823 [[main]-pipeline-manager] INFO  logstash.pipeline - Pipeline main started10:47:39.856 [Api Webserver] INFO  logstash.agent - Successfully started Logstash API endpoint {:port=>9600}^C10:47:47.587 [SIGINT handler] WARN  logstash.runner - SIGINT received. Shutting down the agent.10:47:47.592 [LogStash::Runner] WARN  logstash.agent - stopping pipeline {:id=>"main"}[root@tzgsqvelk01 tzg]# /usr/share/logstash/bin/system-installSuccessfully created system startup script for Logstash[root@tzgsqvelk01 tzg]# chown -R logstash.logstash /usr/share/logstash/data/

redis安装

[root@ tzgsqvelk01 app]# wget http://download.redis.io/releases/redis-3.2.1.tar.gz[root@ tzgsqvelk01 app]# yum install gcc -y[root@ tzgsqvelk01 app]# tar zxvf redis-3.2.1.tar.gz[root@ tzgsqvelk01 app]# cd redis-3.2.1[root@tzgsqvelk01 redis-3.2.1]# make[root@tzgsqvelk01 redis-3.2.1]# make install PREFIX=/opt/app/redis[root@tzgsqvelk01 redis-3.2.1]# cat /opt/app/redis/conf/redis.confdaemonize yespidfile /var/run/redis.pidbind 172.16.5.37port 6379timeout 120loglevel noticelogfile /opt/app/redis/logs/redis.logdatabases 16save 900 1save 300 100save 60 10000stop-writes-on-bgsave-error yesrdbcompression yesrdbchecksum yesdbfilename dump.rdbdir /tzgData/redis/varappendonly yesappendfilename "appendonly.aof"appendfsync everysecno-appendfsync-on-rewrite noauto-aof-rewrite-percentage 100auto-aof-rewrite-min-size 64mbaof-load-truncated yesslowlog-log-slower-than 10000slowlog-max-len 128hash-max-ziplist-entries 512hash-max-ziplist-value 64list-max-ziplist-entries 512list-max-ziplist-value 64set-max-intset-entries 512zset-max-ziplist-entries 128zset-max-ziplist-value 64hll-sparse-max-bytes 3000activerehashing yesmaxmemory 4096000000maxclients 10000slave-serve-stale-data yesslave-read-only yesslave-priority 100repl-diskless-sync norepl-diskless-sync-delay 5repl-disable-tcp-nodelay no[root@tzgsqvelk01 redis-3.2.1]# cp utils/redis_init_script /etc/init.d/redisd[root@tzgsqvelk01 redis-3.2.1]# vim /etc/init.d/redisd#!/bin/sh# chkconfig:   2345 90 10# description:  Redis is a persistent key-value database#REDISPORT=6379EXEC=/opt/app/redis/bin/redis-serverCLIEXEC=/opt/app/redis/bin/redis-cliPIDFILE=/var/run/redis_${REDISPORT}.pidCONF="/opt/app/redis/conf/redis.conf"[root@tzgsqvelk01 tzg]# chkconfig redisd on[root@tzgsqvelk01 tzg]# service redisd start[root@tzgsqvelk01 app]# rm -rf redis-3.2.1[root@tzgsqvelk01 app]# rm -rf redis-3.2.1.tar.gz

logstash和elasticsearch 无需授权配置

参考官方文档
https://www.elastic.co/guide/en/x-pack/current/anonymous-access.html

[root@ tzgsqvelk01 elasticsearch]# vim elasticsearch.ymlxpack.security.authc:  anonymous:    username: anonymous_user    roles: logstash_logs    authz_exception: true######添加 logstash_logs的权限######curl -XPOST -u elastic 'localhost:9200/_xpack/security/role/logstash_logs' -d '{  "cluster": ["manage_index_templates", "monitor"],  "indices": [    {      "names": [ "logstash-*","business-index-*"],       "privileges": ["write","delete","create_index"]    }  ]}'[root@ tzgsqvelk01 elasticsearch]# systemctl restart elasticsearch.service[root@ tzgsqvelk01 elasticsearch]# /usr/share/logstash/bin/logstash -e 'input { stdin{} } output { elasticsearch { hosts => "192.168.3.210:9200"} } 'tzghahahaSending Logstash's logs to /var/log/logstash/logs which is now configured via log4j2.propertiesThe stdin plugin is now waiting for input:[2016-12-30T08:48:07,332][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>["http://192.168.3.210:9200"]}}[2016-12-30T08:48:07,342][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:url=>#<URI::HTTP:0x52cfa9fa URL:http://192.168.3.210:9200>, :healthcheck_path=>"/"}[2016-12-30T08:48:07,592][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>#<URI::HTTP:0x52cfa9fa URL:http://192.168.3.210:9200>}[2016-12-30T08:48:07,599][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>nil}[2016-12-30T08:48:07,753][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>50001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"_all"=>{"enabled"=>true, "norms"=>false}, "dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword"}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date", "include_in_all"=>false}, "@version"=>{"type"=>"keyword", "include_in_all"=>false}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}[2016-12-30T08:48:07,773][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["192.168.3.210:9200"]}[2016-12-30T08:48:07,784][INFO ][logstash.pipeline        ] Starting pipeline {"id"=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>500}[2016-12-30T08:48:07,814][INFO ][logstash.pipeline        ] Pipeline main started[2016-12-30T08:48:07,989][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}