ELK是三个开源软件的缩写，分别表示：Elasticsearch , Logstash, Kibana , 它们都是开源软件
Elasticsearch是一个高度可扩展的开源全文搜索和分析引擎，特点：分布式、零配置、自动发现、索引自动分片、索引副本机制、restful风格接口、多数据源、自动搜索负载等。
Logstash 是一个完全开源的工具，对日志收集、分析、并存储。
Kibana 是一个数据可视化平台，可以通过将数据转化为酷炫而强大的图像而实现与数据的交互，为Logstash和elasticsearch提供WEB界面。
ELK官网：https://www.elastic.co/ https://www.elastic.co/downloads https://www.elastic.co/guide/index.html http://kibana.logstash.es/content/logstash/ https://endymecy.gitbooks.io/elasticsearch-guide-chinese/content/getting-started/README.html http://udn.yyuap.com/doc/logstash-best-practice-cn/index.html
环境：Centos 6.6
要求：JDK 1.8+，最好是1.8.0_73
安装elasticsearch参考文档：https://www.elastic.co/guide/en/elasticsearch/reference/5.1/install-elasticsearch.html https://www.elastic.co/guide/en/kibana/5.1/install.html https://www.elastic.co/guide/en/logstash/5.1/installing-logstash.html https://www.elastic.co/guide/en/logstash/5.1/first-event.html
环境部署 1 2 3 4 5 6 7 8 9 10 [root@ELK ~] [root@ELK ~] [root@ELK ~] [root@ELK ~] [root@ELK ~] java version "1.8.0_73" Java(TM) SE Runtime Environment (build 1 .8 .0 _73-b02) Java HotSpot(TM) 64 -Bit Server VM (build 25 .73 -b02, mixed mode)[root@ELK ~]#
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 [root@ELK ~]# yum -y install elasticsearch-5.1.1.rpm [root@ELK ~]# mkdir -p /var/log/elasticsearch [root@ELK ~]# grep -v '^$' /etc/elasticsearch/elasticsearch.yml | grep -v ^# path.data: /etc/elasticsearch/data path.logs: /var/log/elasticsearch [root@ELK ~]# [root@ELK ~]# mkdir -p /etc/elasticsearch/data/ [root@ELK ~]# chmod 775 /etc/elasticsearch/data/ [root@ELK ~]# chgrp elasticsearch /etc/elasticsearch/data/ -R [root@ELK ~]# chkconfig --add elasticsearch [root@ELK ~]# chkconfig elasticsearch on [root@ELK ~]# /etc/init.d/elasticsearch start [root@ELK ~]# ps -ef | grep elasticsearch | grep -v grep 498 3508 1 75 19:20 ? 00:00:10 /usr/local/jdk1.8.0_73/bin/java -Xms2g -Xmx2g -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+DisableExplicitGC -XX:+AlwaysPreTouch -server -Xss1m -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djna.nosys=true -Djdk.io.permissionsUseCanonicalPath=true -Dio.netty.noUnsafe=true -Dio.netty.noKeySetOptimization=true -Dlog4j.shutdownHookEnabled=false -Dlog4j2.disable.jmx=true -Dlog4j.skipJansi=true -XX:+HeapDumpOnOutOfMemoryError -Des.path.home=/usr/share/elasticsearch -cp /usr/share/elasticsearch/lib/elasticsearch-5.1.1.jar:/usr/share/elasticsearch/lib/* org.elasticsearch.bootstrap.Elasticsearch -p /var/run/elasticsearch/elasticsearch.pid -d -Edefault.path.logs=/var/log/elasticsearch -Edefault.path.data=/var/lib/elasticsearch -Edefault.path.conf=/etc/elasticsearch [root@ELK ~]# [root@ELK ~]# ss -tunlp | grep 9200 tcp LISTEN 0 128 ::ffff:127.0.0.1:9200 :::* users :(("java",3508 ,108 )) tcp LISTEN 0 128 ::1:9200 :::* users :(("java",3508 ,109 )) [root@ELK ~]# ss -tunlp | grep 9300 tcp LISTEN 0 128 ::ffff:127.0.0.1:9300 :::* users :(("java",3508 ,101 )) tcp LISTEN 0 128 ::1:9300 :::* users :(("java",3508 ,99 )) [root@ELK ~]# [root@ELK ~]# curl 'http://localhost:9200' { "name" : "mlayMmR" , "cluster_name" : "elasticsearch" , "cluster_uuid" : "QIe1zquiRS24X-NsfmwTNA" , "version" : { "number" : "5.6.1" , "build_hash" : "667b497" , "build_date" : "2017-09-14T19:22:05.189Z" , "build_snapshot" : false , "lucene_version" : "6.6.1" }, "tagline" : "You Know, for Search" } [root@ELK ~]# [root@ELK ~]# curl localhost:9200/_cat/health?v epoch timestamp cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent 1481282834 19:27:14 elasticsearch yellow 1 1 1 1 0 0 1 0 - 50.0% [root@ELK ~]# curl localhost:9200/_cat/nodes?v ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 127.0.0.1 3 96 9 0.06 0.20 0.19 mdi * 9AsElX3 [root@ELK ~]# [root@ELK ~]# curl localhost:9200/_cat/allocation?v shards disk.indices disk.used disk.avail disk.total disk.percent host ip node 1 3.1kb 4.7gb 44.3gb 49gb 9 127.0.0.1 127.0.0.1 9AsElX3 1 UNASSIGNED [root@ELK ~]# [root@ELK ~]# curl 'localhost:9200/_cat/indices?v' health status index uuid pri rep docs.count docs.deleted store.size pri.store.size yellow open logstash-message-2016.12.23 6apP2ZYOQnyewF9E7zNIsQ 5 1 1999 0 1.1mb 1.1mb yellow open logstash-message-2016.12.27 5cisTsn1RKS3oUcAcTERbw 5 1 1195 0 924.6kb 924.6kb yellow open .kibana z4SDEnWrRnePE3tcz_woLA 1 1 2 0 9.6kb 9.6kb [root@ELK ~]#
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 [root@ELK ~]# yum -y install kibana-5.1.1-x86_64.rpm [root@ELK ~]# grep -v '^$' /etc/kibana/kibana.yml | grep -v '^#' server.host: "192.168.31.100" elasticsearch.url: "http://localhost:9200" [root@ELK ~]# [root@ELK ~]# chkconfig --add kibana [root@ELK ~]# chkconfig kibana on [root@ELK ~]# /etc/init.d/kibana start kibana started [root@ELK ~]# ps -ef | grep kibana | grep -v grep kibana 3613 1 37 19:38 pts/0 00:00:16 /usr/share/kibana/bin/../node/bin/node --no-warnings /usr/share/kibana/bin/../src/cli -c /etc/kibana/kibana.yml [root@ELK ~]# ss -tunlp | grep 5601 tcp LISTEN 0 128 192.168.31.100:5601 *:* users :(("node",3613 ,11 )) [root@ELK ~]# [root@ELK ~]# ls /var/log/kibana/ kibana.stderr kibana.stdout [root@ELK ~]#
1 2 3 4 5 6 7 8 9 10 [root@ELK ~]# ln -s /usr/local/jdk1.8.0_73/bin/java /usr/bin/ [root@ELK ~]# whereis java java: /usr/bin/java [root@ELK ~]# [root@ELK ~]# yum -y install logstash-5.1.1.rpm [root@ELK ~]# ls /usr/share/logstash/bin/ cpdump logstash logstash.bat logstash.lib.sh logstash-plugin logstash-plugin.bat setup.bat system-install [root@ELK ~]#
logstash调试 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 [root@ELK ~]# /usr/share/logstash/bin/logstash --path.settings /etc/logstash -e 'input { stdin { } } output { stdout {} }' Sending Logstash's logs to /var/log/logstash which is now configured via log4j2.properties The stdin plugin is now waiting for input: hello world 2016-12-12T09:36:50.758Z 0.0.0.0 hello world ... 注：要加--path.settings /etc/logstash指定配置文件目录，否则报错 #再启一个窗口可以看到logstash进程，不能关掉上面测试命令，screen(crtl+D)后台运行 [root@ELK ~]# ps -ef | grep logstash root 1934 1153 14 17:33 pts/0 00:03:35 /usr/local/jdk1.8.0_73/bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+DisableExplicitGC -Djava.awt.headless=true -Dfile.encoding=UTF-8 -XX:+HeapDumpOnOutOfMemoryError -Xmx1g -Xms256m -Xss2048k -Djffi.boot.library.path=/usr/share/logstash/vendor/jruby/lib/jni -Xbootclasspath/a:/usr/share/logstash/vendor/jruby/lib/jruby.jar -classpath : -Djruby.home=/usr/share/logstash/vendor/jruby -Djruby.lib=/usr/share/logstash/vendor/jruby/lib -Djruby.script=jruby -Djruby.shell=/bin/sh org.jruby.Main --1.9 /usr/share/logstash/lib/bootstrap/environment.rb logstash/runner.rb --path.settings /etc/logstash -e input { stdin { } } output { stdout {} } root 2012 1982 0 17:57 pts/2 00:00:00 grep logstash [root@ELK ~]# #Logstash日志文件位置 [root@ELK ~]# tail -f /var/log/logstash/logstash-plain.log [2016-12-12T17:34:11,669][INFO ][logstash.agent ] No config files found in path {:path=>"/etc/logstash/conf.d/*"} [2016-12-12T17:34:12,498][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>1, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>125} [2016-12-12T17:34:12,517][INFO ][logstash.pipeline ] Pipeline main started [2016-12-12T17:34:13,000][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600} ···
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 [root@ELK ~] input { stdin {} } output { stdout { codec => rubydebug{} } } [root@ELK ~] [root@ELK ~] Sending Logstash's logs to /var/log/logstash which is now configured via log4j2.properties The stdin plugin is now waiting for input: { "@timestamp" => 2016 -12 -13 T09:24 :18.123 Z, "@version" => "1" , "host" => "0.0.0.0" , "message" => "" , "tags" => [] } hello world { "@timestamp" => 2016 -12 -13 T09:24 :26.328 Z, "@version" => "1" , "host" => "0.0.0.0" , "message" => "hello world" , "tags" => [] }
1 2 3 4 5 6 7 8 9 10 1. nohup /usr/share/logstash/bin/logstash --path.settings /etc/logstash -f /etc/logstash/conf.d/test.conf & 2. screen -S logstash_start screen会新开一个会话 输入/usr/share/logstash/bin/logstash --path.settings /etc/logstash -f /etc/logstash/conf.d/test.conf，按ctrl+A+D [root@ELK ~]# ps -ef | grep logstash | grep -v grep root 11458 1 0 17:34 ? 00:00:00 SCREEN -S logstash_start root 11505 11459 55 17:35 pts/2 00:00:54 /usr/local/jdk1.8.0_73/bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+DisableExplicitGC -Djava.awt.headless=true -Dfile.encoding=UTF-8 -XX:+HeapDumpOnOutOfMemoryError -Xmx1g -Xms256m -Xss2048k -Djffi.boot.library.path=/usr/share/logstash/vendor/jruby/lib/jni -Xbootclasspath/a:/usr/share/logstash/vendor/jruby/lib/jruby.jar -classpath : -Djruby.home=/usr/share/logstash/vendor/jruby -Djruby.lib=/usr/share/logstash/vendor/jruby/lib -Djruby.script=jruby -Djruby.shell=/bin/sh org.jruby.Main --1.9 /usr/share/logstash/lib/bootstrap/environment.rb logstash/runner.rb --path.settings /etc/logstash -f /etc/logstash/conf.d/test.conf logstash 11554 1 79 17:36 ? 00:00:22 /usr/bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+DisableExplicitGC -Djava.awt.headless=true -Dfile.encoding=UTF-8 -XX:+HeapDumpOnOutOfMemoryError -Xmx1g -Xms256m -Xss2048k -Djffi.boot.library.path=/usr/share/logstash/vendor/jruby/lib/jni -Xbootclasspath/a:/usr/share/logstash/vendor/jruby/lib/jruby.jar -classpath : -Djruby.home=/usr/share/logstash/vendor/jruby -Djruby.lib=/usr/share/logstash/vendor/jruby/lib -Djruby.script=jruby -Djruby.shell=/bin/sh org.jruby.Main --1.9 /usr/share/logstash/lib/bootstrap/environment.rb logstash/runner.rb --path.settings /etc/logstash [root@ELK ~]#
1 logstash-output-stdout插件，最主要的用途是调试。在其不太有效时，加上命令行参数 -vv 运行，查看更多详细的调试信息
Q & A 1 2 3 4 5 6 报错信息：Logstash could not be started because there is already another instance using the configured data directory. If you wish to run multiple instances, you must change the "path.data" setting. [root@ELK ~]# /usr/share/logstash/bin/logstash --path.settings /etc/logstash/ -f /etc/logstash/conf.d/logstash_192.168.1.21.conf --path.data /data/ELK_data/logstash/data_1.21_program Sending Logstash's logs to /data/ELK_data/logstash/logs which is now configured via log4j2.properties
访问http://ip:5601打开kibana首页
logstash流程图：
===============================https://discuss.elastic.co/t/elasticsearch-warn-unable-to-install-syscall-filter/42819/1
Question2：1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 需要对Java做软链接，链接到/usr/ bin/java，否则安装logstash-5.1 .1 .rpm时报错 #报错信息如下：↓↓↓↓↓↓↓↓↓ Running Transaction Installing : 1 :logstash-5.1 .1 -1 .noarch 1 /1 Using provided startup.options file : /etc/ logstash/startup.options /usr/ share/logstash/ vendor/jruby/ bin/jruby: line 388: /u sr/bin/ java: No such file or directory/usr/ share/logstash/ vendor/jruby/ bin/jruby: line 388: exec: /u sr/bin/ java: cannot execute: No such file or directoryUnable to install system startup script for Logstash. Verifying : 1 :logstash-5.1 .1 -1 .noarch 1 /1 Installed: logstash.noarch 1 :5.1 .1 -1 Complete! [root@ELK ~]# #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ 卸载重新安装logstash-5.1 .1 .rpm即可 [root@ELK ~]# ln -s /usr/ local/jdk1.8.0_73/ bin/java /u sr/bin/ [root@ELK ~]# whereis java java: /usr/ bin/java [root@ELK ~]#
1 2 3 4 5 6 7 8 9 10 11 12 13 14 [root@ELK ~] WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs to console The stdin plugin is now waiting for input: 17 :06 :57.595 [[main]-pipeline-manager] INFO logstash.pipeline - Starting pipeline {"id" =>"main" , "pipeline.workers" =>1 , "pipeline.batch.size" =>125 , "pipeline.batch.delay" =>5 , "pipeline.max_inflight" =>125 }17 :06 :57.607 [[main]-pipeline-manager] INFO logstash.pipeline - Pipeline main started17 :06 :57.738 [Api Webserver] INFO logstash.agent - Successfully started Logstash API endpoint {:port=>9600 }hello world 2016 -12 -12 T09:10 :08.651 Z 0.0 .0.0 hello world17 :10 :14.145 [LogStash::Runner] WARN logstash.agent - stopping pipeline {:id=>"main" }[root@ELK ~] 注：要加--path.settings /etc/logstash指定配置文件目录，否则报错↑↑↑ 正确方法： [root@ELK ~]
Question4：访问Kibana时返回502，如下图：
附件：logstash-5.1.1.rpm kibana-5.1.1-x86_64.rpm elasticsearch-5.1.1.rpm jdk-8u73-linux-x64.gz
本作品采用知识共享署名 2.5 中国大陆许可协议 进行许可，欢迎转载，但转载请注明来自Jack Wang Blog ，并保持转载后文章内容的完整。本人保留所有版权相关权利。http://www.yfshare.vip/2017/11/22/ELK%E5%AE%89%E8%A3%85-%E5%9F%BA%E7%A1%80/