[{"data":1,"prerenderedAt":287},["ShallowReactive",2],{"\u002F2021-05-31-elasticsearch-elk-install-and-use-note":3},{"id":4,"title":5,"body":6,"date":265,"description":12,"extension":266,"meta":267,"navigation":270,"path":282,"seo":283,"stem":284,"tags":285,"__hash__":286},"blogs\u002F_legacy\u002F2021\u002F2021-05-31-elasticsearch-elk-install-and-use-note.md","Elasticsearch环境安装使用笔记",{"type":7,"value":8,"toc":254},"minimark",[9,13,16,19,24,35,38,41,50,53,57,60,71,74,80,87,90,96,100,103,112,115,120,123,129,138,141,146,150,158,164,167,170,179,182,187,190,196,200,203,209,212,218,221,227,230,239,242,248,251],[10,11,12],"p",{},"想要作一些简单的数据分析，看kibana界面似乎很好用的样子，于是尝试安装了一下。",[10,14,15],{},"实际操作环境：windows10和ubuntu20.04。",[10,17,18],{},"本次的目标是，通过ELK框架对在Mysql中的数据进行分析。",[20,21,23],"h2",{"id":22},"elk安装","ELK安装",[10,25,26,27,34],{},"基础的安装部分比较简单，可以参考网上的其他文章。Ubuntu系统上的安装可以参考DigitalOcean的",[28,29,33],"a",{"href":30,"rel":31},"https:\u002F\u002Fwww.digitalocean.com\u002Fcommunity\u002Ftutorials\u002Fhow-to-install-elasticsearch-logstash-and-kibana-elastic-stack-on-ubuntu-20-04",[32],"nofollow","这篇","，基本上就是使用apt命令就可以了。windows上的就更加单纯一些，只要下载对应的bat文件然后解压就可以了。",[10,36,37],{},"不过由于很多工具都是linux上的，而且windows安装jdk的mysql包之类的会很麻烦，最后切换到ubuntu上了。",[10,39,40],{},"另外，如果是在云服务器上进行安装的话，需要注意主机的配置。实际测试，在腾讯云上这个配置：",[42,43,44,47],"blockquote",{},[10,45,46],{},"1核 2GB 1Mbps",[10,48,49],{},"系统盘：高性能云硬盘",[10,51,52],{},"无法正常的运行，只要一启动logstash，就会导致Elasticsearch挂掉。",[54,55,56],"h3",{"id":56},"指令备忘",[10,58,59],{},"使用时注意开启服务：",[61,62,67],"pre",{"className":63,"code":65,"language":66},[64],"language-text","sudo systemctl start elasticsearch\nsudo systemctl start kibana\n","text",[68,69,65],"code",{"__ignoreMap":70},"",[10,72,73],{},"可以使用下面的指令检查elasticsearch是否成功开启",[61,75,78],{"className":76,"code":77,"language":66},[64],"curl -X GET \"localhost:9200\"\n",[68,79,77],{"__ignoreMap":70},[10,81,82,83,86],{},"kibana直接在本地访问就好：",[68,84,85],{},"http:\u002F\u002Flocalhost:5601","。如果是在服务器上可能需要额外的配置端口开放或者用nginx作反向代理，可以参考上面的digitalocean的文档进行。",[10,88,89],{},"如果需要服务常驻的话可以打开：",[61,91,94],{"className":92,"code":93,"language":66},[64],"sudo systemctl enable elasticsearch\nsudo systemctl enable kibana\n",[68,95,93],{"__ignoreMap":70},[20,97,99],{"id":98},"logstash使用","Logstash使用",[10,101,102],{},"ELK里面，其实Elasticsearch和Kibana一开始只需要安装好就可以了，主要的是Logstash配置好，将数据推送过去。",[10,104,105,106,111],{},"LogStash其实官方的这篇就很好用了：",[28,107,110],{"href":108,"rel":109},"https:\u002F\u002Fwww.elastic.co\u002Fguide\u002Fen\u002Flogstash\u002F7.12\u002Ffirst-event.html",[32],"Stashing Your First Event","。",[10,113,114],{},"这边稍微记录下有用的内容，对于找不到LogStash的路径的话，可以使用指令",[42,116,117],{},[10,118,119],{},"whereis logstash",[10,121,122],{},"logstash最基本的功能测试，可以用于检查功能是否正常",[61,124,127],{"className":125,"code":126,"language":66},[64],"cd logstash-7.12.1\nbin\u002Flogstash -e 'input { stdin { } } output { stdout {} }'\n",[68,128,126],{"__ignoreMap":70},[10,130,131,132,137],{},"官方有个",[28,133,136],{"href":134,"rel":135},"https:\u002F\u002Fwww.elastic.co\u002Fguide\u002Fen\u002Flogstash\u002F7.12\u002Fadvanced-pipeline.html",[32],"FileBeat教程","，使用logstash接入filebeat来分析apache的日志，基本照着做就可以了。",[10,139,140],{},"要测试一个配置文件是否正确：",[42,142,143],{},[10,144,145],{},"sudo bin\u002Flogstash --config.test_and_exit -f first.conf",[54,147,149],{"id":148},"mysql连接","Mysql连接",[10,151,152,153,111],{},"Logstash使用mysql需要额外的安装Mysql连接用套件。windows版可以在这里下载",[28,154,157],{"href":155,"rel":156},"https:\u002F\u002Fdev.mysql.com\u002Fdownloads\u002Fconnector\u002Fj\u002F",[32],"连接套件",[61,159,162],{"className":160,"code":161,"language":66},[64],"input {\n  jdbc {\n    jdbc_driver_library => \"C:\u002F\u002FProgram Files (x86)\u002F\u002FMySQL\u002F\u002FConnector J 8.0\u002F\u002Fmysql-connector-java-8.0.25.jar\"\n    jdbc_driver_class => \"com.mysql.cj.jdbc.Driver\"\n    jdbc_connection_string =>  \"jdbc:mysql:\u002F\u002Flocalhost:3306\u002Fawesome_database\"\n    jdbc_user => \"your_awesome_mysql_user\"\n    jdbc_password =>  \"your_awesome_password\"\n    statement =>  \"SELECT * from awseome_table_name\"\n  }\n}\n\noutput{\n  stdout { codec => rubydebug { metadata => true } }\n}\n\n\n",[68,163,161],{"__ignoreMap":70},[10,165,166],{},"jdbc_driver_library 这里需要指向安装目录中对应的连接套件的jar文件。其他的配置项都比较明了。",[10,168,169],{},"ubuntu下也需要对应的安装一些连接套件：",[42,171,172],{},[10,173,174,175],{},"wget ",[28,176,177],{"href":177,"rel":178},"https:\u002F\u002Fdev.mysql.com\u002Fget\u002FDownloads\u002FConnector-J\u002Fmysql-connector-java-5.1.46.zip",[32],[10,180,181],{},"然后解压",[42,183,184],{},[10,185,186],{},"unzip mysql-connector-java-5.1.46.zip",[10,188,189],{},"将解压后内部的jar路径填到logstash配置的input部分",[61,191,194],{"className":192,"code":193,"language":66},[64],"input {\n  jdbc {\n    jdbc_driver_library => \"usr\u002Fshare\u002Flogstash\u002Fmysql-connector-java-5.1.46\u002Fmysql-connector-java-5.1.46.jar\"\n    jdbc_driver_class =>\"com.mysql.jdbc.Driver\"\n    jdbc_connection_string => \"jdbc:mysql:\u002F\u002Flocalhost:3306\u002Fawesome_database_name\"\n    jdbc_user => \"your_awesome_mysql_user\"\n    jdbc_password => \"your_awesome_password\"\n    statement =>  \"SELECT * from awseome_table_name\"\n  }\n}\n\noutput{\n  stdout { codec => rubydebug { metadata => true } }\n}\n\n",[68,195,193],{"__ignoreMap":70},[54,197,199],{"id":198},"输入到elastic","输入到Elastic",[10,201,202],{},"需要将配置中的output改到对应的elastic",[61,204,207],{"className":205,"code":206,"language":66},[64],"output{\n  # stdout { codec => rubydebug { metadata => true } }\n  elasticsearch {\n        hosts => [ \"localhost:9200\" ]\n        index => \"kibana-index\"\n    }\n   # stdout { codec => dots }\n}\n\n",[68,208,206],{"__ignoreMap":70},[10,210,211],{},"加入到index后，kibana中需要创建新的index parten才能正确看到。",[10,213,214,215],{},"如果是本地的话，使用这个网址就可以了：",[68,216,217],{},"http:\u002F\u002Flocalhost:5601\u002Fapp\u002Fmanagement\u002Fkibana\u002FindexPatterns\u002F",[10,219,220],{},"Mysql推送数据多次推送的情况下，会出现很多重复的数据，可以通过在output这边添加配置解决：",[61,222,225],{"className":223,"code":224,"language":66},[64],"output{\n  #stdout { codec => rubydebug { metadata => true } }\n  elasticsearch {\n       hosts => [ \"localhost:9200\" ]\n       index => \"productinfo\"\n       document_id => \"%{productid}\"\n  }\n   # stdout { codec => dots }\n}\n\n",[68,226,224],{"__ignoreMap":70},[10,228,229],{},"其中，productid就是mysql这边的主键。",[10,231,232,233,238],{},"对于更加复杂的情况，可以在input这边处理，比较简单的处理方式是使用jdbc input提供的预定义配合时间戳来进行。详细的可以看[",[28,234,237],{"href":235,"rel":236},"https:\u002F\u002Fwww.elastic.co\u002Fguide\u002Fen\u002Flogstash\u002Fcurrent\u002Fplugins-inputs-jdbc.html#_predefined_parameters",[32],"官方文档","]。",[10,240,241],{},"比较简单的用法：",[61,243,246],{"className":244,"code":245,"language":66},[64],"input {\n  jdbc {\n    statement => \"SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value\"\n    use_column_value => true\n    tracking_column => \"id\"\n    # ... other configuration bits\n  }\n}\n",[68,247,245],{"__ignoreMap":70},[10,249,250],{},"这边用的是id来避免input的时候重复，如果ID不是自增的，可以用timestamp来代替，其实默认sql_last_value就是时间戳形式的。更加复杂的情况需要结合last_run来进行，可以进一步参考文档。",[10,252,253],{},"一时半会没看到自动根据主键来避免重复的input配置方式。这边暂时在output那边控制下就可以了，不纠结了。",{"title":70,"searchDepth":255,"depth":256,"links":257},2,3,[258,261],{"id":22,"depth":255,"text":23,"children":259},[260],{"id":56,"depth":256,"text":56},{"id":98,"depth":255,"text":99,"children":262},[263,264],{"id":148,"depth":256,"text":149},{"id":198,"depth":256,"text":199},"2021-05-31","md",{"layout":268,"status":269,"published":270,"author":271,"author_login":272,"author_email":273,"author_url":274,"wordpress_id":275,"wordpress_url":276,"date_gmt":277,"excerpt":278},"post","publish",true,{"display_name":272,"login":272,"email":273,"url":274},"chaoshikari","chaoshikari@gmail.com","\u002F",3517,"\u002F?p=3517","2021-05-31 06:37:33 +0000",{"type":7,"value":279},[280],[10,281,12],{},"\u002F2021-05-31-elasticsearch-elk-install-and-use-note",{"title":5,"description":12},"_legacy\u002F2021\u002F2021-05-31-elasticsearch-elk-install-and-use-note",[],"lqqeLbcbNY6obK0Fc0mXsCRffnJ_YCxUAk-E2sF8g8M",1788763177162]