Question

我正在尝试使用logstash在MySQL数据库和Elasticsearch之间同步数据。为此，我创建了一个docker-compose文件，该文件为Elasticsearch，MySQL和Logstash创建了容器。

这是文件：

docker-compose.yml

version: '3.1'

services:
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.1.1
     container_name: pelasticsearch
     environment:
     - discovery.type=single-node
     - cluster.name=docker-cluster
     - bootstrap.memory_lock=true
     - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     ulimits:
       memlock:
         soft: -1
         hard: -1
     volumes:
     - elasticsearch:/home/django/elasticsearch
     ports:
     - 9200:9200

   db:
    image: mysql
    container_name: pmysqldb
    command: --default-authentication-plugin=mysql_native_password
    restart: always
    volumes:
      - ${PWD}/pdata/data:/docker-entrypoint-initdb.d
    environment:
      MYSQL_ROOT_PASSWORD: example
      MYSQL_DATABASE: ecommerce

   adminer:
    image: adminer
    restart: always
    ports:
      - 8080:8080

   logstash:
       image: docker.elastic.co/logstash/logstash:7.1.1
       links:
       - elasticsearch
       volumes:
       - ${PWD}/ls_config:/usr/share/logstash/config/
       command: bin/logstash-plugin install logstash-input-jdbc
       command: logstash -f /usr/share/logstash/config/logstash.conf
       depends_on:
       - elasticsearch

volumes:
  elasticsearch:

我也有logstash.yml和logstash.conf文件来配置我的logstash插件。

logstash.conf

input {
  jdbc {
    jdbc_driver_library => "./mysql-connector-java-5.1.36-bin.jar"
    jdbc_driver_class => "com.mysql.jdbc.Driver"
    jdbc_connection_string => "jdbc:mysql://localhost:3306/ecommerce"
    jdbc_user => "root"
    jdbc_password => "exmaple"
    schedule => "* * * * *"
    statement => "SELECT * from ecommerce.table;"
  }
}
output {
   elasticsearch {
        hosts => ["elasticsearch:9200"]
        user => elastic
        password => changeme
   }
}

logstash.yml

---
http.host: '0.0.0.0'
xpack.monitoring.elasticsearch.hosts: [ "elasticsearch:9200"]
xpack.monitoring.elasticsearch.username: elastic
xpack.monitoring.elasticsearch.password: changeme
xpack.monitoring.enabled: true

因此，当我运行docker-compose up时，我的所有容器都在运行，包括logstash，但是，当logstash尝试运行conf文件时，它似乎无法加载驱动程序并返回以下错误：< / p>

logstash_1       |   2056     LogStash::ConfigurationError
logstash_1       |   2056     com.mysql.jdbc.Driver not loaded. Are you sure you've included the correct jdbc driver in :jdbc_driver_library?
logstash_1       |   2056       /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-jdbc-4.3.13/lib/logstash/plugin_mixins/jdbc/jdbc.rb:163:in `open_jdbc_connection'

由于我是docker和logstash的新手，我在做错什么吗？以及如何解决此错误。

Answer 1

因此，如果有人遇到与我相同的问题，我会发布此信息。我找到了解决方案：因此，第一件事是使用MariaDB而不是MySQL，看来MySQL和logstash有很多错误。我还创建了一个用户，而不是以root用户身份进行连接（这将意味着对logstash.conf文件的更改）我的docker-compose.yml的{{1}}目标如下：

MariaDB

接下来，直接将包含类db: image: mariadb # mysql container_name: ph_mariadb restart: always volumes: - ${PWD}/ph_datalake/data/shopware:/docker-entrypoint-initdb.d environment: MYSQL_USER: "toto" MYSQL_PASSWORD: "123456" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" MYSQL_DATABASE: ecommerce adminer: image: adminer restart: always ports: - 8080:8080的{{1}}文件复制到我的容器中的罐子所在的位置。我是在logstash目标中的jar文件上使用卷目标来完成的，所以它变为：

com.mysql.jdbc.Driver

我的docker-compose.yml文件也将更改：

从配置文件中删除logstash: image: docker.elastic.co/logstash/logstash:7.1.1 links: - elasticsearch - db volumes: - ${PWD}/ls_config/logstash.conf:/usr/share/logstash/config/logstash.conf - ${PWD}/ls_config/logstash.yml:/usr/share/logstash/config/logstash.yml - ${PWD}/ls_config/mariadb-java-client-2.4.3.jar:/usr/share/logstash/logstash-core/lib/jars/mariadb-java-client-2.4.3.jar command: bin/logstash-plugin install logstash-input-jdbc command: logstash -f /usr/share/logstash/config/logstash.conf depends_on: - elasticsearch - db。
将用户名和密码更改为与MariaDB实例上的用户名和密码相同。

所以看起来像这样：

logstash.conf

到目前为止，我的logstash设法找到了该类并启动了一个Driver实例进行连接，但是没有连接。由于某种原因，该连接将地址jdbc_driver_library => "./mysql-connector-java-5.1.36-bin.jar"解释为我的logstash的本地主机，这意味着，当logstash尝试与input { jdbc { jdbc_driver_class => 'org.mariadb.jdbc.Driver' jdbc_connection_string => 'jdbc:mariadb://db:3306/ecommerce' jdbc_user => 'toto' jdbc_password => '123456' schedule => "* * * * *" statement => 'SELECT * from ecommerce.table;' } } output { elasticsearch { hosts => ["elasticsearch:9200"] user => elastic password => changeme index => "logstash" } }连接时，它将尝试连接至其自己的IP地址。

因此，解决方案是仅启动Logstash，并且由于它取决于db（MariaDB）和elasticsearch，因此它已连接 jdbc:mariadb://db:3306/ecommerce

toto

Logstash无法连接到MySQL数据库

docker-compose.yml

logstash.conf

logstash.yml

1 个答案: