Logstash

input {
        jdbc {
                type => "db.table.service"
                # 数据库连接地址
                jdbc_connection_string => "jdbc:mysql://192.168.0.1:3306/db?useUnicode=true&characterEncoding=utf-8&useSSL=false&autoReconnect=true"
                # 数据库连接账号密码；
                jdbc_user => "username"
                jdbc_password => "password"
                # MySQL依赖包路径；
                jdbc_driver_library => "/usr/share/logstash/mysql/mysql-connector-java-5.1.48.jar"
                # the name of the driver class for mysql
                jdbc_driver_class => "com.mysql.jdbc.Driver"
                # 数据库重连尝试次数
                connection_retry_attempts => "3"
                # 判断数据库连接是否可用，默认false不开启
                jdbc_validate_connection => "true"
                # 数据库连接可用校验超时时间，默认3600s
                jdbc_validation_timeout => "3600"
                # 开启分页查询（默认false不开启）
                jdbc_paging_enabled => "true"
                # 单次分页查询条数（默认100000,若字段较多且更新频率较高，建议调低此值）
                jdbc_page_size => "500"
                # statement 为查询数据 sql，如果 sql 较复杂，建议配通过 statement_filepath 配置 sql 文件的存放路径
                # statement_filepath => "/usr/share/logstash/mysql/jdbc.sql"
                # sql_last_value 为内置的变量，存放上次查询结果中最后一条数据 tracking_column 的值，此处即为 meta_latest_update
                # 加 Limit 是为了在数据量过大时，首次同步安全过渡同步,但是要注意 Limit 的值必须大于一秒内更新的数据总量，因为 Logstash
                # 记录的 timestamp 类型的 record_last_run 默认只是精确到秒级别，很坑！
                statement => "SELECT * FROM `hdfs_dir_meta_test` WHERE meta_latest_update >= :sql_last_value order by meta_latest_update asc LIMIT 100000"
                # 是否将字段名转换为小写，默认true（如果有数据序列化、反序列化需求，建议改为 false）
                lowercase_column_names => false
                # Value can be any of: fatal,error,warn,info,debug，默认info
                sql_log_level => warn
                # 是否记录上次执行结果，true 表示会将上次执行结果的 tracking_column 字段的值保存到 last_run_metadata_path 指定的文件中
                record_last_run => true
                # 需要记录查询结果某字段的值时，此字段为 true，否则默认 tracking_column 为 timestamp 的值
                use_column_value => true
                # 需要记录的字段，用于增量同步，需是数据库字段
                tracking_column => "meta_latest_update"
                # Value can be any of: numeric,timestamp，Default value is "numeric"
                tracking_column_type => timestamp
                # record_last_run 上次数据存放位置
                last_run_metadata_path => "/usr/share/logstash/mysql/hdfs_meta_last_id"
                # 是否清除 last_run_metadata_path 的记录，需要增量同步时此字段必须为 false
                clean_run => false
                # 同步频率(分 时 天 月 年)，默认每分钟同步一次
                schedule => "* * * * *"
        }
}

filter {
        # json {
        #       source => "message"
        #       remove_field => ["message"]
        # }
        # convert 字段类型转换，将字段TotalMoney数据类型改为float；
        # mutate {
        #       convert => {
        #               "TotalMoney" => "float"
        #       }
        # }
}

output {
        elasticsearch {
                # host => "192.168.1.1"
                # port => "9200"
                # 配置ES集群地址
                hosts => ["192.168.1.1:9200"]
                # 索引名字，必须小写
                index => "hdfs_dir_meta"
                # 数据唯一索引（建议使用数据库 Primary Key）
                document_id => "%{cluster}-${absolute_path}"
        }
        stdout {
                codec => json_lines
        }
}

java.lang.OutOfMemoryError: Java heap space
Dumping heap to java_pid26719.hprof ...
Heap dump file created [1385285026 bytes in 5.825 secs]
Exception in thread "defaultEventExecutorGroup-4-1" java.lang.OutOfMemoryError: Java heap space
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "kafka-producer-network-thread | producer-1"

Exception in thread "defaultEventExecutorGroup-4-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "LogStash::Runner" java.lang.OutOfMemoryError: Java heap space
Exception in thread "Ruby-0-Thread-1: /root/elk_zhangqiang/logstash-6.2.3-ship/lib/bootstrap/environment.rb:6" java.lang.OutOfMemoryError: Java heap space
Exception in thread "nioEventLoopGroup-3-3" java.lang.OutOfMemoryError: Java heap space

systemctl status logstash
● logstash.service - logstash
   Loaded: loaded (/etc/systemd/system/logstash.service; disabled; vendor preset: disabled)
   Active: failed (Result: start-limit) since Thu 2020-02-27 11:12:47 CST; 4s ago
  Process: 12785 ExecStart=/usr/share/logstash/bin/logstash --path.settings /etc/logstash (code=exited, status=1/FAILURE)
 Main PID: 12785 (code=exited, status=1/FAILURE)