Skip to content

peepeepopapapeepeepo/zabbix-kafka-monitoring

Folders and files

NameName
Last commit message
Last commit date

Latest commit

 

History

4 Commits
 
 
 
 
 
 
 
 

Repository files navigation

Confluent Kafka Monitoring with Zabbix

Zabbix Server

Enable Zabbix JMX Gateway

  1. Install zabbix-java-gataway

    yum install -y zabbix-java-gataway
  2. Configuring zabbix-java-gataway

    vi /etc/zabbix/zabbix_java_gateway.conf

    set START_POLLERS=10

  3. Configuring zabbix-server

    vi /etc/zabbix/zabbix_server.conf

    set to StartJavaPollers=5 Change IP for JavaGateway=IP_address_java_gateway

  4. restart zabbix-server and zabbix-java-gataway

    systemctl restart zabbix-server
    systemctl start zabbix-java-gataway
    systemctl enable zabbix-java-gataway

Install scripts for discovery JMX at Zabbix Server

# cd /tmp/
# git clone https://github.com/peepeepopapapeepeepo/zabbix-kafka-monitoring.git
# cd zabbix-kafka-monitoring/zabbix-server/
# cp jmx_discovery /etc/zabbix/externalscripts
# cp JMXDiscovery-0.0.1.jar /etc/zabbix/externalscripts
# chmod +x /etc/zabbix/externalscripts/jmx_discovery
# chmod +x /etc/zabbix/externalscripts/JMXDiscovery-0.0.1.jar

Kafka Each Servers

Enable JMX Kafka Server

  1. Create authentication files

    # cd /etc/kafka/
    # _Username='kafka'
    # _Password='k@fKaZaB'
    # echo -e "${_Username}\t${_Password}" >> jmxremote.password
    # echo -e "${_Username}\treadonly" >> jmxremote.access
    # chown cp-kafka.confluent jmxremote.*
    # chmod 600 jmxremote.*
  2. Edit systemd startup file

    # systemctl edit confluent-kafka
         | [Service]
         |                  :                       :
        +| Environment="KAFKA_JMX_OPTS=-Dcom.sun.management.jmxremote=true \
        +| -Dcom.sun.management.jmxremote.ssl=false \
        +| -Djava.net.preferIPv4Stack=true \
        +| -Dcom.sun.management.jmxremote.local.only=false \
        +| -Dcom.sun.management.jmxremote.port=12345 \
        +| -Dcom.sun.management.jmxremote.rmi.port=12345 \
        +| -Djava.rmi.server.hostname=<HostIP> \
        +| -Dcom.sun.management.jmxremote.password.file=/etc/kafka/jmxremote.password \
        +| -Dcom.sun.management.jmxremote.access.file=/etc/kafka/jmxremote.access"
    # systemctl restart confluent-kafka
  3. Test JMX

    # mkdir /root/jmxterm && cd /root/jmxterm
    # curl -sLO https://github.com/jiaqi/jmxterm/releases/download/v1.0.1/jmxterm-1.0.1-uber.jar
    # java -jar jmxterm-1.0.1-uber.jar -l 127.0.0.1:12345 -u kafka -p k@fKaZaB
    $> beans
    $> info -b java.lang:type=Threading
    $> get -b kafka.server:name=PartitionCount,type=ReplicaManager Value
    $> get -b java.lang:type=Threading TotalStartedThreadCount
    $> exit
    # mkdir /root/jmxdump && cd /root/jmxdump
    # curl -sLO https://github.com/r4um/jmx-dump/releases/download/0.9.3/jmx-dump-0.9.3-standalone.jar
    # java -jar jmx-dump-0.9.3-standalone.jar -h 127.0.0.1 -p 12345 -c kafka:k@fKaZaB --dump-all
  4. Allow firewalld

    # firewall-cmd --permanent --new-ipset=jmx-zabbix --type=hash:ip
    # firewall-cmd --reload
    # firewall-cmd --permanent --ipset=jmx-zabbix --add-entry=<IP_address_java_gateway>
    # firewall-cmd --reload
    # firewall-cmd --permanent --new-service=jmx-zabbix
    # firewall-cmd --permanent --service=jmx-zabbix --set-description="JMX for Zabbix JMX Gateway"
    # firewall-cmd --permanent --service=jmx-zabbix --set-short="JMX for Zabbix"
    # firewall-cmd --permanent --service=jmx-zabbix --add-port=12345/tcp
    # firewall-cmd --reload
    # firewall-cmd --permanent --new-zone jmx-zabbix
    # firewall-cmd --reload
    # firewall-cmd --permanent --zone=jmx-zabbix --add-source=ipset:jmx-zabbix
    # firewall-cmd --permanent --zone=jmx-zabbix --add-service=jmx-zabbix
    # firewall-cmd --reload
    # firewall-cmd --info-ipset=jmx-zabbix
    # firewall-cmd --zone=jmx-zabbix --list-all
    # firewall-cmd --info-service=jmx-zabbix

Install Burrow

  1. Install Golang

    # cd /tmp
    # curl -sLO https://storage.googleapis.com/golang/go1.13.linux-amd64.tar.gz
    # tar -C /usr/local -xzf /tmp/go1.13.linux-amd64.tar.gz
    # ln -s /usr/local/go/bin/go /usr/local/bin/go
    # ln -s /usr/local/go/bin/godoc /usr/local/bin/godoc
    # ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt
    # export GOROOT=/usr/local/go
    # export PATH=$PATH:$GOROOT/bin
    # vi ~/.bash_profile
      +| export GOROOT=/usr/local/go
      +| export PATH=$PATH:$GOROOT/bin
    # go version
  2. Compile and install Burrow

    # mkdir -p /workspace/go/src/github.com/linkedin
    # cd /tmp
    # curl -sLO https://raw.githubusercontent.com/pote/gpm/v1.4.0/bin/gpm
    # chmod +x gpm
    # mv gpm /usr/local/bin/
    # yum install gcc
    # export GOPATH=/workspace/go
    # cd /workspace/go/src/github.com/linkedin/
    # git clone https://github.com/linkedin/Burrow.git
    # cd Burrow
    # go mod tidy
    # go install
    # ls -l $GOPATH/bin/Burrow
  3. Configure Burrow

    # mkdir -p /opt/burrow/bin/
    # cp $GOPATH/bin/Burrow /opt/burrow/bin/
    # cp -r config /opt/burrow/
    # cd /opt/burrow/config
    # cp burrow.toml burrow.toml.org

    edit burrow.toml to match your environment

    # vi burrow.toml
        +| [general]
        +| pidfile="burrow.pid"
        +| stdout-logfile="burrow.out"
        +| access-control-allow-origin="*"
        +|  
        +| [logging]
        +| filename="logs/burrow.log"
        +| level="info"
        +| maxsize=100
        +| maxbackups=30
        +| maxage=10
        +| use-localtime=false
        +| use-compression=true
        +|  
        +| [zookeeper]
        +| servers=[ "kk1.kafka.local:2181", "kk2.kafka.local:2181", "kk3.kafka.local:2181" ]
        +| timeout=6
        +| root-path="/burrow"
        +|  
        +| [client-profile.test]
        +| client-id="burrow-test"
        +| kafka-version="0.10.0"
        +|  
        +| [cluster.local]
        +| class-name="kafka"
        +| servers=[ "kk1.kafka.local:9092", "kk2.kafka.local:9092", "kk3.kafka.local:9092" ]
        +| client-profile="test"
        +| topic-refresh=120
        +| offset-refresh=30
        +|  
        +| [consumer.local]
        +| class-name="kafka"
        +| cluster="local"
        +| servers=[ "kk1.kafka.local:9092", "kk2.kafka.local:9092", "kk3.kafka.local:9092" ]
        +| client-profile="test"
        +| group-blacklist="^(console-consumer-|python-kafka-consumer-|quick-).*$"
        +| group-whitelist=""
        +|  
        +| [consumer.local_zk]
        +| class-name="kafka_zk"
        +| cluster="local"
        +| servers=[ "kk1.kafka.local:2181", "kk2.kafka.local:2181", "kk3.kafka.local:2181" ]
        +| zookeeper-path="/kafka-cluster"
        +| zookeeper-timeout=30
        +| group-blacklist="^(console-consumer-|python-kafka-consumer-|quick-).*$"
        +| group-whitelist=""
        +|  
        +| [httpserver.default]
        +| address="127.0.0.1:8000"
        +|  
        +| [storage.default]
        +| class-name="inmemory"
        +| workers=20
        +| intervals=15
        +| expire-group=604800
        +| min-distance=1
  4. Install startup script and start Burrow as linux service

    # cd /tmp/
    # git clone https://github.com/peepeepopapapeepeepo/zabbix-kafka-monitoring.git
    # cd zabbix-kafka-monitoring/kafka-consumer/
    # cp burrow.service /etc/systemd/service/
    # systemctl daemon-reload
    # systemctl start burrow.service
    # systemctl enable burrow.service
  5. Functional Test

    # curl --noproxy "*" -s http://127.0.0.1:8000/v3/kafka/local/consumer | jq

Configure Zabbix Agent

  1. Install kafka_consumers.sh

    # mkdir /var/lib/zabbix/
    # cd /tmp/zabbix-kafka-monitoring/kafka-consumer/
    # cp kafka_consumers.sh /var/lib/zabbix/
    # chmod +x /var/lib/zabbix/kafka_consumers.sh
    # chown -R zabbix.zabbix /var/lib/zabbix/
  2. Install zabbix-agent configuration

    # cp userparameter_kafkaconsumer.conf /etc/zabbix/zabbix_agentd.d
    # systemctl restart zabbix-agent

Zabbix Web GUI

Import template and value mapping file

  1. Import template (zbx_kafka_templates.xml, zbx_templates_kafkaconsumers.xml)

    Configuration > Templates > Import > Choose Files > Import

  2. Import value mapping (zbx_valuemaps_kafkaconsumers.xml)

    Administration > General > Drop Down List: Value Mapping > Import > Choose Files > Import

Note: Don't forget to set macro {$JMX_PASS} and {$JMX_USER}

Releases

No releases published

Packages

No packages published

Languages