您的位置:首页 > 运维架构 > Docker

docker-compose安装kafka集群及管理监控工具

u536410 2018-11-26 14:38 183 查看

编写docker-compose文件配置zk 和kafka集群

#vim kafka.yml

version: '2'

services:

    zoo1:

        image: zookeeper

        restart: always

        container_name: zoo1

        ports:

            - "2181:2181"

        volumes:

            - "/data/zookeeper/zookeeper1/data:/data"

            - "/data/zookeeper/zookeeper1/datalog:/datalog"

        environment:

            ZOO_MY_ID: 1

            ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888


    zoo2:

        image: zookeeper

        restart: always

        container_name: zoo2

        ports:

            - "2182:2181"

        volumes:

            - "/data/zookeeper/zookeeper2/data:/data"

            - "/data/zookeeper/zookeeper2/datalog:/datalog"

        environment:

            ZOO_MY_ID: 2

            ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888


    zoo3:

        image: zookeeper

        restart: always

        container_name: zoo3

        ports:

            - "2183:2181"

        volumes:

            - "/data/zookeeper/zookeeper3/data:/data"

            - "/data/zookeeper/zookeeper3/datalog:/datalog"

        environment:

            ZOO_MY_ID: 3

            ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888


    broker1:

        image: wurstmeister/kafka

        restart: always

        container_name: broker1

        ports:

          - "9091:9092"

        depends_on:

          - zoo1

          - zoo2

          - zoo3

        environment:

          KAFKA_BROKER_ID: 1

          KAFKA_ADVERTISED_HOST_NAME: broker1

          KAFKA_ADVERTISED_PORT: 9092

          KAFKA_HOST_NAME: broker1

          KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181

          KAFKA_LISTENERS: PLAINTEXT://broker1:9092

          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker1:9092

        volumes:

          - /var/run/docker.sock:/var/run/docker.sock

          - "/data/kafka/kafka1/:/kafka"


    broker2:

        image: wurstmeister/kafka

        restart: always

        container_name: broker2

        ports:

          - "9092:9092"

        depends_on:

          - zoo1

          - zoo2

          - zoo3

        environment:

          KAFKA_BROKER_ID: 2

          KAFKA_ADVERTISED_HOST_NAME: broker2

          KAFKA_ADVERTISED_PORT: 9092

          KAFKA_HOST_NAME: broker2

          KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181

          KAFKA_LISTENERS: PLAINTEXT://broker2:9092

          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker2:9092

        volumes:

          - /var/run/docker.sock:/var/run/docker.sock

          - "/data/kafka/kafka2/:/kafka"


    broker3:

        image: wurstmeister/kafka

        restart: always

        container_name: broker3

        ports:

          - "9093:9092"

        depends_on:

          - zoo1

          - zoo2

          - zoo3

        environment:

          KAFKA_BROKER_ID: 3

          KAFKA_ADVERTISED_HOST_NAME: broker3

          KAFKA_ADVERTISED_PORT: 9092

          KAFKA_HOST_NAME: broker3

          KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181

          KAFKA_LISTENERS: PLAINTEXT://broker3:9092

          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker3:9092

        volumes:

          - /var/run/docker.sock:/var/run/docker.sock

          - "/data/kafka/kafka3/:/kafka"

运行compose文件

#COMPOSE_PROJECT_NAME=kafkatest docker-compose -f kafka.yml up -d

查看docker启动进程

[root@localhost ~]# docker ps

zk 和kafka已启动


kafka-manager管理工具安装

#vim kafka_manager.yml

version: '3.1'

services:

  kafka-manager:

    container_name: kafka-manager

    image: sheepkiller/kafka-manager

    restart: always

    ports:

      - 9000:9000

    environment:

      KM_VERSION: 1.3.3.18

      ZK_HOSTS: zoo1:2181,zoo2:2182,zoo3:2183

启动docker容器

docker-compsoe -f kafka-manager.yml up -d

docker启动后,登陆到docker容器里

  #vim conf/application.conf


    kafka-manager.zkhosts="zoo1:2181,zoo2:2182,zoo3:2183"

    #kafka-manager.zkhosts=${?ZK_HOSTS}

将配置文件注掉一条,然后将zk的地址写入,不然界面是出不来的

以上是出现如下错误的解决方法

Yikes! Ask timed out on [ActorSelection[Anchor(akka://kafka-manager-system/), Path(/user/kafka-manager)]] after [5000 ms]

kafka-manager不支持硬盘挂载,我也被坑了好久,


kafkaoffsetmonitor监控工具安装配置

#vim kafkaoffsetmonitor.yml

vim kafkaoffsetmonitor.yml

version: '2'

services:

  kafka-offset-monitor:

    image: 564239555/kafkaoffsetmonitor

    volumes:

      - /data/kafkaoffsetmonitor/conf:/kafkaoffsetmonitor

    ports:

      - "8089:8089"

    environment:

     ZK_HOSTS: zoo1:2181,zoo2:2182,zoo3:2183

     KAFKA_BROKERS: 172.16.70.51:9091,172.16.70.51:9092,172.16.70.51:90093

     REFRESH_SECENDS: 10

     RETAIN_DAYS: 2

启动镜像

docker-compose -f kafkaoffsetmonitor.yml up -d 

kafkaoffsetmonitor需要一个文件来启动,KafkaOffsetMonitor-assembly-0.2.1.jar,我已将些文件放到镜像中

或者自行下载这个软件,将KafkaOffsetMonitor-assembly-0.2.1.jar 拷贝到docker 容器里,

docker cp /data/kafkaoffsetmonitor/conf/KafkaOffsetMonitor-assembly-0.2.1.jar a45b7987f267:/kafkaoffsetmonitor/

写个启动脚本

#vim /kafkaoffsetmonitor/start.sh

#!/bin/bash

nohup java -cp KafkaOffsetMonitor-assembly-0.2.1.jar com.quantifind.kafka.offsetapp.OffsetGetterWeb --zk zoo1:2181,zoo2:2182,zoo3:2183 --port 8089 --refresh 10.seconds --retain 2.days &

如果启动报错,将--zk 改成宿主机IP 

给执行权限然后启动脚本 

#chmod +x /kafkaoffsetmonitor/start.sh

#sh /kafkaoffsetmonitor/start.sh

看到8089的端口启动,就可以通过浏览器http:IP:8089去访问

网上很多镜像没有这个文件,启动就报错

或者直接下载我的镜像文件

https://hub.docker.com/r/564239555/kafkaoffsetmonitor/

docker pull 564239555/kafkaoffsetmonitor

docker启动后,登陆到docker容器里,执行脚本,看到8089的端口启动就可以通过浏览器去访问了http://宿主机IP:8089

#docker exec -it $dockerCONTAINER ID bash

修改docker启动所需要的脚本文件

#cd /kafkaoffsetmonitor/conf/

需要将zk更改成你自已集群的IP 地址

#./kafkaoffsetmonitor.sh

#ss -tunl





标签:  docker compose kafka