Centos7.6部署kafka2.5.0+zookeeper3.6.1集群

集群规划：

三台服务器各部署一个kafka集群节点、一个zookeeper集群节点。

PS：命令行开头为#表示在所有主机上操作，命令行开后为具体主机名则表示在该主机上操作。

1、三台服务器基础设置

设置主机名：

# hostnamectl --static set-hostname node1/node2/node3
# vi /etc/hosts
192.168.146.199 node1
192.168.146.200 node2
192.168.146.201 node3

时区调整，时间校准：

# date -R
# timedatectl set-timezone Asia/Shanghai
# yum -y install ntp
# ntpdate ntp1.aliyun.com

安装wget：

$ yum install -y wget

关闭selinux：

# vi /etc/sysconfig/selinux
SELINUX=disabled

重启服务器并验证：

# getenforce
Disabled

设置防火墙

开放zookeeper相关端口2181,2888,3888

# firewall-cmd --zone=public --add-port=2181/tcp --permanent
success
# firewall-cmd --zone=public --add-port=2888/tcp --permanent
success
# firewall-cmd --zone=public --add-port=3888/tcp --permanent
success

开放kafka相关端口9092

# firewall-cmd --zone=public --add-port=9092/tcp --permanent
success

reload 防火墙配置使其生效

# firewall-cmd --reload
success

2、基础环境安装

安装JDK环境

# yum install java-1.8.0-openjdk
# java -version
openjdk version "1.8.0_262"
OpenJDK Runtime Environment (build 1.8.0_262-b10)
OpenJDK 64-Bit Server VM (build 25.262-b10, mixed mode)

创建目录与文件

# mkdir -p /var/zookeeper/log
# touch /var/zookeeper/myid
# mkdir -p /var/log/kafka

3、安装zookeeper

# cd /opt
# wget http://archive.apache.org/dist/zookeeper/zookeeper-3.6.1/apache-zookeeper-3.6.1-bin.tar.gz
# tar -zxvf apache-zookeeper-3.6.1-bin.tar.gz
# mv apache-zookeeper-3.6.1-bin zookeeper-3.6.1

复制模板

# mv /opt/zookeeper-3.6.1/conf/zoo_sample.cfg /opt/zookeeper-3.6.1/conf/zoo.cfg

修改配置文件

# vi /opt/zookeeper-3.6.1/conf/zoo.cfg
dataDir=/var/zookeeper
server.1=node1:2888:3888
server.2=node2:2888:3888
server.3=node3:2888:3888

[root@node1 opt]# echo 1 > /var/zookeeper/myid
[root@node2 opt]# echo 2 > /var/zookeeper/myid
[root@node3 opt]# echo 3 > /var/zookeeper/myid

4、安装kafka

# cd /opt
# wget https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.5.0/kafka_2.13-2.5.0.tgz
# tar -zxvf kafka_2.13-2.5.0.tgz

修改配置文件

[root@node1 opt]# vi /opt/kafka_2.13-2.5.0/config/server.properties
broker.id=0
log.dirs=/var/log/kafka
zookeeper.connect=node1:2181,node2:2181,node3:2181

[root@node2 opt]# vi /opt/kafka_2.13-2.5.0/config/server.properties
broker.id=1
log.dirs=/var/log/kafka
zookeeper.connect=node1:2181,node2:2181,node3:2181

[root@node3 opt]# vi /opt/kafka_2.13-2.5.0/config/server.properties
broker.id=2
log.dirs=/var/log/kafka
zookeeper.connect=node1:2181,node2:2181,node3:2181

5、启动和关闭zookeeper

# /opt/zookeeper-3.6.1/bin/zkServer.sh start
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper-3.6.1/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED

[root@node1 opt]# /opt/zookeeper-3.6.1/bin/zkServer.sh status
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper-3.6.1/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: follower

[root@node2 opt]# /opt/zookeeper-3.6.1/bin/zkServer.sh status
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper-3.6.1/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: follower

[root@node3 opt]# /opt/zookeeper-3.6.1/bin/zkServer.sh status
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper-3.6.1/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: leader

关闭方法

# /opt/zookeeper-3.6.1/bin/zkServer.sh stop

查看日志

# tail -f /opt/zookeeper-3.6.1/logs/zookeeper-root-server-node{myid}.out

6、启动和关闭kafka

# /opt/kafka_2.13-2.5.0/bin/kafka-server-start.sh /opt/kafka_2.13-2.5.0/config/server.properties 1>/dev/null 2>&1 &
# tail -f /opt/kafka_2.13-2.5.0/logs/server.log
关闭方法
# /opt/kafka_2.13-2.5.0/bin/kafka-server-stop.sh

7、测试

# cd /opt/kafka_2.13-2.5.0

创建topic

[root@node1 kafka_2.13-2.5.0]# bin/kafka-topics.sh --create --zookeeper node1:2181,node2:2181,node3:2181 --replication-factor 3 --partitions 3 --topic gxm-test
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
Created topic gxm-test.

查看topic

[root@node2 kafka_2.13-2.5.0]# bin/kafka-topics.sh --describe --zookeeper node1:2181,node2:2181,node3:2181 --topic gxm-test
OpenJDK 64-Bit Server VM warning: If the number of p
71f8
rocessors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
Topic: gxm-test    PartitionCount: 3    ReplicationFactor: 3    Configs:
    Topic: gxm-test    Partition: 0    Leader: 1    Replicas: 1,2,0    Isr: 1,2,0
    Topic: gxm-test    Partition: 1    Leader: 2    Replicas: 2,0,1    Isr: 2,0,1
    Topic: gxm-test    Partition: 2    Leader: 0    Replicas: 0,1,2    Isr: 0,1,2

列出topic

[root@node3 kafka_2.13-2.5.0]# bin/kafka-topics.sh --list --zookeeper node1:2181,node2:2181,node3:2181
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
gxm-test

在node1创建 producer测试

[root@node1 kafka_2.13-2.5.0]# bin/kafka-console-producer.sh --broker-list node1:9092 -topic gxm-test

在node2创建 consumer测试

[root@node2 kafka_2.13-2.5.0]# bin/kafka-console-consumer.sh --bootstrap-server node2:9092 -topic gxm-test --from-beginning

在node3上创建 consumer测试

[root@node3 kafka_2.13-2.5.0]# bin/kafka-console-consumer.sh --bootstrap-server node2:9092 -topic gxm-test --from-beginning

删除topic

[root@node1 kafka_2.13-2.5.0]#  bin/kafka-topics.sh --delete --zookeeper node1:2181,node2:2181,node3:2181 --topic gxm-test