目錄
參考專案: https://hub.docker.com/r/bitnami/kafka
| 主機名稱 | 主機IP | node.name | 角色 |
|---|---|---|---|
| k8s-master71u | 192.168.1.71 | zookeeper1 kafka1 | zookeeper kafka |
| k8s-master72u | 192.168.1.72 | zookeeper2 kafka2 | zookeeper kafka |
| k8s-master73u | 192.168.1.73 | zookeeper3 kafka3 | zookeeper kafka |
Zookeeper+Kafka集群建置
目錄結構
root@k8s-master71u:/data/zookeeper-kafka# tree -a
.
├── docker-compose.yml
├── etc
│ ├── localtime
│ └── timezone
├── kafka
│ └── data
│ └── .gitignore
└── zookeeper
└── data
└── .gitignore
設定docker-compose.yml
# 三台k8s-master71u、k8s-master72u、k8s-master73u設定都要修改
修改如下:
k8s-master71u
1. zookeeper1
2. ZOO_SERVER_ID: 1
3. ZOO_SERVERS: 0.0.0.0:2888:3888,192.168.1.72:2888:3888,192.168.1.73:2888:3888
4. kafka1
5. KAFKA_ADVERTISED_HOST_NAME: 192.168.1.71
6. KAFKA_HOST_NAME: 192.168.1.71
7. KAFKA_BROKER_ID: 1
8. KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://192.168.1.71:9092
k8s-master72u
1. zookeeper2
2. ZOO_SERVER_ID: 2
3. ZOO_SERVERS: 192.168.1.71:2888:3888,0.0.0.0:2888:3888,192.168.1.73:2888:3888
4. kafka2
5. KAFKA_ADVERTISED_HOST_NAME: 192.168.1.72
6. KAFKA_HOST_NAME: 192.168.1.72
7. KAFKA_BROKER_ID: 2
8. KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://192.168.1.72:9092
k8s-master73u
1. zookeeper3
2. ZOO_SERVER_ID: 3
3. ZOO_SERVERS: 192.168.1.71:2888:3888,192.168.1.72:2888:3888,0.0.0.0:2888:3888
4. kafka3
5. KAFKA_ADVERTISED_HOST_NAME: 192.168.1.73
6. KAFKA_HOST_NAME: 192.168.1.73
7. KAFKA_BROKER_ID: 3
8. KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://192.168.1.73:9092
以下是k8s-master71u的docker-compose.yml
root@k8s-master71u:/data/zookeeper-kafka# cat docker-compose.yml
# https://hub.docker.com/r/bitnami/kafka
# https://hub.docker.com/r/bitnami/zookeeper
version: '2'
services:
zookeeper1:
image: 'bitnami/zookeeper:latest'
restart: unless-stopped
network_mode: "host"
hostname: zookeeper1
container_name: zookeeper1
ports:
- 2181:2181
- 2888:2888
- 3888:3888
environment:
ZOO_SERVER_ID: 1
ZOO_SERVERS: 0.0.0.0:2888:3888,192.168.1.72:2888:3888,192.168.1.73:2888:3888
ALLOW_ANONYMOUS_LOGIN: yes
volumes:
- ./zookeeper/data:/bitnami/zookeeper
# 校時文件
- ./etc/localtime:/etc/localtime
- ./etc/timezone:/etc/timezone
kafka1:
image: 'bitnami/kafka:latest'
restart: unless-stopped
network_mode: "host"
hostname: kafka1
container_name: kafka1
ports:
- 9092:9092
environment:
KAFKA_ADVERTISED_HOST_NAME: 192.168.1.71
KAFKA_HOST_NAME: 192.168.1.71
KAFKA_ADVERTISED_PORT: 9092
KAFKA_BROKER_ID: 1
KAFKA_CFG_ZOOKEEPER_CONNECT: 192.168.1.71:2181,192.168.1.72:2181,192.168.1.73:2181
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://192.168.1.71:9092
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true
volumes:
- ./kafka/data:/bitnami/kafka
# 校時文件
- ./etc/localtime:/etc/localtime
- ./etc/timezone:/etc/timezone
目錄權限修改
root@k8s-master71u:/data/zookeeper-kafka# chmod 777 -R zookeeper
root@k8s-master71u:/data/zookeeper-kafka# chmod 777 -R kafka
root@k8s-master72u:/data/zookeeper-kafka# chmod 777 -R zookeeper
root@k8s-master72u:/data/zookeeper-kafka# chmod 777 -R kafka
root@k8s-master73u:/data/zookeeper-kafka# chmod 777 -R zookeeper
root@k8s-master73u:/data/zookeeper-kafka# chmod 777 -R kafka
服務啟動
# 啟動
root@k8s-master71u:/data/zookeeper-kafka# docker compose up -d
[+] Running 4/4
✔ Container kafka1 Started 0.1s
✔ Container zookeeper1 Started 0.1s
! kafka1 Published ports are discarded when using host network mode 0.0s
! zookeeper1 Published ports are discarded when using host network mode 0.0s
root@k8s-master71u:/data/zookeeper-kafka# docker compose ps
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
kafka1 bitnami/kafka:latest "/opt/bitnami/scripts/kafka/entrypoint.sh /opt/bitnami/scripts/kafka/run.sh" kafka1 15 seconds ago Up 14 seconds
zookeeper1 bitnami/zookeeper:latest "/opt/bitnami/scripts/zookeeper/entrypoint.sh /opt/bitnami/scripts/zookeeper/run.sh" zookeeper1 15 seconds ago Up 14 seconds
root@k8s-master72u:/data/zookeeper-kafka# docker compose up -d
[+] Running 4/4
✔ Container kafka2 Started 0.1s
✔ Container zookeeper2 Started 0.1s
! kafka2 Published ports are discarded when using host network mode 0.0s
! zookeeper2 Published ports are discarded when using host network mode 0.0s
root@k8s-master72u:/data/zookeeper-kafka# docker compose ps
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
kafka2 bitnami/kafka:latest "/opt/bitnami/scripts/kafka/entrypoint.sh /opt/bitnami/scripts/kafka/run.sh" kafka2 15 seconds ago Up 14 seconds
zookeeper2 bitnami/zookeeper:latest "/opt/bitnami/scripts/zookeeper/entrypoint.sh /opt/bitnami/scripts/zookeeper/run.sh" zookeeper2 15 seconds ago Up 14 seconds
root@k8s-master73u:/data/zookeeper-kafka# docker compose up -d
[+] Running 4/4
✔ Container kafka3 Started 0.1s
✔ Container zookeeper3 Started 0.1s
! kafka3 Published ports are discarded when using host network mode 0.0s
! zookeeper3 Published ports are discarded when using host network mode 0.0s
root@k8s-master73u:/data/zookeeper-kafka# docker compose ps
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
kafka3 bitnami/kafka:latest "/opt/bitnami/scripts/kafka/entrypoint.sh /opt/bitnami/scripts/kafka/run.sh" kafka3 15 seconds ago Up 14 seconds
zookeeper3 bitnami/zookeeper:latest "/opt/bitnami/scripts/zookeeper/entrypoint.sh /opt/bitnami/scripts/zookeeper/run.sh" zookeeper3 15 seconds ago Up 14 seconds
測試kafka消息推送與消費功能是否正常
# 進到一個kafka (71的kafka)，測試kafka消息推送與消費功能是否正常
root@k8s-master71u:/data/zookeeper-kafka# docker exec -ti kafka1 bash
I have no name!@kafka1:/$
# 創建一個topic為test
I have no name!@kafka1:/$ kafka-topics.sh --create --topic test --bootstrap-server localhost:9092 --partitions 3 --replication-factor 2
Created topic test.
# 消費
I have no name!@kafka1:/$ kafka-console-consumer.sh --topic test --bootstrap-server localhost:9092
# 進到另一個kafka (72的kafka)，推送數據到topic test
root@k8s-master72u:/data/zookeeper-kafka# docker exec -ti kafka2 bash
I have no name!@kafka2:/$
I have no name!@kafka2:/$ kafka-console-producer.sh --topic test --broker-list localhost:9092
>
>
>
>hello
>
>
>nihao
>
>
>^CI have no name!@kafka2:/$
# 回到kafka (71的kafka)，可以看到消費
root@k8s-master71u:/data/zookeeper-kafka# docker exec -it kafka bash
I have no name!@kafka1:/$ kafka-console-consumer.sh --topic test --bootstrap-server localhost:9092
hello
nihao
^CProcessed a total of 9 messages
I have no name!@kafka1:/$