How to build a kafka cluster with docker
1. Native Docker commands
1. Delete all dangling data volumes (i.e. useless Volumes, zombie files)
docker volume rm $(docker volume ls -qf dangling=true)
2. Delete all dangling images (that is, images without tags)
docker rmi $(docker images | grep "^<none>" | awk "{print $3}"
3. Remove all closed containers
docker ps -a | grep Exit | cut -d " " -f 1 | xargs docker rm
2. Mirror selection
The environment is M1 version of mbp:
Zookeeper adopts zookeeper
Kafka adopts wurstmeister/kafka
Kafka-Manager adopts scjtqs/kafka-manager
Mysql adopts mysql/mysql-server
3. Cluster planning
1. Create a new docker network
docker network create docker-net --subnet 172.20.10.0/16
docker network ls
2. Cluster planning
hostname | Ip addr | port | listener |
---|---|---|---|
zook1 | 172.20.10.11 | 2184:2181 | |
zook2 | 172.20.10.12 | 2185:2181 | |
zook3 | 172.20.10.13 | 2186:2181 | |
kafka1 | 172.20.10.14 | Internal 9092:9092, External 9192:9192 | kafka1 |
kafka2 | 172.20.10.15 | Internal 9093:9093, External 9193:9193 | kafka2 |
Kafka3 | 172.20.10.16 | Internal 9094:9094, External 9194:9194 | Kafka3 |
This machine (host Mbp) | 172.20.10.2 | ||
kafka manager | 172.20.10.10 | 9000:9000 |
4. Zookeeper cluster installation
1. Create a new file zk-docker-compose.yml
version: "3.4"
services:
zook1:
image: zookeeper:latest
restart: always
hostname: zook1
container_name: zook1 #Container name, convenient to display meaningful names in rancher
ports:
- 2183:2181 #Map the default port number of zookeeper of this container
volumes: # mount data volumes
- "/Users/konsy/Development/volume/zkcluster/zook1/data:/data"
- "/Users/konsy/Development/volume/zkcluster/zook1/datalog:/datalog"
- "/Users/konsy/Development/volume/zkcluster/zook1/logs:/logs"
environment:
ZOO_MY_ID: 1 #It is the node value of zookeeper and the brokerid value of kafka
ZOO_SERVERS: server.1=zook1:2888:3888;2181 server.2=zook2:2888:3888;2181 server.3=zook3:2888:3888;2181
networks:
docker-net:
ipv4_address: 172.20.10.11
zook2:
image: zookeeper:latest
restart: always
hostname: zook2
container_name: zook2 #Container name, convenient to display meaningful names in rancher
ports:
- 2184:2181 #Map the default port number of zookeeper of this container
volumes:
- "/Users/konsy/Development/volume/zkcluster/zook2/data:/data"
- "/Users/konsy/Development/volume/zkcluster/zook2/datalog:/datalog"
- "/Users/konsy/Development/volume/zkcluster/zook2/logs:/logs"
environment:
ZOO_MY_ID: 2 #It is the node value of zookeeper and the brokerid value of kafka
ZOO_SERVERS: server.1=zook1:2888:3888;2181 server.2=zook2:2888:3888;2181 server.3=zook3:2888:3888;2181
networks:
docker-net:
ipv4_address: 172.20.10.12
zook3:
image: zookeeper:latest
restart: always
hostname: zook3
container_name: zook3 #Container name, convenient to display meaningful names in rancher
ports:
- 2185:2181 #Map the default port number of zookeeper of this container
volumes:
- "/Users/konsy/Development/volume/zkcluster/zook3/data:/data"
- "/Users/konsy/Development/volume/zkcluster/zook3/datalog:/datalog"
- "/Users/konsy/Development/volume/zkcluster/zook3/logs:/logs"
environment:
ZOO_MY_ID: 3 #It is the node value of zookeeper and the brokerid value of kafka
ZOO_SERVERS: server.1=zook1:2888:3888;2181 server.2=zook2:2888:3888;2181 server.3=zook3:2888:3888;2181
networks:
docker-net:
ipv4_address: 172.20.10.13
networks:
docker-net:
external:
name: docker-net
2. Execute the script to deploy zookeeper to Docker:
docker compose -f ./zk-docker-compose.yml up -d
Five, Kafka cluster installation
1. Create a new file kafka-docker-compose.yml
version: "2"
services:
kafka1:
image: docker.io/wurstmeister/kafka
restart: always
hostname: kafka1
container_name: kafka1
ports:
- 9093:9093
- 9193:9193
environment:
KAFKA_BROKER_ID: 1
KAFKA_LISTENERS: INSIDE://:9093,OUTSIDE://:9193
#KAFKA_ADVERTISED_LISTENERS=INSIDE://<container>:9092,OUTSIDE://<host>:9094
SKAFKA_ADVERTISED_LISTENERS: INSIDE://kafka1:9093,OUTSIDE://localhost:9193
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
KAFKA_ZOOKEEPER_CONNECT: zook1:2181,zook2:2181,zook3:2181
ALLOW_PLAINTEXT_LISTENER : "yes"
JMX_PORT: 9999 #Open JMX monitoring port to monitor cluster data
volumes:
- /Users/konsy/Development/volume/kafka/kafka1/wurstmeister/kafka:/wurstmeister/kafka
- /Users/konsy/Development/volume/kafka/kafka1/kafka:/kafka
external_links:
- zook1
- zook2
- zook3
networks:
docker-net:
ipv4_address: 172.20.10.14
kafka2:
image: docker.io/wurstmeister/kafka
restart: always
hostname: kafka2
container_name: kafka2
ports:
- 9094:9094
- 9194:9194
environment:
KAFKA_BROKER_ID: 2
KAFKA_LISTENERS: INSIDE://:9094,OUTSIDE://:9194
#KAFKA_ADVERTISED_LISTENERS=INSIDE://<container>:9092,OUTSIDE://<host>:9094
KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka2:9094,OUTSIDE://localhost:9194
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
KAFKA_ZOOKEEPER_CONNECT: zook1:2181,zook2:2181,zook3:2181
ALLOW_PLAINTEXT_LISTENER : "yes"
JMX_PORT: 9999 #Open JMX monitoring port to monitor cluster data
volumes:
- /Users/konsy/Development/volume/kafka/kafka2/wurstmeister/kafka:/wurstmeister/kafka
- /Users/konsy/Development/volume/kafka/kafka2/kafka:/kafka
external_links:
- zook1
- zook2
- zook3
networks:
docker-net:
ipv4_address: 172.20.10.15
kafka3:
image: docker.io/wurstmeister/kafka
restart: always
hostname: kafka3
container_name: kafka3
ports:
- 9095:9095
- 9195:9195
environment:
KAFKA_BROKER_ID: 3
KAFKA_LISTENERS: INSIDE://:9095,OUTSIDE://:9195
#KAFKA_ADVERTISED_LISTENERS=INSIDE://<container>:9092,OUTSIDE://<host>:9094
KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka3:9095,OUTSIDE://localhost:9195
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
KAFKA_ZOOKEEPER_CONNECT: zook1:2181,zook2:2181,zook3:2181
ALLOW_PLAINTEXT_LISTENER : "yes"
JMX_PORT: 9999 #Open JMX monitoring port to monitor cluster data
volumes:
- /Users/konsy/Development/volume/kafka/kafka3/wurstmeister/kafka:/wurstmeister/kafka
- /Users/konsy/Development/volume/kafka/kafka3/kafka:/kafka
external_links:
- zook1
- zook2
- zook3
networks:
docker-net:
ipv4_address: 172.20.10.16
networks:
docker-net:
external:
name: docker-net
2. Execute the script to deploy kafka to Docker:
docker compose -f ./kafka-docker-compose.yml up -d
3. listeners and advertised.listeners
listeners: The scientific name is listener, which actually tells the external connector what protocol to use to access the Kafka service with the specified host name and port open.
advertised.listeners: Compared with listeners, there is one more advertised. The meaning of Advertised means declared and announced, that is to say, this group of listeners is used by Broker for external release.
For example:
listeners: INSIDE://172.17.0.10:9092,OUTSIDE://172.17.0.10:9094
advertised_listeners: INSIDE://172.17.0.10:9092,OUTSIDE://<public network ip>:port
kafka_listener_security_protocol_map: "INSIDE:SASL_PLAINTEXT,OUTSIDE:SASL_PLAINTEXT"
kafka_inter_broker_listener_name: "INSIDE"
The advertised_listeners listener will be registered in zookeeper;
When we establish a connection to the 172.17.0.10:9092 request, the kafka server will find the INSIDE listener through the listener registered in zookeeper, and then find the corresponding communication ip and port through the listeners;
Similarly, when we establish a connection to the <public network ip>: port request, the kafka server will find the OUTSIDE listener through the listener registered in zookeeper, and then find the corresponding communication ip and port 172.17.0.10:9094 through the listeners;
Summary: advertised_listeners are exposed service ports, and listeners are used to establish connections.
4. Install kafka-manager
4.1 Create a new file kafka-manager-docker-compose.yml
version: "2"
services:
kafka-manager:
image: scjtqs/kafka-manager:latest
restart: always
hostname: kafka-manager
container_name: kafka-manager
ports:
- 9000:9000
external_links: # Connect to containers other than this compose file
- zook1
- zook2
- zook3
- kafka1
- kafka2
- kafka3
environment:
ZK_HOSTS: zook1:2181,zook2:2181,zook3:2181
KAFKA_BROKERS: kafka1:9093,kafka2:9094,kafka3:9095
APPLICATION_SECRET: letmein
KM_ARGS: -Djava.net.preferIPv4Stack=true
networks:
docker-net:
ipv4_address: 172.20.10.10
networks:
docker-net:
external:
name: docker-net
4.2 Execute the script to deploy kafka-manager to Docker:
docker compose -f ./kafka-manager-docker-compose.yml up -d
4.3 Configure Cluster
5. Test Kafka
5.1 Connecting the container
Enter the command line of the kafka container:
docker exec -ti kafka /bin/bash
Enter the directory where kafka is located:
cd opt/kafka_2.13-2.8.1/
5.2 Create a topic with Replication 2 and Partition 2
bin/kafka-topics.sh --create --zookeeper zook1:2181 --replication-factor 2 --partitions 2 --topic partopic
5.3 View topic status
Enter in the opt/kafka_2.12-1.1.0/ directory in the kafka container
bin/kafka-topics.sh --describe --zookeeper zook1:2181 --topic partopic
So far, this article about the implementation of docker to build a kafka cluster is introduced. For more information about docker to build a kafka cluster, please search for previous articles in the Yunhaitian tutorial or continue to browse the related articles below. I hope you will support Yunhaitian more in the future. Tutorial!
Original address: https://blog.csdn.net/sinat_36053757/article/details/123724748
lipitor 40mg oral & lt;a href="https://lipiws.top/"& gt;order lipitor 10mg for sale& lt;/a& gt; lipitor 10mg cheap
Bxucri
2024-03-07