通过docker进行部署 Zookeeper、Kafka集群

最近需要本地搭建一个kafka服务进行，想了想还是docker拉一个镜像比较方便，记录下搭建的过程

1、安装虚拟机

个人比较喜欢virtualbox，给人一种很轻量级的感觉，Linux安装的Centos7.3最简版。之前比较迷Linux桌面，但是虚拟机资源实在有限，还是安装个简易版吧。

2、安装docker

添加Docker的存储库

yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo

安装Docker-ce

yum install -y docker-ce

启动Docker

systemctl start docker

测试运行 hello-world

docker run hello-world

参考：https://blog.csdn.net/TangXuZ/article/details/100082144

3、安装docker-compose

其实如果单机部署的话，直接docker search kafka查询一个镜像，然后pull一个就可以了，如果部署一个伪集群，docker-compose的优势就十分明显了：

首先，需要安装python3-pip:

yum install python-pip

然后，安装docker-compose

pip3 install docker-compose

4、创建docker内部网络

内部网络用来进行容器间的通信：

docker network create --subnet 172.23.0.0/25 --gateway 172.23.0.1 baccano

5、部署zookeeper集群

网上很多攻略都是把zookeeper和kafka放到一个编排里去的，个人认为分开的话，管理会比较容易，zookeeper的docker-compose.yml如下：

version: '3'
services:
    zoo1:
        image: zookeeper   
        container_name: zoo1
        restart: always
        hostname: zoo1            
        ports:
            - 2181:2181 
        environment:       
            ZOO_MY_ID: 1   
            ZOO_SERVERS: server.1=0.0.0.0:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181  
    zoo2:
        image: zookeeper
        container_name: zoo2
        restart: always
        hostname: zoo2        
        ports:
            - 2182:2181
        environment:
            ZOO_MY_ID: 2
            ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=0.0.0.0:2888:3888;2181 server.3=zoo3:2888:3888;2181
    zoo3:
        image: zookeeper
        container_name: zoo3
        restart: always
        hostname: zoo3     
        ports:
            - 2183:2181
        environment:
            ZOO_MY_ID: 3
            ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=0.0.0.0:2888:3888;2181
networks:
        default:
                external:
                        name:
                                baccano

可以保存配置到一个特定文件夹，例如/usr/local/dockerconfig/zookeeper，然后在目录下执行：

docker-compose up -d

这样，一个包含3个节点的zookeeper服务就搭建起来了，用docker ps，可看到已启动的容器：

605d6d6679fa        zookeeper            "/docker-entrypoint.…"   34 hours ago        Up 34 hours         2888/tcp, 3888/tcp, 8080/tcp, 0.0.0.0:2183->2181/tcp   zoo3
7cc46763292d        zookeeper            "/docker-entrypoint.…"   34 hours ago        Up 34 hours         2888/tcp, 3888/tcp, 8080/tcp, 0.0.0.0:2182->2181/tcp   zoo2
7ca9de7fc9a9        zookeeper            "/docker-entrypoint.…"   34 hours ago        Up 34 hours         2888/tcp, 3888/tcp, 0.0.0.0:2181->2181/tcp, 8080/tcp   zoo1

6、Kafka集群搭建

步骤和zookeeper基本一致，以3个节点为例，docker-compose.yml内容如下：

version: '2'

services:
  kafka1:
    image: wurstmeister/kafka
    restart: always
    hostname: kafka1
    container_name: kafka1
    ports:
    - 9092:9092
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092
      KAFKA_LISTENERS: PLAINTEXT://kafka1:9092
      KAFKA_ADVERTISED_HOST_NAME: kafka1
      KAFKA_ADVERTISED_PORT: 9092
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
    volumes:
    - "/data/kafka/kafka1/:/kafka"
    external_links:
    - zoo1
    - zoo2
    - zoo3


  kafka2:
    image: wurstmeister/kafka
    restart: always
    hostname: kafka2
    container_name: kafka2
    ports:
    - 9093:9093
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:9093
      KAFKA_LISTENERS: PLAINTEXT://kafka2:9093
      KAFKA_ADVERTISED_HOST_NAME: kafka2
      KAFKA_ADVERTISED_PORT: 9093
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
    volumes:
    - "/data/kafka/kafka2/:/kafka"
    external_links:  # 连接本compose文件以外的container
    - zoo1
    - zoo2
    - zoo3

  kafka3:
    image: wurstmeister/kafka
    restart: always
    hostname: kafka3
    container_name: kafka3
    ports:
    - 9094:9094
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka3:9094
      KAFKA_LISTENERS: PLAINTEXT://kafka3:9094
      KAFKA_ADVERTISED_HOST_NAME: kafka3
      KAFKA_ADVERTISED_PORT: 9094
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
    volumes:
    - "/data/kafka/kafka3/:/kafka"
    external_links:  # 连接本compose文件以外的container
    - zoo1
    - zoo2
    - zoo3

networks:
  default:
    external:   # 使用已创建的网络
      name: baccano

启动：