zookeeper 설치
# zookeeper 설치 파일 다운로드
[kalva@kalva-zk01 ~]$ wget https://dlcdn.apache.org/zookeeper/zookeeper-3.8.1/apache-zookeeper-3.8.1-bin.tar.gz
# zookeeper 설치 파일 압축해제
[kalva@kalva-zk01 ~]$ sudo tar xvzf apache-zookeeper-3.8.1-bin.tar.gz -C /usr/local/
# zookeeper 설치파일 /usr/local/zookeeper 로 경로 설정
[kalva@kalva-zk01 ~]$ cd /usr/local
[kalva@kalva-zk01 local]$ sudo ln -s ./apache-zookeeper-3.8.1-bin ./zookeeper
# zookeeper logs 디렉토리 생성
[kalva@kalva-zk01 local]$ sudo mkdir -p /usr/local/zookeeper/logs
# zookeeper 데이터 디렉토리 생성
[kalva@kalva-zk01 local]$ mkdir -p /data/zk
# zookeeper 구분 id 파일 생성 (각 서버에 맞게 생성)
## kalva-zk01 : 1
## kalva-zk02 : 2
## kalva-zk03 : 3
[kalva@kalva-zk01 local]$ echo 1 | sudo tee /data/zk/myid
# zookeeper Configuration 파일 설정
[kalva@kalva-zk01 local]$ echo "tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data/zk
clientPort=2181
autopurge.snapRetainCount=3
autopurge.purgeInterval=1
server.1=kalva-zk01:2888:3888;2181
server.2=kalva-zk02:2888:3888;2181
server.3=kalva-zk03:2888:3888;2181" | sudo tee /usr/local/zookeeper/conf/zoo.cfg
# systemctl zookeeper-server.service 파일 생성
[kalva@kalva-zk01 local]$ echo "[Unit]
Description=zookeeper-server
After=network.target
[Service]
Type=forking
User=zookeeper
Group=zookeeper
SyslogIdentifier=zookeeper-server
WorkingDirectory=/usr/local/zookeeper
Restart=always
RestartSec=0s
ExecStart=/usr/local/zookeeper/bin/zkServer.sh start
ExecStop=/usr/local/zookeeper/bin/zkServer.sh stop
ExecReload=/usr/local/zookeeper/bin/zkServer.sh restart
[Install]
WantedBy=multi-user.target" | sudo tee /etc/systemd/system/zookeeper-server.service
# systemctl daemon-reload, zookeeper-server 사용설정
[kalva@kalva-zk01 local]$ sudo systemctl daemon-reload
[kalva@kalva-zk01 local]$ sudo systemctl enable zookeeper-server
# zookeeper 디렉토리 owner 변경
[kalva@kalva-zk01 local]$ sudo chown -R zookeeper:zookeeper /usr/local/apache-zookeeper-3.8.1-bin
[kalva@kalva-zk01 local]$ sudo chown -R zookeeper:zookeeper /usr/local/zookeeper
[kalva@kalva-zk01 local]$ sudo chown -R zookeeper:zookeeper /data
# zookeeper-server 구동, 상태 확인
[kalva@kalva-zk01 local]$ sudo systemctl start zookeeper-server
[kalva@kalva-zk01 local]$ sudo systemctl status zookeeper-server
● zookeeper-server.service - zookeeper-server
Loaded: loaded (/etc/systemd/system/zookeeper-server.service; enabled; vendor preset: disabled)
Active: active (running) since Fri 2023-03-17 00:18:57 KST; 37s ago
Process: 2264 ExecStart=/usr/local/zookeeper/bin/zkServer.sh start (code=exited, status=0/SUCCESS)
Main PID: 2281 (java)
CGroup: /system.slice/zookeeper-server.service
└─2281 java -Dzookeeper.log.dir=/usr/local/zookeeper/bin/../logs -Dzookeeper.log.file=zookeeper-zookeeper-server-kalva-zk01.log -XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryEr...
Mar 17 00:18:56 kalva-zk01 systemd[1]: Starting zookeeper-server...
Mar 17 00:18:56 kalva-zk01 zookeeper-server[2264]: /usr/bin/java
Mar 17 00:18:56 kalva-zk01 zookeeper-server[2264]: ZooKeeper JMX enabled by default
Mar 17 00:18:56 kalva-zk01 zookeeper-server[2264]: Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Mar 17 00:18:57 kalva-zk01 systemd[1]: Started zookeeper-server.
kafka 설치
# kafka 설치파일 다운로드
[kalva@kalva-kafka01 ~]$ wget https://downloads.apache.org/kafka/3.4.0/kafka_2.13-3.4.0.tgz
# kafka 설치파일 압축해제
[kalva@kalva-kafka01 ~]$ tar xzf kafka_2.13-3.4.0.tgz
# kafka 디렉토리 설정
[kalva@kalva-kafka01 ~]$ sudo mv kafka_2.13-3.4.0 /usr/local/
[kalva@kalva-kafka01 ~]$ sudo ln -s /usr/local/kafka_2.13-3.4.0 /usr/local/kafka
# kafka Configuration 파일 설정 (각 서버에 맞게 설정)
## kalva-kafka01 : broker.id=1, PLAINTEXT://kalva-kafka01:9092
## kalva-kafka02 : broker.id=3, PLAINTEXT://kalva-kafka02:9092
## kalva-kafka03 : broker.id=3, PLAINTEXT://kalva-kafka03:9092
[kalva@kalva-kafka01 ~]$ echo "broker.id=1
listeners=PLAINTEXT://0.0.0.0:9092
advertised.listeners=PLAINTEXT://kalva-kafka01:9092
num.network.threads=4
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/data/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=3
transaction.state.log.replication.factor=3
transaction.state.log.min.isr=2
log.retention.hours=72
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=kalva-zk01:2181,kalva-zk02:2181,kalva-zk03:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=3000" | sudo tee /usr/local/kafka/config/server.properties
# jmx 설정 파일 설정
[kalva@kalva-kafka01 ~]$ echo "JMX_PORT=9999" | sudo tee /usr/local/kafka/config/jmx
# systemctl kafka-server.service 파일 생성
[kalva@kalva-kafka01 ~]$ echo "[Unit]
Description=kafka-server
After=network.target
[Service]
Type=simple
SyslogIdentifier=kafka-server
WorkingDirectory=/usr/local/kafka
EnvironmentFile=/usr/local/kafka/config/jmx
Restart=always
ExecStart=/usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties
ExecStop=/usr/local/kafka/bin/kafka-server-stop.sh
[Install]
WantedBy=multi-user.target" | sudo tee /etc/systemd/system/kafka-server.service
# systemctl daemon-reload, kafka-server 사용설정
[kalva@kalva-kafka01 ~]$ sudo systemctl daemon-reload
[kalva@kalva-kafka01 ~]$ sudo systemctl enable kafka-server
# kafka-server 구동, 상태 확인
[kalva@kalva-kafka01 ~]$ sudo systemctl start kafka-server
[kalva@kalva-kafka01 ~]$ sudo systemctl status kafka-server
● kafka-server.service - kafka-server
Loaded: loaded (/etc/systemd/system/kafka-server.service; enabled; vendor preset: disabled)
Active: active (running) since Fri 2023-03-17 21:51:39 KST; 1s ago
Main PID: 15487 (java)
CGroup: /system.slice/kafka-server.service
└─15487 java -Xmx1G -Xms1G -server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -XX:MaxInlineLevel=15 -Djava.awt.headless=tr...
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: at kafka.server.KafkaServer.startup(KafkaServer.scala:234)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: at kafka.Kafka$.main(Kafka.scala:115)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: at kafka.Kafka.main(Kafka.scala)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: [2023-03-17 21:51:41,575] INFO shutting down (kafka.server.KafkaServer)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: [2023-03-17 21:51:41,577] INFO [ZooKeeperClient Kafka server] Closing. (kafka.zookeeper.ZooKeeperClient)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: [2023-03-17 21:51:41,597] WARN An exception was thrown while closing send thread for session 0x300000075ce0002. (org.apache.zoo....ClientCnxn)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: EndOfStreamException: Unable to read additional data from server sessionid 0x300000075ce0002, likely server has closed socket
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:77)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:350)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1290)
Hint: Some lines were ellipsized, use -l to show in full.
# 토픽 생성
[kalva@kalva-kafka01 ~]$ /usr/local/kafka/bin/kafka-topics.sh --bootstrap-server kalva-kafka01:9092 --replication-factor 1 --partitions 1 --create --topic test01
Created topic test01.
# 메시지 전송
[kalva@kalva-kafka01 ~]$ /usr/local/kafka/bin/kafka-console-producer.sh --broker-list kalva-kafka01:9092,kalva-kafka02:9092,kalva-kafka03:9092 --topic test01
>hi! hi! hi!
# 메시지 읽어오기
[kalva@kalva-kafka01 ~]$ /usr/local/kafka/bin/kafka-console-consumer.sh --bootstrap-server kalva-kafka01:9092 --topic test01 --from-beginning
hi! hi! hi!
