kafka 操作

1. Disabling automatically create topic

auto.create.topics.enable=false

2. Add topic

bin/kafka-topics.sh --zookeeper zk_host:port/chroot --create --topic my_topic_name --partitions 10 --replication-factor 2 --configretention.bytes=1073741824 --config retention.ms=2073600000

3. Modify topics

(1) add partition

bin/kafka-topics.sh --zookeeper zk_host:port/chroot --alter --topic my_topic_name --partitions 20

(2) add(update) config

bin/kafka-topics.sh --zookeeper zk_host:port/chroot --alter --topic my_topic_name --config retention.bytes=2147483648

(3) remove a config

bin/kafka-topics.sh --zookeeper zk_host:port/chroot --alter --topic my_topic_name --delete-config retention.ms

(4) delete topic

bin/kafka-topics.sh --zookeeper zk_host:port/chroot --delete --topic my_topic_name

delete.topic.enable=true

4. Graceful shutdown

controlled.shutdown.enable=true

5. Balancing leadership

bin/kafka-preferred-replica-election.sh --zookeeper zk_host:port/chroot

auto.leader.rebalance.enable=true

6. Checking consumer position

bin/kafka-run-class.sh kafka.tools.ConsumerOffsetChecker --zookeeper zk_host:port/chroot --group groupname

bin/kafka-run-class.sh kafka.tools.ConsumerOffsetChecker --zookeeper zk_host:port/chroot --group groupname --topic topicname

7. Expanding your cluster

(1) Automatically migrating data to new machines

vim topics-to-move.json

{"topics": [{"topic": "yurun"}], "version":1}

bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --topics-to-move-json-file topics-to-move.json --broker-list "0,1" --generate

vim expand-cluster-reassignment.json

{"version":1,"partitions":[{"topic":"yurun","partition":8,"replicas":[0,1]},{"topic":"yurun","partition":7,"replicas":[1,0]},{"topic":"yurun","partition":1,"replicas":[1,0]},{"topic":"yurun","partition":0,"replicas":[0,1]},{"topic":"yurun","partition":5,"replicas":[1,0]},{"topic":"yurun","partition":4,"replicas":[0,1]},{"topic":"yurun","partition":2,"replicas":[0,1]},{"topic":"yurun","partition":3,"replicas":[1,0]},{"topic":"yurun","partition":6,"replicas":[0,1]},{"topic":"yurun","partition":9,"replicas":[1,0]}]}

bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file expand-cluster-reassignment.json --execute

bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file expand-cluster-reassignment.json --verify

(2) Custom partition assignment and migration

vim custom-reassignment.json

{"version":1,"partitions":[{"topic":"yurun","partition":0,"replicas":[2,3]},{"topic":"yurun","partition":1,"replicas":[2,3]}]}

bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file custom-reassignment.json --execute

bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file custom-reassignment.json --verify

8. Decommissioning brokers

use 7

9. Increasing replication factor

use 7