安装JDK
yum install java-1.8.0-openjdk java-1.8.0-openjdk-devel -y
java -version
安装 Maven
mkdir -p /opt
cd /opt
wget http://mirrors.tuna.tsinghua.edu.cn/apache/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz
tar zxf ./apache-maven-3.5.4-bin.tar.gz
ln -s apache-maven-3.5.4 maven
设置环境变量
vim /etc/profile & vim ~/.bashrc
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk.x86_64
export M2_HOME=/opt/maven
export PATH=$JAVA_HOME/bin:$M2_HOME/bin:$PATH
source /etc/profile & source ~/.bashrc
查看版本
java -version
mvn -version
设置host
11.10.0.18 mx-kafka-1
11.10.80.94 mx-kafka-2
11.10.125.117 mx-kafka-3
11.10.4.149 mx-kafka-4
11.10.104.108 mx-kafka-5
11.10.0.18 mx-zookeeper-1
11.10.80.94 mx-zookeeper-2
11.10.125.117 mx-zookeeper-3
11.19.89.187 zookeeper-4
11.19.7.108 zookeeper-5
11.19.123.38 zookeeper-6
11.19.123.38 kafka-3
11.19.10.168 kafka-7
11.19.62.19 kafka-6
11.10.32.252 kafka-5
11.10.184.98 kafka-4
11.19.89.187 kafka-1
11.19.7.108 kafka-2
构建uReplication
yum install git -y
cd /opt
git clone https://github.com/uber/uReplicator.git
cd uReplicator
mvn clean package -DskipTests
设置配置文件
1、zookeeper
cd /opt/uReplicator/
vim ./config/zookeeper.properties
2、helix
vim /opt/uReplicator/config/helix.properties
zkServer=mx-zookeeper-1:2181,mx-zookeeper-2:2181,mx-zookeeper-3:2181
instanceId=uReplicator-instance
helixClusterName=mx-kafka-uReplicator
3、 consumer
vim /opt/uReplicator/config/consumer.properties
zookeeper.connect=zookeeper-4:2181,zookeeper-5:2181,zookeeper-6:2181
zookeeper.connection.timeout.ms=30000
zookeeper.session.timeout.ms=30000
group.id=mx-kafka-uReplicator
consumer.id=mx-kafka-uReplicator-consumer-01
partition.assignment.strategy=roundrobin
socket.receive.buffer.bytes=1048576
fetch.message.max.bytes=8388608
queued.max.message.chunks=5
consumer.timeout.ms=5000
auto.offset.reset=smallest
4、producer
vim /opt/uReplicator/config/producer.properties
bootstrap.servers=mx-kafka-1:9092,mx-kafka-2:9092,mx-kafka-3:9092
client.id=mx-kafka.client
producer.type=async
compression.type=none
serializer.class=kafka.serializer.DefaultEncoder
key.serializer=org.apache.kafka.common.serialization.ByteArraySerializer
value.serializer=org.apache.kafka.common.serialization.ByteArraySerializer
启动 controller
/opt/uReplicator/uReplicator-Distribution/target/uReplicator-Distribution-pkg/bin/start-controller.sh -help
/opt/uReplicator/uReplicator-Distribution/target/uReplicator-Distribution-pkg/bin/start-controller.sh \
-port 9000 \
-zookeeper mx-zookeeper-1:2181,mx-zookeeper-2:2181,mx-zookeeper-3:2181 \
-helixClusterName mx-kafka-uReplicator \
-backUpToGit false \
-autoRebalanceDelayInSeconds 120 \
-localBackupFilePath /data/logs/uReplicator/mx-kafka-uReplicator
nohup /opt/uReplicator/uReplicator-Distribution/target/uReplicator-Distribution-pkg/bin/start-controller.sh -port 9000 -zookeeper mx-zookeeper-1:2181,mx-zookeeper-2:2181,mx-zookeeper-3:2181 -helixClusterName mx-kafka-uReplicator -backUpToGit false -autoRebalanceDelayInSeconds 120 -localBackupFilePath /data/logs/uReplicator/mx-kafka-uReplicator &
启动 worker
/opt/uReplicator/uReplicator-Distribution/target/uReplicator-Distribution-pkg/bin/start-worker.sh -help
/opt/uReplicator/uReplicator-Distribution/target/uReplicator-Distribution-pkg/bin/start-worker.sh \
--consumer.config /opt/uReplicator/config/consumer.properties \
--helix.config /opt/uReplicator/config/helix.properties \
--producer.config /opt/uReplicator/config/producer.properties \
--offset.commit.interval.ms 1000
nohup /opt/uReplicator/uReplicator-Distribution/target/uReplicator-Distribution-pkg/bin/start-worker.sh --consumer.config /opt/uReplicator/config/consumer.properties --helix.config /opt/uReplicator/config/helix.properties --producer.config /opt/uReplicator/config/producer.properties --offset.commit.interval.ms 1000 &
关闭所有
/opt/uReplicator/bin/pkg/stop-all.sh