forked from erdemcer/kafka-connect-oracle
-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy path部署启动说明.txt
86 lines (58 loc) · 4.65 KB
/
部署启动说明.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
#部署镜像构建
docker build -t akaiot/kafka-connect-oracle:1.0.72 --build-arg NAME=LogminerSyncKafka --build-arg VERSION=1.0.72 --build-arg JAR_FILE=kafka-connect-oracle-1.0.72.jar .
#镜像标签设置
docker tag akaiot/kafka-connect-oracle:1.0.72 mes.com/tools/kafka-connect-oracle:1.0.72
#docker-compose部署
docker-compose up -d
Oracle同步数据到kafka
https://www.cnblogs.com/bicewow/p/13717143.html
https://kafka.apache.org/documentation.html#connect_rest
--开启归档日志
sqlplus / as sysdba
SQL>shutdown immediate
SQL>startup mount
SQL>alter database archivelog;
SQL>alter database open;
--开启附加日志
SQL>alter database add supplemental log data (all) columns;
--创建kafka-connect-oracle-master连接用户
create role logmnr_role;
grant create session to logmnr_role;
grant execute_catalog_role,select any transaction ,select any dictionary to logmnr_role;
create user kminer identified by kminerpass;
grant logmnr_role to kminer;
alter user kminer quota unlimited on users;
docker exec -it oracle_11g /bin/bash
source /home/oracle/.bash_profile
select * from dba_sys_privs where GRANTEE='kminer';
select * from dba_role_privs where GRANTEE='logmnr_role';
create pfile='/home/oracle/pfile.ora' from spfile;
第一步：启动zookeeper
docker run -d --name zookeeper --publish 2181:2181 --volume /etc/localtime:/etc/localtime --restart=always mes.com/library/zookeeper:3.4.13
第二步：启动kafka
docker run -d --name kafka --publish 9092:9092 --link zookeeper:zookeeper --env JMX_PORT=9999 --env KAFKA_BROKER_ID=100 --env HOST_IP=192.168.223.167 --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 --env KAFKA_ADVERTISED_HOST_NAME=192.168.223.167 --env KAFKA_ADVERTISED_PORT=9092 --restart=always --volume /etc/localtime:/etc/localtime mes.com/library/kafka:2.13_2.7.0
第三步：启动kafkaManger
docker run -d --name kafka-manager --link zookeeper:zookeeper --link kafka:kafka -p 9001:9000 --restart=always --env ZK_HOSTS=zookeeper:2181 mes.com/library/kafka-manager:1.3.3.22
第四步：运行数据库同步连接器
docker run -d --name oraclesync -p 8083:8083 -v /opt/oracle/datasync/config:/home/java-app/config -v /opt/oracle/datasync/sync:/home/java-app/sync -v /opt/oracle/datasync/logs:/home/java-app/logs --restart=always mes.com/tools/kafka-connect-oracle:1.0.72
curl -X POST http://192.168.223.167:8083/connectors/oracle-logminer-connector/tasks/0/restart
#连接器API
https://docs.confluent.io/platform/current/connect/references/restapi.html
#收集器列表
curl http://192.168.223.167:8083/connectors
#重启收集器的第1个任务
curl -X POST http://192.168.223.167:8083/connectors/oracle-logminer-connector/tasks/0/restart
#添加重做收集器
curl -H "Content-Type: application/json" -X POST -d '{"name":"oracle-logminer-redoscn","config":{"connector.class":"com.ecer.kafka.connect.oracle.OracleSourceConnector","tasks.max":"1","topic":"P3OLTP","db.name.alias":"P3OLTP","db.name":"P3OLTP","db.hostname":"192.168.230.15","db.port":"1521","db.user":"kf_dba","db.user.password":"kf_dba#1234","db.fetch.size":"1","table.whitelist":"INSITE.*","table.blacklist":"INSITE.NLOG","parse.dml.data":"true","reset.offset":"true","multitenant":"false","start.scn":""}}' "http://192.168.223.167:8083/connectors"
#删除redo收集器
curl -X DELETE http://192.168.223.167:8083/connectors/oracle-logminer-redoscn
http://192.168.223.167:8083/connector-plugins
[{"class":"com.ecer.kafka.connect.oracle.OracleSinkConnector","type":"sink","version":"1.0.72"},
{"class":"com.ecer.kafka.connect.oracle.OracleSourceConnector","type":"source","version":"1.0.72"},
{"class":"org.apache.kafka.connect.file.FileStreamSinkConnector","type":"sink","version":"2.6.0"},
{"class":"org.apache.kafka.connect.file.FileStreamSourceConnector","type":"source","version":"2.6.0"},
{"class":"org.apache.kafka.connect.mirror.MirrorCheckpointConnector","type":"source","version":"1"},
{"class":"org.apache.kafka.connect.mirror.MirrorHeartbeatConnector","type":"source","version":"1"},
{"class":"org.apache.kafka.connect.mirror.MirrorSourceConnector","type":"source","version":"1"}]
#配置规则验证
curl -H "Content-Type: application/json" -X PUT -d '{"name":"oracle-logminer-redoscn","connector.class":"com.ecer.kafka.connect.oracle.OracleSourceConnector","tasks.max":"1","topic":"P3OLTP","db.name.alias":"P3OLTP","db.name":"P3OLTP","db.hostname":"192.168.230.15","db.port":"1521","db.user":"kf_dba","db.user.password":"kf_dba#1234","db.fetch.size":"1","table.whitelist":"INSITE.*","table.blacklist":"INSITE.NLOG","parse.dml.data":"true","reset.offset":"true","multitenant":"false","start.scn":""}' http://192.168.223.167:8083/connector-plugins/com.ecer.kafka.connect.oracle.OracleSinkConnector/config/validate/