From 06358699749d9a48129b0c0f0296fb3af6c03efe Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 19 Dec 2019 17:32:27 +0800 Subject: [PATCH 01/69] oracle null not replace --- .../flink/sql/sink/oracle/OracleSink.java | 17 ++++----- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 36 +++++++++---------- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index c30dc9c60..599cd160c 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -84,18 +84,14 @@ private void buildInsertSql(String scheam, String tableName, List fields * @return */ @Override - public String buildUpdateSql(String scheam, String tableName, List fieldNames, Map> realIndexes, List fullField) { - tableName = DtStringUtil.getTableFullPath(scheam, tableName); - + public String buildUpdateSql(String schema, String tableName, List fieldNames, Map> realIndexes, List fullField) { + tableName = DtStringUtil.getTableFullPath(schema, tableName); StringBuilder sb = new StringBuilder(); - sb.append("MERGE INTO " + tableName + " T1 USING " + "(" + makeValues(fieldNames) + ") T2 ON (" + updateKeySql(realIndexes) + ") "); - String updateSql = getUpdateSql(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)); - if (StringUtils.isNotEmpty(updateSql)) { sb.append(" WHEN MATCHED THEN UPDATE SET "); sb.append(updateSql); @@ -104,7 +100,6 @@ public String buildUpdateSql(String scheam, String tableName, List field sb.append(" WHEN NOT MATCHED THEN " + "INSERT (" + quoteColumns(fieldNames) + ") VALUES (" + quoteColumns(fieldNames, "T2") + ")"); - return sb.toString(); } @@ -141,7 +136,7 @@ protected List keyColList(Map> realIndexes) { } /** - * build update sql , such as UPDATE SET "T1".A="T2".A + * build update sql , such as UPDATE SET "T1"."b"= nvl("T2"."b","T1"."b"),"T1"."c"= nvl("T2"."c","T1"."c") * @param updateColumn create table contained column columns * @param fullColumn real columns , query from db * @param leftTable alias @@ -159,9 +154,9 @@ public String getUpdateSql(List updateColumn, List fullColumn, S continue; } if (containsIgnoreCase(updateColumn,col)) { - list.add(prefixLeft + DtStringUtil.addQuoteForStr(col) + "=" + prefixRight + DtStringUtil.addQuoteForStr(col)); - } else { - list.add(prefixLeft + DtStringUtil.addQuoteForStr(col) + "=null"); + String leftCol = prefixLeft + DtStringUtil.addQuoteForStr(col); + String rightCol = prefixRight + DtStringUtil.addQuoteForStr(col); + list.add(leftCol + "= nvl(" + rightCol + "," + leftCol + ")"); } } return StringUtils.join(list, ","); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index 0481e3443..0fc012a8c 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -72,13 +72,13 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { dealOneEqualCon(sqlNode, sideTableName); } - List whereConditionList = Lists.newArrayList();; - Map physicalFields = rdbSideTableInfo.getPhysicalFields(); - SqlNode whereNode = ((SqlSelect) joinInfo.getSelectNode()).getWhere(); - if (whereNode != null) { - // 解析维表中的过滤条件 - ParseUtils.parseSideWhere(whereNode, physicalFields, whereConditionList); - } +// List whereConditionList = Lists.newArrayList();; +// Map physicalFields = rdbSideTableInfo.getPhysicalFields(); +// SqlNode whereNode = ((SqlSelect) joinInfo.getSelectNode()).getWhere(); +// if (whereNode != null) { +// // 解析维表中的过滤条件 +// ParseUtils.parseSideWhere(whereNode, physicalFields, whereConditionList); +// } sqlCondition = "select ${selectField} from ${tableName} where "; for (int i = 0; i < equalFieldList.size(); i++) { @@ -89,17 +89,17 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { sqlCondition += " and "; } } - if (0 != whereConditionList.size()) { - // 如果where条件中第一个符合条件的是维表中的条件 - String firstCondition = whereConditionList.get(0); - if (!"and".equalsIgnoreCase(firstCondition) && !"or".equalsIgnoreCase(firstCondition)) { - whereConditionList.add(0, "and ("); - } else { - whereConditionList.add(1, "("); - } - whereConditionList.add(whereConditionList.size(), ")"); - sqlCondition += String.join(" ", whereConditionList); - } +// if (0 != whereConditionList.size()) { +// // 如果where条件中第一个符合条件的是维表中的条件 +// String firstCondition = whereConditionList.get(0); +// if (!"and".equalsIgnoreCase(firstCondition) && !"or".equalsIgnoreCase(firstCondition)) { +// whereConditionList.add(0, "and ("); +// } else { +// whereConditionList.add(1, "("); +// } +// whereConditionList.add(whereConditionList.size(), ")"); +// sqlCondition += String.join(" ", whereConditionList); +// } sqlCondition = sqlCondition.replace("${tableName}", rdbSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); From 2d4360afddfb19f347106a30728e35f5ba3b196a Mon Sep 17 00:00:00 2001 From: chuixue Date: Fri, 20 Dec 2019 16:39:41 +0800 Subject: [PATCH 02/69] df --- .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 9016d853f..5f46787ca 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -125,9 +125,11 @@ public DataStreamSink consumeDataStream(DataStream> da serializationSchema ); - DataStream ds = dataStream.map((Tuple2 record) -> { - return record.f1; - }).returns(getOutputType().getTypeAt(1)).setParallelism(parallelism); + DataStream ds = dataStream + .filter((Tuple2 record) -> record.f0) + .map((Tuple2 record) -> {return record.f1;}) + .returns(getOutputType().getTypeAt(1)) + .setParallelism(parallelism); DataStreamSink dataStreamSink = (DataStreamSink) kafkaTableSink.consumeDataStream(ds); return dataStreamSink; From 9613a69498559fb1f090ec2ef55e53fba09bc809 Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Tue, 24 Dec 2019 09:56:35 +0800 Subject: [PATCH 03/69] =?UTF-8?q?=E8=A7=A3=E5=86=B3kafka=20source=E8=A7=A3?= =?UTF-8?q?=E6=9E=90value=E4=B8=BAjson=E7=B1=BB=E5=9E=8B=E4=B8=BAnull?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/source/kafka/CustomerJsonDeserialization.java | 11 +++++++---- .../sql/source/kafka/CustomerJsonDeserialization.java | 11 +++++++---- .../sql/source/kafka/CustomerJsonDeserialization.java | 11 +++++++---- .../sql/source/kafka/CustomerJsonDeserialization.java | 11 +++++++---- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index f0ff94830..25369d483 100644 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -186,9 +186,8 @@ private void parseTree(JsonNode jsonNode, String prefix){ JsonNode child = jsonNode.get(next); String nodeKey = getNodeKey(prefix, next); - if (child.isValueNode()){ - nodeAndJsonNodeMapping.put(nodeKey, child); - }else if(child.isArray()){ + nodeAndJsonNodeMapping.put(nodeKey, child); + if(child.isArray()){ parseTree(child, nodeKey); }else { parseTree(child, nodeKey); @@ -258,7 +257,11 @@ private Object convert(JsonNode node, TypeInformation info) { if (info.getTypeClass().equals(Types.BOOLEAN.getTypeClass())) { return node.asBoolean(); } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { - return node.asText(); + String value = node.asText(); + if ("".equals(value)){ + value = node.toString(); + } + return value; } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { return Date.valueOf(node.asText()); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index cd0121756..70ebd3151 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -187,9 +187,8 @@ private void parseTree(JsonNode jsonNode, String prefix){ JsonNode child = jsonNode.get(next); String nodeKey = getNodeKey(prefix, next); - if (child.isValueNode()){ - nodeAndJsonNodeMapping.put(nodeKey, child); - } else if(child.isArray()){ + nodeAndJsonNodeMapping.put(nodeKey, child); + if(child.isArray()){ parseTree(child, nodeKey); }else { parseTree(child, nodeKey); @@ -261,7 +260,11 @@ private Object convert(JsonNode node, TypeInformation info) { if (info.getTypeClass().equals(Types.BOOLEAN.getTypeClass())) { return node.asBoolean(); } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { - return node.asText(); + String value = node.asText(); + if ("".equals(value)){ + value = node.toString(); + } + return value; } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { return Date.valueOf(node.asText()); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index f0ba947d0..b041fb7b5 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -192,9 +192,8 @@ private void parseTree(JsonNode jsonNode, String prefix){ JsonNode child = jsonNode.get(next); String nodeKey = getNodeKey(prefix, next); - if (child.isValueNode()) { - nodeAndJsonNodeMapping.put(nodeKey, child); - } else if(child.isArray()){ + nodeAndJsonNodeMapping.put(nodeKey, child); + if(child.isArray()){ parseTree(child, nodeKey); } else { parseTree(child, nodeKey); @@ -266,7 +265,11 @@ private Object convert(JsonNode node, TypeInformation info) { if (info.getTypeClass().equals(Types.BOOLEAN.getTypeClass())) { return node.asBoolean(); } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { - return node.asText(); + String value = node.asText(); + if ("".equals(value)){ + value = node.toString(); + } + return value; } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { return Date.valueOf(node.asText()); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index 3a1d332bc..d829c0ff0 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -192,9 +192,8 @@ private void parseTree(JsonNode jsonNode, String prefix){ JsonNode child = jsonNode.get(next); String nodeKey = getNodeKey(prefix, next); - if (child.isValueNode()){ - nodeAndJsonNodeMapping.put(nodeKey, child); - }else if(child.isArray()){ + nodeAndJsonNodeMapping.put(nodeKey, child); + if(child.isArray()){ parseTree(child, nodeKey); }else { parseTree(child, nodeKey); @@ -266,7 +265,11 @@ private Object convert(JsonNode node, TypeInformation info) { if (info.getTypeClass().equals(Types.BOOLEAN.getTypeClass())) { return node.asBoolean(); } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { - return node.asText(); + String value = node.asText(); + if ("".equals(value)){ + value = node.toString(); + } + return value; } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { return Date.valueOf(node.asText()); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { From c3139e3f94c68a5537489996bbcd69a0a7b98c51 Mon Sep 17 00:00:00 2001 From: chuixue Date: Tue, 24 Dec 2019 10:33:15 +0800 Subject: [PATCH 04/69] =?UTF-8?q?kakfa=E8=A7=A3=E6=9E=90=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 8 +++++--- .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 8 +++++--- .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 8 +++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 44bf9f98b..23d384b7c 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -113,9 +113,11 @@ public void emitDataStream(DataStream> dataStream) { serializationSchema ); - DataStream ds = dataStream.map((Tuple2 record) -> { - return record.f1; - }).returns(getOutputType().getTypeAt(1)).setParallelism(parallelism); + DataStream ds = dataStream + .filter((Tuple2 record) -> record.f0) + .map((Tuple2 record) -> {return record.f1;}) + .returns(getOutputType().getTypeAt(1)) + .setParallelism(parallelism); kafkaTableSink.emitDataStream(ds); } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index a6c3b98a4..17d3bec51 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -117,9 +117,11 @@ public void emitDataStream(DataStream> dataStream) { serializationSchema ); - DataStream ds = dataStream.map((Tuple2 record) -> { - return record.f1; - }).returns(getOutputType().getTypeAt(1)).setParallelism(parallelism); + DataStream ds = dataStream + .filter((Tuple2 record) -> record.f0) + .map((Tuple2 record) -> {return record.f1;}) + .returns(getOutputType().getTypeAt(1)) + .setParallelism(parallelism); kafkaTableSink.emitDataStream(ds); } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 8c757ce4a..ad8616b23 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -120,9 +120,11 @@ public void emitDataStream(DataStream> dataStream) { serializationSchema ); - DataStream ds = dataStream.map((Tuple2 record) -> { - return record.f1; - }).returns(getOutputType().getTypeAt(1)).setParallelism(parallelism); + DataStream ds = dataStream + .filter((Tuple2 record) -> record.f0) + .map((Tuple2 record) -> {return record.f1;}) + .returns(getOutputType().getTypeAt(1)) + .setParallelism(parallelism); kafkaTableSink.emitDataStream(ds); } From 21a3d19788444b88764b7dd31d5668458f86b760 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 24 Dec 2019 19:41:40 +0800 Subject: [PATCH 05/69] hbase side kerberos --- .../flink/sql/side/hbase/HbaseAllReqRow.java | 61 ++++++- .../sql/side/hbase/HbaseAsyncReqRow.java | 7 +- .../sql/side/hbase/table/HbaseSideParser.java | 10 ++ .../side/hbase/table/HbaseSideTableInfo.java | 7 + .../side/hbase/utils/HbaseConfigUtils.java | 150 ++++++++++++++++++ 5 files changed, 228 insertions(+), 7 deletions(-) create mode 100644 hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index f476bbf3d..d8ceaf9fd 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; +import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.map.HashedMap; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,10 +36,12 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.security.PrivilegedAction; import java.sql.SQLException; import java.sql.Timestamp; import java.time.LocalDateTime; @@ -55,6 +58,11 @@ public class HbaseAllReqRow extends AllReqRow { private AtomicReference>> cacheRef = new AtomicReference<>(); + private Connection conn = null; + private Table table = null; + private ResultScanner resultScanner = null; + private Configuration conf = null; + public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new HbaseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); tableName = ((HbaseSideTableInfo)sideTableInfo).getTableName(); @@ -157,13 +165,35 @@ public void flatMap(Row value, Collector out) throws Exception { private void loadData(Map> tmpCache) throws SQLException { SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; - Configuration conf = new Configuration(); - conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); - Connection conn = null; - Table table = null; - ResultScanner resultScanner = null; + boolean openKerberos = HbaseConfigUtils.openKerberos(hbaseSideTableInfo.getHbaseConfig()); + try { - conn = ConnectionFactory.createConnection(conf); + if (openKerberos) { + conf = HbaseConfigUtils.getHadoopConfiguration(hbaseSideTableInfo.getHbaseConfig()); + conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); + String principal = HbaseConfigUtils.getPrincipal(hbaseSideTableInfo.getHbaseConfig()); + String keytab = HbaseConfigUtils.getKeytab(hbaseSideTableInfo.getHbaseConfig()); + + UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, principal, keytab); + Configuration finalConf = conf; + userGroupInformation.doAs(new PrivilegedAction() { + @Override + public Connection run() { + try { + return ConnectionFactory.createConnection(finalConf); + } catch (IOException e) { + LOG.error("Get connection fail with config:{}", finalConf); + throw new RuntimeException(e); + } + } + }); + + } else { + conf = HbaseConfigUtils.getConfig(hbaseSideTableInfo.getHbaseConfig()); + conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); + conn = ConnectionFactory.createConnection(conf); + } + table = conn.getTable(TableName.valueOf(tableName)); resultScanner = table.getScanner(new Scan()); for (Result r : resultScanner) { @@ -192,4 +222,23 @@ private void loadData(Map> tmpCache) throws SQLExcep } } } + + @Override + public void close() throws Exception { + try { + if (null != conn) { + conn.close(); + } + + if (null != table) { + table.close(); + } + + if (null != resultScanner) { + resultScanner.close(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } } \ No newline at end of file diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 62519cef1..d0dd9837b 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -38,6 +38,7 @@ import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; +import org.hbase.async.Config; import org.hbase.async.HBaseClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,7 +95,11 @@ public void open(Configuration parameters) throws Exception { 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new DTThreadFactory("hbase-aysnc")); - hBaseClient = new HBaseClient(hbaseSideTableInfo.getHost(), hbaseSideTableInfo.getParent(), executorService); + Config config = new Config(); + config.overrideConfig("hbase.zookeeper.quorum",hbaseSideTableInfo.getHost()); + config.overrideConfig("hbase.zookeeper.znode.parent",hbaseSideTableInfo.getParent()); + + hBaseClient = new HBaseClient(config, executorService); try { Deferred deferred = hBaseClient.ensureTableExists(tableName) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 3031de100..4b0b615a2 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -24,10 +24,14 @@ import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.MathUtil; +import scala.collection.mutable.HashMap$; +import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; @@ -72,6 +76,12 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map entity.getKey().startsWith("hbase")) + .map(entity -> hbaseTableInfo.getHbaseConfig().put(entity.getKey(), entity.getValue())) + .count(); + return hbaseTableInfo; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java index 377ca393f..070c51f08 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java @@ -50,6 +50,8 @@ public class HbaseSideTableInfo extends SideTableInfo { private Map columnNameFamily; + private Map hbaseConfig = Maps.newHashMap(); + private String tableName; private String[] columnRealNames; @@ -148,6 +150,11 @@ public void setPreRowKey(boolean preRowKey) { this.preRowKey = preRowKey; } + public Map getHbaseConfig() { + return hbaseConfig; + } + + @Override public void finish(){ super.finish(); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java new file mode 100644 index 000000000..1e2be0486 --- /dev/null +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.hbase.utils; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * + * The utility class of HBase connection + * + * Date: 2019/12/24 + * Company: www.dtstack.com + * @author maqi + */ +public class HbaseConfigUtils { + + private static final Logger LOG = LoggerFactory.getLogger(HbaseConfigUtils.class); + + private final static String AUTHENTICATION_TYPE = "Kerberos"; + private final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; + private final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; + private final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; + private final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; + private final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; + private final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; + + private static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + + private static List KEYS_KERBEROS_REQUIRED = Arrays.asList( + KEY_HBASE_SECURITY_AUTHENTICATION, + KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, + KEY_HBASE_MASTER_KEYTAB_FILE, + KEY_HBASE_REGIONSERVER_KEYTAB_FILE, + KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL + ); + + + public static Configuration getConfig(Map hbaseConfigMap) { + Configuration hConfiguration = HBaseConfiguration.create(); + + for (Map.Entry entry : hbaseConfigMap.entrySet()) { + if (entry.getValue() != null && !(entry.getValue() instanceof Map)) { + hConfiguration.set(entry.getKey(), entry.getValue().toString()); + } + } + return hConfiguration; + } + + public static boolean openKerberos(Map hbaseConfigMap) { + if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHORIZATION)) { + return false; + } + return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); + } + + public static Configuration getHadoopConfiguration(Map hbaseConfigMap) { + for (String key : KEYS_KERBEROS_REQUIRED) { + if (StringUtils.isEmpty(MapUtils.getString(hbaseConfigMap, key))) { + throw new IllegalArgumentException(String.format("Must provide [%s] when authentication is Kerberos", key)); + } + } + loadKrb5Conf(hbaseConfigMap); + + Configuration conf = new Configuration(); + if (hbaseConfigMap == null) { + return conf; + } + + hbaseConfigMap.forEach((key, val) -> { + if (val != null) { + conf.set(key, val.toString()); + } + }); + + return conf; + } + + public static String getPrincipal(Map hbaseConfigMap) { + String principal = MapUtils.getString(hbaseConfigMap, KEY_HBASE_MASTER_KERBEROS_PRINCIPAL); + if (StringUtils.isNotEmpty(principal)) { + return principal; + } + + throw new IllegalArgumentException(""); + } + + public static String getKeytab(Map hbaseConfigMap) { + String keytab = MapUtils.getString(hbaseConfigMap, KEY_HBASE_MASTER_KEYTAB_FILE); + if (StringUtils.isNotEmpty(keytab)) { + return keytab; + } + + throw new IllegalArgumentException(""); + } + + public static void loadKrb5Conf(Map kerberosConfig) { + String krb5FilePath = MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); + if (org.apache.commons.lang.StringUtils.isEmpty(krb5FilePath)) { + return; + } + kerberosConfig.put(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); + } + + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { + if (conf == null) { + throw new IllegalArgumentException("kerberos conf can not be null"); + } + + if (org.apache.commons.lang.StringUtils.isEmpty(principal)) { + throw new IllegalArgumentException("principal can not be null"); + } + + if (org.apache.commons.lang.StringUtils.isEmpty(keytab)) { + throw new IllegalArgumentException("keytab can not be null"); + } + + conf.set("hadoop.security.authentication", "Kerberos"); + UserGroupInformation.setConfiguration(conf); + + return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); + } +} From 80f619b11262908011770df021a8f1c988e76ba8 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 24 Dec 2019 20:36:11 +0800 Subject: [PATCH 06/69] fix hbase sink bug --- .../sql/sink/hbase/HbaseOutputFormat.java | 107 +++++++++++------- .../flink/sql/sink/hbase/HbaseSink.java | 50 +++++--- .../sql/sink/hbase/table/HbaseSinkParser.java | 37 ++++++ 3 files changed, 142 insertions(+), 52 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index e88f34372..b893892c0 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -21,6 +21,7 @@ package com.dtstack.flink.sql.sink.hbase; import com.dtstack.flink.sql.sink.MetricOutputFormat; +import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -36,8 +37,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Set; /** * author: jingzhen@dtstack.com @@ -53,6 +55,7 @@ public class HbaseOutputFormat extends MetricOutputFormat { private String tableName; private String[] columnNames; private String[] columnTypes; + private Map columnNameFamily; private String[] families; private String[] qualifiers; @@ -62,9 +65,10 @@ public class HbaseOutputFormat extends MetricOutputFormat { private transient Table table; public final SimpleDateFormat ROWKEY_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss"); - public final SimpleDateFormat FIELD_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); private static int rowLenth = 1000; + private static int dirtyDataPrintFrequency = 1000; + @Override public void configure(Configuration parameters) { @@ -87,7 +91,7 @@ public void open(int taskNumber, int numTasks) throws IOException { } @Override - public void writeRecord(Tuple2 tuple2) throws IOException { + public void writeRecord(Tuple2 tuple2) { Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); @@ -97,52 +101,71 @@ public void writeRecord(Tuple2 tuple2) throws IOException { } Row record = tupleTrans.getField(1); - - List list = new ArrayList<>(); - for(int i = 0; i < rowkey.length; ++i) { - String colName = rowkey[i]; - int j = 0; - for(; j < columnNames.length; ++j) { - if(columnNames[j].equals(colName)) { - break; - } - } - if(j != columnNames.length && record.getField(i) != null) { - Object field = record.getField(j); - if(field == null ) { - list.add("null"); - } else if (field instanceof java.util.Date){ - java.util.Date d = (java.util.Date)field; - list.add(ROWKEY_DATE_FORMAT.format(d)); - } else { - list.add(field.toString()); - } - } + List rowKeyValues = getRowKeyValues(record); + // all rowkey not null + if (rowKeyValues.size() != rowkey.length ) { + LOG.error("row key value must not null,record is ..", record); + outDirtyRecords.inc(); + return; } - String key = StringUtils.join(list, "-"); + String key = StringUtils.join(rowKeyValues, "-"); Put put = new Put(key.getBytes()); for(int i = 0; i < record.getArity(); ++i) { - Object field = record.getField(i); - byte[] val = null; - if (field != null) { - val = field.toString().getBytes(); + Object fieldVal = record.getField(i); + if (fieldVal == null) { + continue; } + byte[] val = fieldVal.toString().getBytes(); byte[] cf = families[i].getBytes(); byte[] qualifier = qualifiers[i].getBytes(); - put.addColumn(cf, qualifier, val); + put.addColumn(cf, qualifier, val); } - table.put(put); + try { + table.put(put); + } catch (IOException e) { + outDirtyRecords.inc(); + if (outDirtyRecords.getCount() % dirtyDataPrintFrequency == 0 || LOG.isDebugEnabled()) { + LOG.error("record insert failed ..", record.toString()); + LOG.error("", e); + } + } - if (outRecords.getCount()%rowLenth == 0){ + if (outRecords.getCount() % rowLenth == 0) { LOG.info(record.toString()); } outRecords.inc(); } + private List getRowKeyValues(Row record) { + List rowKeyValues = Lists.newArrayList(); + for (int i = 0; i < rowkey.length; ++i) { + String colName = rowkey[i]; + int rowKeyIndex = 0; //rowkey index + for (; rowKeyIndex < columnNames.length; ++rowKeyIndex) { + if (columnNames[rowKeyIndex].equals(colName)) { + break; + } + } + + if (rowKeyIndex != columnNames.length && record.getField(rowKeyIndex) != null) { + Object field = record.getField(rowKeyIndex); + if (field == null) { + continue; + } else if (field instanceof java.util.Date) { + java.util.Date d = (java.util.Date) field; + rowKeyValues.add(ROWKEY_DATE_FORMAT.format(d)); + } else { + rowKeyValues.add(field.toString()); + } + } + } + return rowKeyValues; + } + @Override public void close() throws IOException { if(conn != null) { @@ -196,6 +219,11 @@ public HbaseOutputFormatBuilder setColumnTypes(String[] columnTypes) { return this; } + public HbaseOutputFormatBuilder setColumnNameFamily(Map columnNameFamily) { + format.columnNameFamily = columnNameFamily; + return this; + } + public HbaseOutputFormat finish() { Preconditions.checkNotNull(format.host, "zookeeperQuorum should be specified"); Preconditions.checkNotNull(format.tableName, "tableName should be specified"); @@ -205,13 +233,16 @@ public HbaseOutputFormat finish() { String[] families = new String[format.columnNames.length]; String[] qualifiers = new String[format.columnNames.length]; - for(int i = 0; i < format.columnNames.length; ++i) { - String col = format.columnNames[i]; - String[] part = col.split(":"); - families[i] = part[0]; - qualifiers[i] = part[1]; + if (format.columnNameFamily != null) { + Set keySet = format.columnNameFamily.keySet(); + String[] columns = keySet.toArray(new String[keySet.size()]); + for (int i = 0; i < columns.length; ++i) { + String col = columns[i]; + String[] part = col.split(":"); + families[i] = part[0]; + qualifiers[i] = part[1]; + } } - format.families = families; format.qualifiers = qualifiers; diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index b5a97f335..b448643ec 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -36,6 +36,8 @@ import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; +import java.util.Map; + /** * Date: 2018/09/14 * Company: www.dtstack.com @@ -44,12 +46,16 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener { protected String[] fieldNames; + protected Map columnNameFamily; TypeInformation[] fieldTypes; protected String zookeeperQuorum; protected String port; protected String parent; protected String tableName; protected String[] rowkey; + protected String registerTabName; + + private int parallelism = -1; public HbaseSink() { // TO DO NOTHING @@ -63,26 +69,19 @@ public HbaseSink genStreamSink(TargetTableInfo targetTableInfo) { this.parent = hbaseTableInfo.getParent(); this.tableName = hbaseTableInfo.getTableName(); this.rowkey = hbaseTableInfo.getRowkey(); + this.columnNameFamily = hbaseTableInfo.getColumnNameFamily(); + this.registerTabName = hbaseTableInfo.getName(); + + Integer tmpSinkParallelism = hbaseTableInfo.getParallelism(); + if (tmpSinkParallelism != null) { + setParallelism(tmpSinkParallelism); + } return this; } @Override public void emitDataStream(DataStream> dataStream) { - consumeDataStream(dataStream); - } - - @Override - public DataStreamSink> consumeDataStream(DataStream> dataStream) { - HbaseOutputFormat.HbaseOutputFormatBuilder builder = HbaseOutputFormat.buildHbaseOutputFormat(); - builder.setHost(this.zookeeperQuorum).setZkParent(this.parent).setTable(this.tableName); - builder.setRowkey(rowkey); - builder.setColumnNames(fieldNames); - - HbaseOutputFormat outputFormat = builder.finish(); - RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); - DataStreamSink dataStreamSink = dataStream.addSink(richSinkFunction); - return dataStreamSink; } @Override @@ -111,4 +110,27 @@ public String[] getFieldNames() { public TypeInformation[] getFieldTypes() { return fieldTypes; } + + @Override + public DataStreamSink> consumeDataStream(DataStream> dataStream) { + HbaseOutputFormat.HbaseOutputFormatBuilder builder = HbaseOutputFormat.buildHbaseOutputFormat(); + builder.setHost(this.zookeeperQuorum).setZkParent(this.parent).setTable(this.tableName); + + builder.setRowkey(rowkey); + builder.setColumnNames(fieldNames); + builder.setColumnNameFamily(columnNameFamily); + + HbaseOutputFormat outputFormat = builder.finish(); + RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); + DataStreamSink streamSink = dataStream.addSink(richSinkFunction); + streamSink.name(registerTabName); + if (parallelism > 0) { + streamSink.setParallelism(parallelism); + } + return streamSink; + } + + public void setParallelism(int parallelism) { + this.parallelism = parallelism; + } } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index d5a3fd4b7..1afdbd481 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -23,8 +23,11 @@ import com.dtstack.flink.sql.table.AbsTableParser; import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.MathUtil; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; @@ -65,4 +68,38 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); + Map columnFamilies = new LinkedHashMap<>(); + for(String fieldRow : fieldRows){ + fieldRow = fieldRow.trim(); + + String[] filedInfoArr = fieldRow.split("\\s+"); + if(filedInfoArr.length < 2 ){ + throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); + } + + boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); + if(isMatcherKey){ + continue; + } + + //Compatible situation may arise in space in the fieldName + String[] filedNameArr = new String[filedInfoArr.length - 1]; + System.arraycopy(filedInfoArr, 0, filedNameArr, 0, filedInfoArr.length - 1); + String fieldName = String.join(" ", filedNameArr); + String fieldType = filedInfoArr[filedInfoArr.length - 1 ].trim(); + Class fieldClass = dbTypeConvertToJavaType(fieldType); + String[] columnFamily = fieldName.trim().split(":"); + columnFamilies.put(fieldName.trim(),columnFamily[1]); + tableInfo.addPhysicalMappings(filedInfoArr[0],filedInfoArr[0]); + tableInfo.addField(columnFamily[1]); + tableInfo.addFieldClass(fieldClass); + tableInfo.addFieldType(fieldType); + tableInfo.addFieldExtraInfo(null); + } + tableInfo.setColumnNameFamily(columnFamilies); + tableInfo.finish(); + } } From 988e99a86f8f97d32f69641c472a41483ea9a52a Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 25 Dec 2019 10:56:10 +0800 Subject: [PATCH 07/69] async side kerberos --- docs/hbaseSide.md | 16 +- .../flink/sql/side/hbase/HbaseAllReqRow.java | 13 +- .../sql/side/hbase/HbaseAsyncReqRow.java | 9 +- .../sql/side/hbase/table/HbaseSideParser.java | 4 +- .../side/hbase/table/HbaseSideTableInfo.java | 5 +- .../side/hbase/utils/HbaseConfigUtils.java | 5 +- .../sql/sink/hbase/HbaseOutputFormat.java | 68 ++++++-- .../flink/sql/sink/hbase/HbaseSink.java | 5 +- .../sql/sink/hbase/table/HbaseSinkParser.java | 6 + .../sql/sink/hbase/table/HbaseTableInfo.java | 11 ++ .../sink/hbase/utils/HbaseConfigUtils.java | 153 ++++++++++++++++++ .../sql/sink/hbase/{ => utils}/HbaseUtil.java | 2 +- 12 files changed, 268 insertions(+), 29 deletions(-) create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java rename hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/{ => utils}/HbaseUtil.java (97%) diff --git a/docs/hbaseSide.md b/docs/hbaseSide.md index 0c4e545f9..729581643 100644 --- a/docs/hbaseSide.md +++ b/docs/hbaseSide.md @@ -48,7 +48,21 @@ * LRU: * cacheSize: 缓存的条目数量 * cacheTTLMs:缓存的过期时间(ms) - +> kerberos 配置 + * ALL模式 + * hbase.security.authentication = 'kerberos', + * hbase.security.authorization = 'true', + * hbase.master.kerberos.principal = 'hbase/cdh01@DTSTACK.COM', + * hbase.master.keytab.file = '/Users/hbase.keytab', + * hbase.regionserver.keytab.file = '/Users/hbase.keytab', + * hbase.regionserver.kerberos.principal = 'hbase/cdh01@DTSTACK.COM' + * (非必选)java.security.krb5.conf = '/etc/krb5.conf' + * LRU/NONE模式 + * hbase.security.auth.enable= 'true', + * hbase.security.authentication='kerberos', + * hbase.sasl.clientconfig = 'Client', + * hbase.kerberos.regionserver.principal='hbase/_HOST@TDH', + * (非必选)java.security.krb5.conf = '/etc/krb5.conf' ## 4.样例 ``` CREATE TABLE sideTable( diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index d8ceaf9fd..9b3e237c6 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -25,6 +25,7 @@ import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.map.HashedMap; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -170,13 +171,14 @@ private void loadData(Map> tmpCache) throws SQLExcep try { if (openKerberos) { conf = HbaseConfigUtils.getHadoopConfiguration(hbaseSideTableInfo.getHbaseConfig()); - conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, hbaseSideTableInfo.getParent()); String principal = HbaseConfigUtils.getPrincipal(hbaseSideTableInfo.getHbaseConfig()); String keytab = HbaseConfigUtils.getKeytab(hbaseSideTableInfo.getHbaseConfig()); UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, principal, keytab); Configuration finalConf = conf; - userGroupInformation.doAs(new PrivilegedAction() { + conn = userGroupInformation.doAs(new PrivilegedAction() { @Override public Connection run() { try { @@ -190,7 +192,8 @@ public Connection run() { } else { conf = HbaseConfigUtils.getConfig(hbaseSideTableInfo.getHbaseConfig()); - conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, hbaseSideTableInfo.getParent()); conn = ConnectionFactory.createConnection(conf); } @@ -211,14 +214,14 @@ public Connection run() { tmpCache.put(new String(r.getRow()), kv); } } catch (IOException e) { - e.printStackTrace(); + throw new RuntimeException(e); } finally { try { conn.close(); table.close(); resultScanner.close(); } catch (IOException e) { - e.printStackTrace(); + LOG.error("", e); } } } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index d0dd9837b..6cbf83c78 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -31,6 +31,7 @@ import com.dtstack.flink.sql.side.hbase.rowkeydealer.RowKeyEqualModeDealer; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import com.dtstack.flink.sql.factory.DTThreadFactory; +import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; import com.google.common.collect.Maps; import com.stumbleupon.async.Deferred; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -96,8 +97,12 @@ public void open(Configuration parameters) throws Exception { new LinkedBlockingQueue<>(), new DTThreadFactory("hbase-aysnc")); Config config = new Config(); - config.overrideConfig("hbase.zookeeper.quorum",hbaseSideTableInfo.getHost()); - config.overrideConfig("hbase.zookeeper.znode.parent",hbaseSideTableInfo.getParent()); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, hbaseSideTableInfo.getParent()); + HbaseConfigUtils.loadKrb5Conf(hbaseSideTableInfo.getHbaseConfig()); + hbaseSideTableInfo.getHbaseConfig().entrySet().forEach(entity -> { + config.overrideConfig(entity.getKey(), (String) entity.getValue()); + }); hBaseClient = new HBaseClient(config, executorService); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 4b0b615a2..2b785c335 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -78,8 +78,8 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map entity.getKey().startsWith("hbase")) - .map(entity -> hbaseTableInfo.getHbaseConfig().put(entity.getKey(), entity.getValue())) + .filter(entity -> entity.getKey().contains(".")) + .map(entity -> hbaseTableInfo.getHbaseConfig().put(entity.getKey(), String.valueOf(entity.getValue()))) .count(); return hbaseTableInfo; diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java index 070c51f08..505c3bb57 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java @@ -50,7 +50,7 @@ public class HbaseSideTableInfo extends SideTableInfo { private Map columnNameFamily; - private Map hbaseConfig = Maps.newHashMap(); + private Map hbaseConfig = Maps.newHashMap(); private String tableName; @@ -154,6 +154,9 @@ public Map getHbaseConfig() { return hbaseConfig; } + public void setHbaseConfig(Map hbaseConfig) { + this.hbaseConfig = hbaseConfig; + } @Override public void finish(){ diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index 1e2be0486..6504700f3 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -45,11 +45,14 @@ public class HbaseConfigUtils { private final static String AUTHENTICATION_TYPE = "Kerberos"; private final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; - private final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; + private final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; private final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; private final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; private final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; private final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; + public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; + public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; + private static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index b893892c0..b0aa10989 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -16,11 +16,11 @@ * limitations under the License. */ - package com.dtstack.flink.sql.sink.hbase; import com.dtstack.flink.sql.sink.MetricOutputFormat; +import com.dtstack.flink.sql.sink.hbase.utils.HbaseConfigUtils; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; @@ -33,9 +33,12 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.IOException; +import java.security.PrivilegedAction; import java.text.SimpleDateFormat; import java.util.List; import java.util.Map; @@ -55,7 +58,9 @@ public class HbaseOutputFormat extends MetricOutputFormat { private String tableName; private String[] columnNames; private String[] columnTypes; - private Map columnNameFamily; + private Map columnNameFamily; + private Map hbaseConfig; + private String[] families; private String[] qualifiers; @@ -73,29 +78,55 @@ public class HbaseOutputFormat extends MetricOutputFormat { @Override public void configure(Configuration parameters) { LOG.warn("---configure---"); - conf = HBaseConfiguration.create(); - conf.set("hbase.zookeeper.quorum", host); - if(zkParent != null && !"".equals(zkParent)){ - conf.set("zookeeper.znode.parent", zkParent); + boolean openKerberos = HbaseConfigUtils.openKerberos(hbaseConfig); + try { + if (openKerberos) { + conf = HbaseConfigUtils.getHadoopConfiguration(hbaseConfig); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); + String principal = HbaseConfigUtils.getPrincipal(hbaseConfig); + String keytab = HbaseConfigUtils.getKeytab(hbaseConfig); + + UserGroupInformation userGroupInformation = null; + + userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, principal, keytab); + org.apache.hadoop.conf.Configuration finalConf = conf; + conn = userGroupInformation.doAs(new PrivilegedAction() { + @Override + public Connection run() { + try { + return ConnectionFactory.createConnection(finalConf); + } catch (IOException e) { + LOG.error("Get connection fail with config:{}", finalConf); + throw new RuntimeException(e); + } + } + }); + } else { + conf = HbaseConfigUtils.getConfig(hbaseConfig); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); + conn = ConnectionFactory.createConnection(conf); + } + } catch (IOException e) { + LOG.error("", e); } - LOG.warn("---configure end ---"); } @Override public void open(int taskNumber, int numTasks) throws IOException { LOG.warn("---open---"); - conn = ConnectionFactory.createConnection(conf); table = conn.getTable(TableName.valueOf(tableName)); LOG.warn("---open end(get table from hbase) ---"); initMetric(); } @Override - public void writeRecord(Tuple2 tuple2) { + public void writeRecord(Tuple2 tuple2) { Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); - if(!retract){ + if (!retract) { //FIXME 暂时不处理hbase删除操作--->hbase要求有key,所有认为都是可以执行update查找 return; } @@ -103,7 +134,7 @@ public void writeRecord(Tuple2 tuple2) { Row record = tupleTrans.getField(1); List rowKeyValues = getRowKeyValues(record); // all rowkey not null - if (rowKeyValues.size() != rowkey.length ) { + if (rowKeyValues.size() != rowkey.length) { LOG.error("row key value must not null,record is ..", record); outDirtyRecords.inc(); return; @@ -111,7 +142,7 @@ public void writeRecord(Tuple2 tuple2) { String key = StringUtils.join(rowKeyValues, "-"); Put put = new Put(key.getBytes()); - for(int i = 0; i < record.getArity(); ++i) { + for (int i = 0; i < record.getArity(); ++i) { Object fieldVal = record.getField(i); if (fieldVal == null) { continue; @@ -168,13 +199,14 @@ private List getRowKeyValues(Row record) { @Override public void close() throws IOException { - if(conn != null) { + if (conn != null) { conn.close(); conn = null; } } - private HbaseOutputFormat() {} + private HbaseOutputFormat() { + } public static HbaseOutputFormatBuilder buildHbaseOutputFormat() { return new HbaseOutputFormatBuilder(); @@ -193,7 +225,7 @@ public HbaseOutputFormatBuilder setHost(String host) { return this; } - public HbaseOutputFormatBuilder setZkParent(String parent){ + public HbaseOutputFormatBuilder setZkParent(String parent) { format.zkParent = parent; return this; } @@ -224,6 +256,12 @@ public HbaseOutputFormatBuilder setColumnNameFamily(Map columnNa return this; } + public HbaseOutputFormatBuilder setHbaseConfig(Map hbaseConfig) { + format.hbaseConfig = hbaseConfig; + return this; + } + + public HbaseOutputFormat finish() { Preconditions.checkNotNull(format.host, "zookeeperQuorum should be specified"); Preconditions.checkNotNull(format.tableName, "tableName should be specified"); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index b448643ec..c918c60af 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -54,6 +54,7 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener< protected String tableName; protected String[] rowkey; protected String registerTabName; + protected Map hbaseConfig; private int parallelism = -1; @@ -71,6 +72,7 @@ public HbaseSink genStreamSink(TargetTableInfo targetTableInfo) { this.rowkey = hbaseTableInfo.getRowkey(); this.columnNameFamily = hbaseTableInfo.getColumnNameFamily(); this.registerTabName = hbaseTableInfo.getName(); + this.hbaseConfig = hbaseTableInfo.getHbaseConfig(); Integer tmpSinkParallelism = hbaseTableInfo.getParallelism(); if (tmpSinkParallelism != null) { @@ -81,7 +83,7 @@ public HbaseSink genStreamSink(TargetTableInfo targetTableInfo) { @Override public void emitDataStream(DataStream> dataStream) { - + consumeDataStream(dataStream); } @Override @@ -119,6 +121,7 @@ public DataStreamSink> consumeDataStream(DataStream entity.getKey().contains(".")) + .map(entity -> hbaseTableInfo.getHbaseConfig().put(entity.getKey(), entity.getValue())) + .count(); + return hbaseTableInfo; } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java index ed28f781d..9f5104e73 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java @@ -23,6 +23,8 @@ import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; + import java.util.Map; /** @@ -52,6 +54,8 @@ public class HbaseTableInfo extends TargetTableInfo { private String tableName; + private Map hbaseConfig = Maps.newHashMap(); + public HbaseTableInfo(){ setType(CURR_TYPE); } @@ -139,4 +143,11 @@ public String getType() { return super.getType().toLowerCase(); } + public Map getHbaseConfig() { + return hbaseConfig; + } + + public void setHbaseConfig(Map hbaseConfig) { + this.hbaseConfig = hbaseConfig; + } } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java new file mode 100644 index 000000000..a463bb686 --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.hbase.utils; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * + * The utility class of HBase connection + * + * Date: 2019/12/24 + * Company: www.dtstack.com + * @author maqi + */ +public class HbaseConfigUtils { + + private static final Logger LOG = LoggerFactory.getLogger(HbaseConfigUtils.class); + + private final static String AUTHENTICATION_TYPE = "Kerberos"; + private final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; + private final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; + private final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; + private final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; + private final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; + private final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; + + public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; + public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; + + private static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + + private static List KEYS_KERBEROS_REQUIRED = Arrays.asList( + KEY_HBASE_SECURITY_AUTHENTICATION, + KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, + KEY_HBASE_MASTER_KEYTAB_FILE, + KEY_HBASE_REGIONSERVER_KEYTAB_FILE, + KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL + ); + + + public static Configuration getConfig(Map hbaseConfigMap) { + Configuration hConfiguration = HBaseConfiguration.create(); + + for (Map.Entry entry : hbaseConfigMap.entrySet()) { + if (entry.getValue() != null && !(entry.getValue() instanceof Map)) { + hConfiguration.set(entry.getKey(), entry.getValue().toString()); + } + } + return hConfiguration; + } + + public static boolean openKerberos(Map hbaseConfigMap) { + if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHORIZATION)) { + return false; + } + return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); + } + + public static Configuration getHadoopConfiguration(Map hbaseConfigMap) { + for (String key : KEYS_KERBEROS_REQUIRED) { + if (StringUtils.isEmpty(MapUtils.getString(hbaseConfigMap, key))) { + throw new IllegalArgumentException(String.format("Must provide [%s] when authentication is Kerberos", key)); + } + } + loadKrb5Conf(hbaseConfigMap); + + Configuration conf = new Configuration(); + if (hbaseConfigMap == null) { + return conf; + } + + hbaseConfigMap.forEach((key, val) -> { + if (val != null) { + conf.set(key, val.toString()); + } + }); + + return conf; + } + + public static String getPrincipal(Map hbaseConfigMap) { + String principal = MapUtils.getString(hbaseConfigMap, KEY_HBASE_MASTER_KERBEROS_PRINCIPAL); + if (StringUtils.isNotEmpty(principal)) { + return principal; + } + + throw new IllegalArgumentException(""); + } + + public static String getKeytab(Map hbaseConfigMap) { + String keytab = MapUtils.getString(hbaseConfigMap, KEY_HBASE_MASTER_KEYTAB_FILE); + if (StringUtils.isNotEmpty(keytab)) { + return keytab; + } + + throw new IllegalArgumentException(""); + } + + public static void loadKrb5Conf(Map kerberosConfig) { + String krb5FilePath = MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); + if (org.apache.commons.lang.StringUtils.isEmpty(krb5FilePath)) { + return; + } + kerberosConfig.put(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); + } + + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { + if (conf == null) { + throw new IllegalArgumentException("kerberos conf can not be null"); + } + + if (org.apache.commons.lang.StringUtils.isEmpty(principal)) { + throw new IllegalArgumentException("principal can not be null"); + } + + if (org.apache.commons.lang.StringUtils.isEmpty(keytab)) { + throw new IllegalArgumentException("keytab can not be null"); + } + + conf.set("hadoop.security.authentication", "Kerberos"); + UserGroupInformation.setConfiguration(conf); + + return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); + } +} diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseUtil.java similarity index 97% rename from hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java rename to hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseUtil.java index 8f1166574..8294b646c 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseUtil.java @@ -18,7 +18,7 @@ -package com.dtstack.flink.sql.sink.hbase; +package com.dtstack.flink.sql.sink.hbase.utils; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; From 9fa81b600cc6c47fc106f95ead19112fb1243314 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 25 Dec 2019 11:00:43 +0800 Subject: [PATCH 08/69] modify hbasesink doc --- docs/hbaseSink.md | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/docs/hbaseSink.md b/docs/hbaseSink.md index c6a15e766..3926101ed 100644 --- a/docs/hbaseSink.md +++ b/docs/hbaseSink.md @@ -37,19 +37,39 @@ hbase2.0 |rowKey | hbase的rowkey关联的列信息|是|| |parallelism | 并行度设置|否|1| - +> kerberos 配置 + + * hbase.security.authentication = 'kerberos', + * hbase.security.authorization = 'true', + * hbase.master.kerberos.principal = 'hbase/cdh01@DTSTACK.COM', + * hbase.master.keytab.file = '/Users/hbase.keytab', + * hbase.regionserver.keytab.file = '/Users/hbase.keytab', + * hbase.regionserver.kerberos.principal = 'hbase/cdh01@DTSTACK.COM' + * (非必选)java.security.krb5.conf = '/etc/krb5.conf' ## 5.样例： ``` -CREATE TABLE MyResult( - cf:channel varchar, - cf:pv BIGINT + + CREATE TABLE MyResult( + cf:info VARCHAR, + cf:name VARCHAR, + cf:channel varchar )WITH( type ='hbase', - zookeeperQuorum ='rdos1:2181', - tableName ='workerinfo', - rowKey ='cf:channel', - parallelism ='1', - zookeeperParent ='/hbase' - ) + zookeeperQuorum ='172.16.10.104:2181,172.16.10.224:2181,172.16.10.252:2181', + zookeeperParent ='/hbase', + tableName ='workerinfo01', + rowKey ='channel' + + ---hbase.security.authentication = 'kerberos', + ---hbase.security.authorization = 'true', + ---hbase.master.kerberos.principal = 'hbase/cdh01@DTSTACK.COM', + ---hbase.master.keytab.file = '/Users/chuixue/Desktop/hbase.keytab', + ---hbase.regionserver.keytab.file = '/Users/chuixue/Desktop/hbase.keytab', + ---hbase.regionserver.kerberos.principal = 'hbase/cdh01@DTSTACK.COM' + + + + ); + ``` \ No newline at end of file From de3067337c504b8fadae69852575217e14e23cca Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 25 Dec 2019 13:03:01 +0800 Subject: [PATCH 09/69] fix hbase async time convert --- .../rowkeydealer/AbsRowKeyModeDealer.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java index d73b31e75..4a2374cbb 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java @@ -22,13 +22,16 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.hbase.HbaseAsyncReqRow; import org.apache.calcite.sql.JoinType; import com.google.common.collect.Maps; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.hbase.async.HBaseClient; import java.sql.Timestamp; +import java.time.LocalDateTime; import java.util.Collections; import java.util.List; import java.util.Map; @@ -86,11 +89,10 @@ protected Row fillData(Row input, Object sideInput){ List sideInputList = (List) sideInput; Row row = new Row(outFieldInfoList.size()); + for(Map.Entry entry : inFieldIndex.entrySet()){ Object obj = input.getField(entry.getValue()); - if(obj instanceof Timestamp){ - obj = ((Timestamp)obj).getTime(); - } + obj = convertTimeIndictorTypeInfo(entry.getValue(), obj); row.setField(entry.getKey(), obj); } @@ -105,6 +107,16 @@ protected Row fillData(Row input, Object sideInput){ return row; } + protected Object convertTimeIndictorTypeInfo(Integer index, Object obj) { + boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(outFieldInfoList.get(index).getTypeInformation().getClass()); + + //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. + if (obj instanceof LocalDateTime && isTimeIndicatorTypeInfo) { + obj = Timestamp.valueOf(((LocalDateTime) obj)); + } + return obj; + } + public abstract void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFuture resultFuture, AbsSideCache sideCache); } From d7a5e3b13c616a24c66161bfffc4424b97fa09e7 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 25 Dec 2019 14:39:59 +0800 Subject: [PATCH 10/69] hbase all side close not null check --- .../flink/sql/side/hbase/HbaseAllReqRow.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 9b3e237c6..6ff2b062c 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -217,9 +217,17 @@ public Connection run() { throw new RuntimeException(e); } finally { try { - conn.close(); - table.close(); - resultScanner.close(); + if (null != conn) { + conn.close(); + } + + if (null != table) { + table.close(); + } + + if (null != resultScanner) { + resultScanner.close(); + } } catch (IOException e) { LOG.error("", e); } From 811ab8fc2f3cf4428bcd9fcc70f2911a8a827417 Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Wed, 25 Dec 2019 16:36:51 +0800 Subject: [PATCH 11/69] =?UTF-8?q?=E8=A7=A3=E5=86=B3kafka=20source=20?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E5=AD=97=E6=AE=B5=E5=80=BC=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E4=B8=BAjson=E6=97=B6null=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/source/kafka/CustomerJsonDeserialization.java | 9 +++++---- .../sql/source/kafka/CustomerJsonDeserialization.java | 9 +++++---- .../sql/source/kafka/CustomerJsonDeserialization.java | 9 +++++---- .../sql/source/kafka/CustomerJsonDeserialization.java | 9 +++++---- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index 25369d483..bb8a17ca9 100644 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -32,6 +32,7 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.JsonNodeType; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.TextNode; import org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread; import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; @@ -257,11 +258,11 @@ private Object convert(JsonNode node, TypeInformation info) { if (info.getTypeClass().equals(Types.BOOLEAN.getTypeClass())) { return node.asBoolean(); } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { - String value = node.asText(); - if ("".equals(value)){ - value = node.toString(); + if (node instanceof ObjectNode) { + return node.toString(); + } else { + return node.asText(); } - return value; } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { return Date.valueOf(node.asText()); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index 70ebd3151..fe1604891 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -34,6 +34,7 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread; import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; import org.apache.flink.types.Row; @@ -260,11 +261,11 @@ private Object convert(JsonNode node, TypeInformation info) { if (info.getTypeClass().equals(Types.BOOLEAN.getTypeClass())) { return node.asBoolean(); } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { - String value = node.asText(); - if ("".equals(value)){ - value = node.toString(); + if (node instanceof ObjectNode) { + return node.toString(); + } else { + return node.asText(); } - return value; } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { return Date.valueOf(node.asText()); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index b041fb7b5..9c97276e7 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -34,6 +34,7 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread; import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; import org.apache.flink.types.Row; @@ -265,11 +266,11 @@ private Object convert(JsonNode node, TypeInformation info) { if (info.getTypeClass().equals(Types.BOOLEAN.getTypeClass())) { return node.asBoolean(); } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { - String value = node.asText(); - if ("".equals(value)){ - value = node.toString(); + if (node instanceof ObjectNode) { + return node.toString(); + } else { + return node.asText(); } - return value; } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { return Date.valueOf(node.asText()); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index d829c0ff0..3a857075e 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -34,6 +34,7 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread; import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; import org.apache.flink.types.Row; @@ -265,11 +266,11 @@ private Object convert(JsonNode node, TypeInformation info) { if (info.getTypeClass().equals(Types.BOOLEAN.getTypeClass())) { return node.asBoolean(); } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { - String value = node.asText(); - if ("".equals(value)){ - value = node.toString(); + if (node instanceof ObjectNode) { + return node.toString(); + } else { + return node.asText(); } - return value; } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { return Date.valueOf(node.asText()); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { From 9770f2181a0e31565400c709e18a71c8b1b8fe6c Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 25 Dec 2019 17:13:13 +0800 Subject: [PATCH 12/69] build jaas --- docs/hbaseSide.md | 2 + .../sql/side/hbase/HbaseAsyncReqRow.java | 13 +++- .../side/hbase/utils/HbaseConfigUtils.java | 76 ++++++++++++++++++- 3 files changed, 85 insertions(+), 6 deletions(-) diff --git a/docs/hbaseSide.md b/docs/hbaseSide.md index 729581643..7d0fa7f7d 100644 --- a/docs/hbaseSide.md +++ b/docs/hbaseSide.md @@ -62,6 +62,8 @@ * hbase.security.authentication='kerberos', * hbase.sasl.clientconfig = 'Client', * hbase.kerberos.regionserver.principal='hbase/_HOST@TDH', + * hbase.keytab = '/Users/chuixue/Desktop/hbase.keytab', + * hbase.principal = 'hbase/cdh01@DTSTACK.COM' * (非必选)java.security.krb5.conf = '/etc/krb5.conf' ## 4.样例 ``` diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 6cbf83c78..ee13e9f66 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -92,6 +92,8 @@ public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List hbaseConfig = hbaseSideTableInfo.getHbaseConfig(); + ExecutorService executorService =new ThreadPoolExecutor(DEFAULT_POOL_SIZE, DEFAULT_POOL_SIZE, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new DTThreadFactory("hbase-aysnc")); @@ -99,11 +101,18 @@ public void open(Configuration parameters) throws Exception { Config config = new Config(); config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, hbaseSideTableInfo.getParent()); - HbaseConfigUtils.loadKrb5Conf(hbaseSideTableInfo.getHbaseConfig()); - hbaseSideTableInfo.getHbaseConfig().entrySet().forEach(entity -> { + HbaseConfigUtils.loadKrb5Conf(hbaseConfig); + hbaseConfig.entrySet().forEach(entity -> { config.overrideConfig(entity.getKey(), (String) entity.getValue()); }); + if (HbaseConfigUtils.asyncOpenKerberos(hbaseConfig)) { + String jaasStr = HbaseConfigUtils.buildJaasStr(hbaseConfig); + String jaasFilePath = HbaseConfigUtils.creatJassFile(jaasStr); + config.overrideConfig(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); + } + + hBaseClient = new HBaseClient(config, executorService); try { diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index 6504700f3..c97cfeba2 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -26,10 +26,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.UUID; /** * @@ -42,7 +46,7 @@ public class HbaseConfigUtils { private static final Logger LOG = LoggerFactory.getLogger(HbaseConfigUtils.class); - + // sync side kerberos private final static String AUTHENTICATION_TYPE = "Kerberos"; private final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; private final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; @@ -50,11 +54,25 @@ public class HbaseConfigUtils { private final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; private final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; private final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; + + // async side kerberos + private final static String KEY_HBASE_SECURITY_AUTH_ENABLE = "hbase.security.auth.enable"; + private final static String KEY_HBASE_SASL_CLIENTCONFIG = "hbase.sasl.clientconfig"; + private final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; + private static final String KEY_KEY_TAB = "hbase.keytab"; + private static final String KEY_PRINCIPAL = "hbase.principal"; + public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; private static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + public static final String KEY_JAVA_SECURITY_AUTH_LOGIN_CONF = "java.security.auth.login.config"; + + + private static final String SP = File.separator; + private static final String KEY_KRB5_CONF = "krb5.conf"; + private static List KEYS_KERBEROS_REQUIRED = Arrays.asList( KEY_HBASE_SECURITY_AUTHENTICATION, @@ -64,6 +82,15 @@ public class HbaseConfigUtils { KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL ); + private static List ASYNC_KEYS_KERBEROS_REQUIRED = Arrays.asList( + KEY_HBASE_SECURITY_AUTH_ENABLE, + KEY_HBASE_SASL_CLIENTCONFIG, + KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL, + KEY_HBASE_SECURITY_AUTHENTICATION, + KEY_KEY_TAB, + KEY_PRINCIPAL + ); + public static Configuration getConfig(Map hbaseConfigMap) { Configuration hConfiguration = HBaseConfiguration.create(); @@ -83,6 +110,16 @@ public static boolean openKerberos(Map hbaseConfigMap) { return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); } + public static boolean asyncOpenKerberos(Map hbaseConfigMap) { + if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTH_ENABLE)) { + return false; + } + return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); + } + + + + public static Configuration getHadoopConfiguration(Map hbaseConfigMap) { for (String key : KEYS_KERBEROS_REQUIRED) { if (StringUtils.isEmpty(MapUtils.getString(hbaseConfigMap, key))) { @@ -125,12 +162,43 @@ public static String getKeytab(Map hbaseConfigMap) { public static void loadKrb5Conf(Map kerberosConfig) { String krb5FilePath = MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); - if (org.apache.commons.lang.StringUtils.isEmpty(krb5FilePath)) { - return; + if (!org.apache.commons.lang.StringUtils.isEmpty(krb5FilePath)) { + System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath);; } - kerberosConfig.put(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); } + public static String creatJassFile(String configStr) throws IOException { + String fileName = System.getProperty("user.dir"); + File krbConf = new File(fileName); + File temp = File.createTempFile("", ".conf", krbConf); + temp.deleteOnExit(); + BufferedWriter out = new BufferedWriter(new FileWriter(temp, false)); + out.write(configStr + "\n"); + out.close(); + return temp.getAbsolutePath(); + } + + public static String buildJaasStr(Map kerberosConfig) { + for (String key : ASYNC_KEYS_KERBEROS_REQUIRED) { + if (StringUtils.isEmpty(MapUtils.getString(kerberosConfig, key))) { + throw new IllegalArgumentException(String.format("Must provide [%s] when authentication is Kerberos", key)); + } + } + + String keyTab = MapUtils.getString(kerberosConfig, KEY_KEY_TAB); + String principal = MapUtils.getString(kerberosConfig, KEY_PRINCIPAL); + + StringBuilder jaasSB = new StringBuilder("Client {\n" + + " com.sun.security.auth.module.Krb5LoginModule required\n" + + " useKeyTab=true\n" + + " useTicketCache=false\n"); + jaasSB.append(" keyTab=\"").append(keyTab).append("\"").append("\n"); + jaasSB.append(" principal=\"").append(principal).append("\"").append(";\n"); + jaasSB.append("};"); + return jaasSB.toString(); + } + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { if (conf == null) { From 72cf8abab04c25bd4a841c279ec0c5fa4a2bc7ca Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Wed, 25 Dec 2019 18:26:47 +0800 Subject: [PATCH 13/69] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=A1=A8=E6=9C=80?= =?UTF-8?q?=E5=90=8E=E5=AD=97=E6=AE=B5=E4=B8=BA=E7=A9=BA=E4=B8=B2=E6=97=B6?= =?UTF-8?q?console=E6=95=B0=E7=BB=84=E4=B8=8B=E6=A0=87=E8=B6=8A=E7=95=8C?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/sink/console/ConsoleOutputFormat.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java index 7658e9979..55325a8c5 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java @@ -69,7 +69,11 @@ public void writeRecord(Tuple2 tuple2) throws IOException { List data = new ArrayList<>(); data.add(fieldNames); - data.add(record.toString().split(",")); + String[] recordStr = new String[record.getArity()]; + for (int i=0; i < record.getArity(); i++) { + recordStr[i] = (String.valueOf(record.getField(i))); + } + data.add(recordStr); TablePrintUtil.build(data).print(); outRecords.inc(); From 5d24c785ac41f87bf1fb6877bc97af274746189c Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 25 Dec 2019 20:11:29 +0800 Subject: [PATCH 14/69] null check --- .../dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 4 +++- .../flink/sql/side/hbase/utils/HbaseConfigUtils.java | 8 +++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index ee13e9f66..230479601 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -204,7 +204,9 @@ public Row fillData(Row input, Object sideInput){ @Override public void close() throws Exception { super.close(); - hBaseClient.shutdown(); + if (null!=hBaseClient) { + hBaseClient.shutdown(); + } } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index c97cfeba2..3ff1fa3cb 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -87,9 +87,7 @@ public class HbaseConfigUtils { KEY_HBASE_SASL_CLIENTCONFIG, KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL, KEY_HBASE_SECURITY_AUTHENTICATION, - KEY_KEY_TAB, - KEY_PRINCIPAL - ); + KEY_KEY_TAB); public static Configuration getConfig(Map hbaseConfigMap) { @@ -170,7 +168,7 @@ public static void loadKrb5Conf(Map kerberosConfig) { public static String creatJassFile(String configStr) throws IOException { String fileName = System.getProperty("user.dir"); File krbConf = new File(fileName); - File temp = File.createTempFile("", ".conf", krbConf); + File temp = File.createTempFile("JAAS", ".conf", krbConf); temp.deleteOnExit(); BufferedWriter out = new BufferedWriter(new FileWriter(temp, false)); out.write(configStr + "\n"); @@ -186,7 +184,7 @@ public static String buildJaasStr(Map kerberosConfig) { } String keyTab = MapUtils.getString(kerberosConfig, KEY_KEY_TAB); - String principal = MapUtils.getString(kerberosConfig, KEY_PRINCIPAL); + String principal = MapUtils.getString(kerberosConfig, KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); StringBuilder jaasSB = new StringBuilder("Client {\n" + " com.sun.security.auth.module.Krb5LoginModule required\n" + From 848ebc18fd010ac5c6db055d05bcbc17b01db706 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 25 Dec 2019 21:31:49 +0800 Subject: [PATCH 15/69] add hbase.principal --- .../dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index 3ff1fa3cb..a7708aaae 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -184,7 +184,7 @@ public static String buildJaasStr(Map kerberosConfig) { } String keyTab = MapUtils.getString(kerberosConfig, KEY_KEY_TAB); - String principal = MapUtils.getString(kerberosConfig, KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); + String principal = MapUtils.getString(kerberosConfig, KEY_PRINCIPAL); StringBuilder jaasSB = new StringBuilder("Client {\n" + " com.sun.security.auth.module.Krb5LoginModule required\n" + From 550e1e11f3f3985a6c6febe577a3dea174d10a43 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 26 Dec 2019 16:57:24 +0800 Subject: [PATCH 16/69] exist check --- .../dtstack/flink/sql/sink/rdb/format/ExtendOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/ExtendOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/ExtendOutputFormat.java index 6265bbadd..89732af8d 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/ExtendOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/ExtendOutputFormat.java @@ -98,7 +98,7 @@ public void fillFullColumns() throws SQLException { ResultSet rs = getDbConn().getMetaData().getColumns(null, getSchema(), getTableName(), null); while (rs.next()) { String columnName = rs.getString("COLUMN_NAME"); - if (StringUtils.isNotBlank(columnName)) { + if (StringUtils.isNotBlank(columnName) && !getFullField().contains(columnName)) { fullFieldAdd(columnName); } } From 5d6ef5e305d002d9c053d98fdbd99ca7627a0e87 Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Thu, 26 Dec 2019 19:05:28 +0800 Subject: [PATCH 17/69] =?UTF-8?q?=E8=A7=A3=E5=86=B3kafka=E8=A7=A3=E6=9E=90?= =?UTF-8?q?null=20->=20'null'=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/source/kafka/CustomerJsonDeserialization.java | 7 +++---- .../sql/source/kafka/CustomerJsonDeserialization.java | 3 +++ .../sql/source/kafka/CustomerJsonDeserialization.java | 3 +++ .../sql/source/kafka/CustomerJsonDeserialization.java | 3 +++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index bb8a17ca9..6a28bba2b 100644 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -30,10 +30,7 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.JsonNodeType; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.TextNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.*; import org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread; import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; import org.apache.flink.types.Row; @@ -260,6 +257,8 @@ private Object convert(JsonNode node, TypeInformation info) { } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { if (node instanceof ObjectNode) { return node.toString(); + } else if (node instanceof NullNode) { + return null; } else { return node.asText(); } diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index fe1604891..c00f335ee 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -34,6 +34,7 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.NullNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread; import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; @@ -263,6 +264,8 @@ private Object convert(JsonNode node, TypeInformation info) { } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { if (node instanceof ObjectNode) { return node.toString(); + } else if (node instanceof NullNode) { + return null; } else { return node.asText(); } diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index 9c97276e7..039fb7aec 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -34,6 +34,7 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.NullNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread; import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; @@ -268,6 +269,8 @@ private Object convert(JsonNode node, TypeInformation info) { } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { if (node instanceof ObjectNode) { return node.toString(); + } else if (node instanceof NullNode) { + return null; } else { return node.asText(); } diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index 3a857075e..a249c1cc9 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -34,6 +34,7 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.NullNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread; import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; @@ -268,6 +269,8 @@ private Object convert(JsonNode node, TypeInformation info) { } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { if (node instanceof ObjectNode) { return node.toString(); + } else if (node instanceof NullNode) { + return null; } else { return node.asText(); } From a15ea030c79b6c7704d74a37cd11937310be21db Mon Sep 17 00:00:00 2001 From: chuixue Date: Fri, 27 Dec 2019 17:11:02 +0800 Subject: [PATCH 18/69] readme update --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c2bf703ae..c51a74e97 100644 --- a/README.md +++ b/README.md @@ -64,9 +64,12 @@ mvn clean package -Dmaven.test.skip #### 1.4.1 启动命令 ``` -sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack/150_flinkplugin/sqlplugin -localSqlPluginPath D:\gitspace\flinkStreamSQL\plugins -addjar \["udf.jar\"\] -mode yarn -flinkconf D:\flink_home\kudu150etc -yarnconf D:\hadoop\etc\hadoopkudu -confProp \{\"time.characteristic\":\"EventTime\",\"sql.checkpoint.interval\":10000\} -yarnSessionConf \{\"yid\":\"application_1564971615273_38182\"} +sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack/150_flinkplugin/sqlplugin -localSqlPluginPath D:\gitspace\flinkStreamSQL\plugins -addjar ["udf.jar"] -mode yarn -flinkconf D:\flink_home\kudu150etc -yarnconf D:\hadoop\etc\hadoopkudu -confProp {\"time.characteristic\":\"EventTime\",\"sql.checkpoint.interval\":10000} -yarnSessionConf \{\"yid\":\"application_1564971615273_38182\"} + +注意：若在其他环境中无法使用 -addjar和 -confProp参数，可先将该参数对应的值进行urlencode编码，然后再传入。例如：-addjar %5B%22udf.jar%22%5D ``` + #### 1.4.2 命令行参数选项 * **mode** From f6ed244f33c3033eaf48044625b421431b955939 Mon Sep 17 00:00:00 2001 From: chuixue Date: Fri, 27 Dec 2019 17:23:08 +0800 Subject: [PATCH 19/69] readme update --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c51a74e97..34e51cfd6 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ * 9.支持udaf * 10.支持谓词下移 * 11.支持状态的ttl + * 12.支持hbase kerberos认证 ## BUG修复： * 1.修复不能解析sql中orderby,union语法。 From 03da4181a8dfd00b699542cb252c842608f4e76a Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Tue, 7 Jan 2020 13:17:46 +0800 Subject: [PATCH 20/69] =?UTF-8?q?=E5=BC=82=E6=AD=A5=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E9=98=BB=E5=A1=9Ebug=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/metric/MetricConstant.java | 2 + .../dtstack/flink/sql/side/AsyncReqRow.java | 28 ++++++++++++- .../sql/side/hbase/HbaseAsyncReqRow.java | 20 +++++++--- .../PreRowKeyModeDealerDealer.java | 34 +++++++++------- .../rowkeydealer/RowKeyEqualModeDealer.java | 30 ++++++++------ .../sql/side/rdb/async/RdbAsyncReqRow.java | 40 ++++++++++++------- 6 files changed, 105 insertions(+), 49 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/metric/MetricConstant.java b/core/src/main/java/com/dtstack/flink/sql/metric/MetricConstant.java index 17bb75e82..d78d691a8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/metric/MetricConstant.java +++ b/core/src/main/java/com/dtstack/flink/sql/metric/MetricConstant.java @@ -56,4 +56,6 @@ public class MetricConstant { public static final String DT_TOPIC_GROUP = "topic"; public static final String DT_PARTITION_GROUP = "partition"; + + public static final String DT_NUM_SIDE_PARSE_ERROR_RECORDS = "dtNumSideParseErrorRecords"; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index 74343f082..34b82c15e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -21,16 +21,20 @@ package com.dtstack.flink.sql.side; import com.dtstack.flink.sql.enums.ECacheType; +import com.dtstack.flink.sql.metric.MetricConstant; import com.dtstack.flink.sql.side.cache.AbsSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cache.LRUSideCache; import org.apache.calcite.sql.JoinType; import org.apache.flink.configuration.Configuration; +import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.sql.Timestamp; import java.time.LocalDateTime; @@ -48,10 +52,14 @@ public abstract class AsyncReqRow extends RichAsyncFunction implements ISideReqRow { + private static final Logger LOG = LoggerFactory.getLogger(AsyncReqRow.class); + private static final long serialVersionUID = 2098635244857937717L; protected SideInfo sideInfo; + protected transient Counter parseErrorRecords; + public AsyncReqRow(SideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -61,13 +69,17 @@ public void timeout(Row input, ResultFuture resultFuture) throws Exception StreamRecordQueueEntry future = (StreamRecordQueueEntry)resultFuture; try { if (null == future.get()) { - new TimeoutException("Async function call has timed out."); + resultFuture.completeExceptionally(new TimeoutException("Async function call has timed out.")); } } catch (Exception e) { - throw new Exception(e); + resultFuture.completeExceptionally(new Exception(e)); } } + private void initMetric() { + parseErrorRecords = getRuntimeContext().getMetricGroup().counter(MetricConstant.DT_NUM_SIDE_PARSE_ERROR_RECORDS); + } + private void initCache(){ SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); if(sideTableInfo.getCacheType() == null || ECacheType.NONE.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ @@ -118,10 +130,22 @@ protected void dealMissKey(Row input, ResultFuture resultFuture){ } } + protected void dealFillDataError(ResultFuture resultFuture, Exception e, Object sourceData, Object sideData) { + LOG.debug("source data {} join side table error ", sourceData.toString()); + LOG.debug("async buid row error..{}", e); + parseErrorRecords.inc(); + if (parseErrorRecords.getCount() % 1000 == 0 || parseErrorRecords.getCount() == 1) { + LOG.error("source error data {} ", sourceData.toString()); + LOG.error("side error data {} ", sideData.toString()); + } + resultFuture.complete(Collections.emptyList()); + } + @Override public void open(Configuration parameters) throws Exception { super.open(parameters); initCache(); + initMetric(); } @Override diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 230479601..378bc9d66 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -164,12 +164,22 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except dealMissKey(input, resultFuture); return; }else if(ECacheContentType.SingleLine == val.getType()){ - Row row = fillData(input, val); - resultFuture.complete(Collections.singleton(row)); - }else if(ECacheContentType.MultiLine == val.getType()){ - for(Object one : (List)val.getContent()){ - Row row = fillData(input, one); + try { + Row row = fillData(input, val); resultFuture.complete(Collections.singleton(row)); + } catch (Exception e) { + dealFillDataError(resultFuture, e, input, val); + } + }else if(ECacheContentType.MultiLine == val.getType()){ + Object debugData = null; + try { + for(Object one : (List)val.getContent()){ + debugData = one; + Row row = fillData(input, one); + resultFuture.complete(Collections.singleton(row)); + } + } catch (Exception e) { + dealFillDataError(resultFuture, e, input, debugData); } } return; diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index b75bca40c..89635d4e0 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -104,24 +104,28 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, sideMap.put(mapKey, val); } - if (oneRow.size() > 0) { - //The order of the fields defined in the data conversion table - List sideVal = Lists.newArrayList(); - for (String key : colNames) { - Object val = sideMap.get(key); - if (val == null) { - System.out.println("can't get data with column " + key); - LOG.error("can't get data with column " + key); + try { + if (oneRow.size() > 0) { + //The order of the fields defined in the data conversion table + List sideVal = Lists.newArrayList(); + for (String key : colNames) { + Object val = sideMap.get(key); + if (val == null) { + System.out.println("can't get data with column " + key); + LOG.error("can't get data with column " + key); + } + + sideVal.add(val); } - sideVal.add(val); - } - - Row row = fillData(input, sideVal); - if (openCache) { - cacheContent.add(sideVal); + Row row = fillData(input, sideVal); + if (openCache) { + cacheContent.add(sideVal); + } + rowList.add(row); } - rowList.add(row); + } catch (Exception e) { + resultFuture.completeExceptionally(e); } } catch (Exception e) { resultFuture.complete(null); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index 8604db18e..0cf9eb8c7 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -80,23 +80,27 @@ public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFu if(arg.size() > 0){ //The order of the fields defined in the data conversion table - List sideVal = Lists.newArrayList(); - for(String key : colNames){ - Object val = sideMap.get(key); - if(val == null){ - System.out.println("can't get data with column " + key); - LOG.error("can't get data with column " + key); + try { + List sideVal = Lists.newArrayList(); + for(String key : colNames){ + Object val = sideMap.get(key); + if(val == null){ + System.out.println("can't get data with column " + key); + LOG.error("can't get data with column " + key); + } + + sideVal.add(val); } - sideVal.add(val); - } + Row row = fillData(input, sideVal); + if(openCache){ + sideCache.putCache(rowKeyStr, CacheObj.buildCacheObj(ECacheContentType.SingleLine, row)); + } - Row row = fillData(input, sideVal); - if(openCache){ - sideCache.putCache(rowKeyStr, CacheObj.buildCacheObj(ECacheContentType.SingleLine, row)); + resultFuture.complete(Collections.singleton(row)); + } catch (Exception e) { + resultFuture.completeExceptionally(e); } - - resultFuture.complete(Collections.singleton(row)); }else{ dealMissKey(input, resultFuture); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 348a2a3b9..4754df650 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -93,11 +93,17 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except return; } else if (ECacheContentType.MultiLine == val.getType()) { List rowList = Lists.newArrayList(); - for (Object jsonArray : (List) val.getContent()) { - Row row = fillData(input, jsonArray); - rowList.add(row); + Object debugData = null; + try { + for (Object jsonArray : (List) val.getContent()) { + debugData = jsonArray; + Row row = fillData(input, jsonArray); + rowList.add(row); + } + resultFuture.complete(rowList); + } catch (Exception e) { + dealFillDataError(resultFuture, e, input, debugData); } - resultFuture.complete(rowList); } else { resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); } @@ -126,20 +132,24 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except int resultSize = rs.result().getResults().size(); if (resultSize > 0) { List rowList = Lists.newArrayList(); + Object debugData = null; + try { + for (JsonArray line : rs.result().getResults()) { + debugData = line; + Row row = fillData(input, line); + if (openCache()) { + cacheContent.add(line); + } + rowList.add(row); + } - for (JsonArray line : rs.result().getResults()) { - Row row = fillData(input, line); if (openCache()) { - cacheContent.add(line); + putCache(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); } - rowList.add(row); - } - - if (openCache()) { - putCache(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); + resultFuture.complete(rowList); + } catch (Exception e) { + dealFillDataError(resultFuture, e, input, debugData); } - - resultFuture.complete(rowList); } else { dealMissKey(input, resultFuture); if (openCache()) { @@ -157,6 +167,8 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except }); } + + @Override public Row fillData(Row input, Object line) { JsonArray jsonArray = (JsonArray) line; From d5d63f2d59e4a210fd21d7c5bff4ebe87508249f Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Wed, 8 Jan 2020 15:57:50 +0800 Subject: [PATCH 21/69] =?UTF-8?q?oracle=E8=BF=9E=E6=8E=A5=E6=95=B0?= =?UTF-8?q?=E8=BF=87=E5=A4=9A=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 4754df650..8c2ebf0c8 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -52,7 +52,7 @@ public class RdbAsyncReqRow extends AsyncReqRow { public final static int DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE = 1; - public final static int DEFAULT_VERTX_WORKER_POOL_SIZE = Runtime.getRuntime().availableProcessors() * 2; + public final static int DEFAULT_VERTX_WORKER_POOL_SIZE = 3; public final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; From 0090f3472a08cb4d7486166a1d927780d2a08e18 Mon Sep 17 00:00:00 2001 From: chuixue Date: Fri, 10 Jan 2020 14:23:35 +0800 Subject: [PATCH 22/69] TimeCharacteristic bug fix --- .../dtstack/flink/sql/environment/StreamEnvConfigManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index c0208484a..478feedac 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -191,7 +191,7 @@ public static Optional getStreamTimeCharacteristic(Propertie } String characteristicStr = properties.getProperty(ConfigConstrant.FLINK_TIME_CHARACTERISTIC_KEY); Optional characteristic = Arrays.stream(TimeCharacteristic.values()) - .filter(tc -> !characteristicStr.equalsIgnoreCase(tc.toString())).findAny(); + .filter(tc -> characteristicStr.equalsIgnoreCase(tc.toString())).findAny(); if (!characteristic.isPresent()) { throw new RuntimeException("illegal property :" + ConfigConstrant.FLINK_TIME_CHARACTERISTIC_KEY); From 13f1798c99f9bb3724052e21b6e6096764fb52e4 Mon Sep 17 00:00:00 2001 From: chuixue Date: Fri, 10 Jan 2020 14:42:55 +0800 Subject: [PATCH 23/69] async timeout bugfix --- .../com/dtstack/flink/sql/side/AsyncReqRow.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index 34b82c15e..607098a93 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -60,20 +60,21 @@ public abstract class AsyncReqRow extends RichAsyncFunction implements protected transient Counter parseErrorRecords; + private static int TIMEOUT_LOG_FLUSH_NUM = 10; + private int timeOutNum = 0; + public AsyncReqRow(SideInfo sideInfo){ this.sideInfo = sideInfo; } @Override public void timeout(Row input, ResultFuture resultFuture) throws Exception { - StreamRecordQueueEntry future = (StreamRecordQueueEntry)resultFuture; - try { - if (null == future.get()) { - resultFuture.completeExceptionally(new TimeoutException("Async function call has timed out.")); - } - } catch (Exception e) { - resultFuture.completeExceptionally(new Exception(e)); + if(timeOutNum % TIMEOUT_LOG_FLUSH_NUM == 0){ + LOG.warn("Async function call has timed out, since timeoutNum:{}. current: input:{}", timeOutNum, input.toString()); } + + timeOutNum++; + resultFuture.complete(null); } private void initMetric() { From b001388229547c26e986ccc286a6b7f6d73b7e31 Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Tue, 14 Jan 2020 10:19:52 +0800 Subject: [PATCH 24/69] =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E5=85=B3=E8=81=94error?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/side/AsyncReqRow.java | 5 +++-- .../com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index 607098a93..74356b9e6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -136,8 +136,9 @@ protected void dealFillDataError(ResultFuture resultFuture, Exception e, Ob LOG.debug("async buid row error..{}", e); parseErrorRecords.inc(); if (parseErrorRecords.getCount() % 1000 == 0 || parseErrorRecords.getCount() == 1) { - LOG.error("source error data {} ", sourceData.toString()); - LOG.error("side error data {} ", sideData.toString()); + LOG.error("source error data: {} ", sourceData.toString()); + LOG.error("side error data: {} ", sideData.toString()); + LOG.error("async buid row error: {}", e); } resultFuture.complete(Collections.emptyList()); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 8c2ebf0c8..3ffd0e81a 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -52,7 +52,7 @@ public class RdbAsyncReqRow extends AsyncReqRow { public final static int DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE = 1; - public final static int DEFAULT_VERTX_WORKER_POOL_SIZE = 3; + public final static int DEFAULT_VERTX_WORKER_POOL_SIZE = 2; public final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; From cc1989672cf23906b504f67ae7f394b997fc6ada Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Wed, 15 Jan 2020 14:31:25 +0800 Subject: [PATCH 25/69] =?UTF-8?q?oracle=20NUMBER=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E8=BD=ACdouble=E7=B1=BB=E5=9E=8B=E8=BD=AC=E6=8D=A2=E5=BC=82?= =?UTF-8?q?=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/rdb/util/MathUtil.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/MathUtil.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/MathUtil.java index d03b33045..b39c90cc4 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/MathUtil.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/MathUtil.java @@ -119,17 +119,19 @@ public static Float getFloatVal(Object obj, float defaultVal) { public static Double getDoubleVal(Object obj) { if (obj == null) { return null; - } - - if (obj instanceof String) { + } else if (obj instanceof String) { return Double.valueOf((String) obj); } else if (obj instanceof Float) { return (Double) obj; } else if (obj instanceof BigDecimal) { return ((BigDecimal) obj).doubleValue(); + } else if (obj instanceof Double) { + return (Double) obj; + } else if (obj instanceof BigInteger) { + return ((BigInteger) obj).doubleValue(); } - throw new RuntimeException("not support type of " + obj.getClass() + " convert to Double."); + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Double. value is: " + obj); } public static Double getDoubleVal(Object obj, double defaultVal) { @@ -228,7 +230,7 @@ public static Date getDate(Object obj) { try { return new Date(format.parse((String) obj).getTime()); } catch (ParseException e) { - throw new RuntimeException("String convert to Date fail."); + throw new RuntimeException("String convert to Date fail. value is: " + obj); } } else if (obj instanceof Timestamp) { return new Date(((Timestamp) obj).getTime()); From a92880c825971f2cf84731e4b665ab88771e95f4 Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 16 Jan 2020 15:16:24 +0800 Subject: [PATCH 26/69] =?UTF-8?q?oracle=20=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/oracle/OracleAsyncReqRow.java | 35 +++++++++++++++++++ .../flink/sql/side/rdb/all/RdbAllReqRow.java | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index f973ffee2..fa9b61d5e 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -24,6 +24,7 @@ import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.util.JDBCUtils; import io.vertx.core.Vertx; import io.vertx.core.VertxOptions; import io.vertx.core.json.JsonObject; @@ -33,6 +34,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; import java.util.List; @@ -51,6 +55,7 @@ public void open(Configuration parameters) throws Exception { super.open(parameters); JsonObject oracleClientConfig = new JsonObject(); RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + establishConnection(rdbSideTableInfo); oracleClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", ORACLE_DRIVER) .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) @@ -70,4 +75,34 @@ public void open(Configuration parameters) throws Exception { Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); } + + private void establishConnection(RdbSideTableInfo rdbSideTableInfo) { + Connection connection = null; + JDBCUtils.forName(ORACLE_DRIVER, getClass().getClassLoader()); + try { + if (rdbSideTableInfo.getUserName() == null) { + connection = DriverManager.getConnection(rdbSideTableInfo.getUrl()); + } else { + connection = DriverManager.getConnection(rdbSideTableInfo.getUrl(), rdbSideTableInfo.getUserName(), rdbSideTableInfo.getPassword()); + } + if (null != connection) { + if (connection.getMetaData().getTables(null, rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName(), null).next()) { + LOG.error("Table " + rdbSideTableInfo.getTableName() + " doesn't exist"); + throw new RuntimeException("Table " + rdbSideTableInfo.getTableName() + " doesn't exist"); + } + } + } catch (SQLException sqe) { + LOG.error("", sqe); + throw new IllegalArgumentException("open() failed.", sqe); + } finally { + if (null != connection) { + try { + connection.close(); + } catch (SQLException e) { + LOG.error("", e); + throw new IllegalArgumentException("close() failed.", e); + } + } + } + } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index 7cb2c47e4..3f12d90d6 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -220,7 +220,7 @@ private void loadData(Map>> tmpCache) throws SQ } public int getFetchSize() { - return 1000; + return sideInfo.getSideTableInfo().getCacheSize(); } } From ba4decba4c70abd705ec65bb256ab5413345e8f4 Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 16 Jan 2020 19:13:21 +0800 Subject: [PATCH 27/69] oracle async bug fix --- .../com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index fa9b61d5e..1c5c107c3 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -86,7 +86,7 @@ private void establishConnection(RdbSideTableInfo rdbSideTableInfo) { connection = DriverManager.getConnection(rdbSideTableInfo.getUrl(), rdbSideTableInfo.getUserName(), rdbSideTableInfo.getPassword()); } if (null != connection) { - if (connection.getMetaData().getTables(null, rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName(), null).next()) { + if (!connection.getMetaData().getTables(null, rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName(), null).next()) { LOG.error("Table " + rdbSideTableInfo.getTableName() + " doesn't exist"); throw new RuntimeException("Table " + rdbSideTableInfo.getTableName() + " doesn't exist"); } From c97455ba178ad2270e457d82d1db873f5c1f995a Mon Sep 17 00:00:00 2001 From: chuixue Date: Fri, 17 Jan 2020 14:35:39 +0800 Subject: [PATCH 28/69] =?UTF-8?q?fetch=20size=E4=B8=BA=E6=AF=8F=E6=AC=A1?= =?UTF-8?q?=E6=8B=89=E5=8F=961000=E6=9D=A1,=E5=B9=B6=E9=9D=9Elimit=201000?= =?UTF-8?q?=E6=9D=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index 3f12d90d6..7cb2c47e4 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -220,7 +220,7 @@ private void loadData(Map>> tmpCache) throws SQ } public int getFetchSize() { - return sideInfo.getSideTableInfo().getCacheSize(); + return 1000; } } From 5e5b175e280e646bf867fa7f22e240e15e9bb32a Mon Sep 17 00:00:00 2001 From: chuixue Date: Fri, 17 Jan 2020 17:45:29 +0800 Subject: [PATCH 29/69] =?UTF-8?q?=E8=BF=87=E6=BB=A4=E4=B8=AD=E9=97=B4?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E7=9A=84=E6=92=A4=E5=9B=9E=E6=B5=81=E6=95=B0?= =?UTF-8?q?=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/dtstack/flink/sql/Main.java | 4 +--- .../src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index b7968dad4..593237789 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -238,9 +238,7 @@ private static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironmen RowTypeInfo typeInfo = new RowTypeInfo(adaptTable.getSchema().getFieldTypes(), adaptTable.getSchema().getFieldNames()); DataStream adaptStream = tableEnv.toRetractStream(adaptTable, typeInfo) - .map((Tuple2 f0) -> { - return f0.f1; - }) + .filter((Tuple2 f0) -> f0.f0).map((Tuple2 f0) -> f0.f1) .returns(typeInfo); String fields = String.join(",", typeInfo.getFieldNames()); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index d5bac551e..bb51be5b7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -749,7 +749,7 @@ private void joinFun(Object pollObj, Map localTableCache, RowTypeInfo typeInfo = new RowTypeInfo(targetTable.getSchema().getFieldTypes(), targetTable.getSchema().getFieldNames()); DataStream adaptStream = tableEnv.toRetractStream(targetTable, org.apache.flink.types.Row.class) - .map((Tuple2 f0) -> { return f0.f1; }) + .filter((Tuple2 f0) -> f0.f0).map((Tuple2 f0) -> f0.f1) .returns(Row.class); //adaptStream.getTransformation().setOutputType(leftTypeInfo); From 6e3f7e7695449d686f08348c797bc684d339521f Mon Sep 17 00:00:00 2001 From: chuixue Date: Mon, 20 Jan 2020 13:39:47 +0800 Subject: [PATCH 30/69] =?UTF-8?q?=E5=BC=82=E6=AD=A5=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E9=93=BE=E6=8E=A5=E6=B1=A0=E5=8F=AF=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clickhouse/ClickhouseAsyncReqRow.java | 4 +- .../dtstack/flink/sql/side/SideTableInfo.java | 24 +++++++--- .../flink/sql/table/AbsSideTableParser.java | 45 +++++++++++-------- .../sql/side/mysql/MysqlAsyncReqRow.java | 4 +- .../sql/side/oracle/OracleAsyncReqRow.java | 4 +- .../postgresql/PostgresqlAsyncReqRow.java | 4 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 11 ++++- .../side/sqlserver/SqlserverAsyncReqRow.java | 4 +- 8 files changed, 65 insertions(+), 35 deletions(-) diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 305d65118..defd8598f 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -48,14 +48,14 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); clickhouseClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", CLICKHOUSE_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbPoolSize) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS); System.setProperty("vertx.disableFileCPResolving", "true"); VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbPoolSize); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java index e21389ea7..e342fbf37 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java @@ -16,7 +16,6 @@ * limitations under the License. */ - package com.dtstack.flink.sql.side; @@ -30,6 +29,7 @@ * Reason: * Date: 2018/7/25 * Company: www.dtstack.com + * * @author xuchao */ @@ -57,19 +57,31 @@ public abstract class SideTableInfo extends TableInfo implements Serializable { private long cacheTimeout = 60 * 1000;// - private int asyncCapacity=100; + private int asyncCapacity = 100; - private int asyncTimeout=10000; + private int asyncTimeout = 10000; private boolean partitionedJoin = false; - private String cacheMode="ordered"; + private String cacheMode = "ordered"; + + public static final String ASYNC_REQ_POOL_KEY = "asyncPoolSize"; + + private int asyncPoolSize = 0; + + public int getAsyncPoolSize() { + return asyncPoolSize; + } + + public void setAsyncPoolSize(int asyncPoolSize) { + this.asyncPoolSize = asyncPoolSize; + } - public RowTypeInfo getRowTypeInfo(){ + public RowTypeInfo getRowTypeInfo() { Class[] fieldClass = getFieldClasses(); TypeInformation[] types = new TypeInformation[fieldClass.length]; String[] fieldNames = getFields(); - for(int i=0; i Get information cache - protected void parseCacheProp(SideTableInfo sideTableInfo, Map props){ - if(props.containsKey(SideTableInfo.CACHE_KEY.toLowerCase())){ + protected void parseCacheProp(SideTableInfo sideTableInfo, Map props) { + if (props.containsKey(SideTableInfo.CACHE_KEY.toLowerCase())) { String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY.toLowerCase())); - if(cacheType == null){ + if (cacheType == null) { return; } - if(!ECacheType.isValid(cacheType)){ + if (!ECacheType.isValid(cacheType)) { throw new RuntimeException("can't not support cache type :" + cacheType); } sideTableInfo.setCacheType(cacheType); - if(props.containsKey(SideTableInfo.CACHE_SIZE_KEY.toLowerCase())){ + if (props.containsKey(SideTableInfo.CACHE_SIZE_KEY.toLowerCase())) { Integer cacheSize = MathUtil.getIntegerVal(props.get(SideTableInfo.CACHE_SIZE_KEY.toLowerCase())); - if(cacheSize < 0){ + if (cacheSize < 0) { throw new RuntimeException("cache size need > 0."); } sideTableInfo.setCacheSize(cacheSize); } - if(props.containsKey(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())){ + if (props.containsKey(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())) { Long cacheTTLMS = MathUtil.getLongVal(props.get(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())); - if(cacheTTLMS < 1000){ + if (cacheTTLMS < 1000) { throw new RuntimeException("cache time out need > 1000 ms."); } sideTableInfo.setCacheTimeout(cacheTTLMS); } - if(props.containsKey(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())){ + if (props.containsKey(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())) { Boolean partitionedJoinKey = MathUtil.getBoolean(props.get(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())); - if(partitionedJoinKey){ + if (partitionedJoinKey) { sideTableInfo.setPartitionedJoin(true); } } - if(props.containsKey(SideTableInfo.CACHE_MODE_KEY.toLowerCase())){ + if (props.containsKey(SideTableInfo.CACHE_MODE_KEY.toLowerCase())) { String cachemode = MathUtil.getString(props.get(SideTableInfo.CACHE_MODE_KEY.toLowerCase())); - if(!cachemode.equalsIgnoreCase("ordered") && !cachemode.equalsIgnoreCase("unordered")){ + if (!cachemode.equalsIgnoreCase("ordered") && !cachemode.equalsIgnoreCase("unordered")) { throw new RuntimeException("cachemode must ordered or unordered!"); } sideTableInfo.setCacheMode(cachemode.toLowerCase()); } - if(props.containsKey(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ + if (props.containsKey(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())) { Integer asyncCap = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())); - if(asyncCap < 0){ + if (asyncCap < 0) { throw new RuntimeException("asyncCapacity size need > 0."); } sideTableInfo.setAsyncCapacity(asyncCap); } - if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ + if (props.containsKey(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())) { Integer asyncTimeout = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); - if (asyncTimeout<0){ + if (asyncTimeout < 0) { throw new RuntimeException("asyncTimeout size need > 0."); } sideTableInfo.setAsyncTimeout(asyncTimeout); } + + if (props.containsKey(SideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())) { + Integer asyncPoolSize = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())); + if (asyncPoolSize < 0) { + throw new RuntimeException("asyncPoolSize size need > 0."); + } + sideTableInfo.setAsyncPoolSize(asyncPoolSize); + } } } } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index 6120767ef..f78febd28 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -61,7 +61,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); mysqlClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", MYSQL_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbPoolSize) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -73,7 +73,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbPoolSize); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index 1c5c107c3..c8d881b5a 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -58,7 +58,7 @@ public void open(Configuration parameters) throws Exception { establishConnection(rdbSideTableInfo); oracleClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", ORACLE_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbPoolSize) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -70,7 +70,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbPoolSize); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 02b333819..2d22650d6 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -60,13 +60,13 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); pgClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", POSTGRESQL_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbPoolSize) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()); VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbPoolSize); Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, pgClientConfig)); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 3ffd0e81a..d17ef1e24 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import io.vertx.core.json.JsonArray; import io.vertx.ext.sql.SQLClient; @@ -66,8 +67,17 @@ public class RdbAsyncReqRow extends AsyncReqRow { private transient SQLClient rdbSQLClient; + protected int rdbPoolSize = 3; + public RdbAsyncReqRow(SideInfo sideInfo) { super(sideInfo); + init(sideInfo); + } + + protected void init(SideInfo sideInfo) { + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + rdbPoolSize = rdbSideTableInfo.getAsyncPoolSize() == 0 ? DEFAULT_MAX_DB_CONN_POOL_SIZE : rdbSideTableInfo.getAsyncPoolSize(); + LOG.info("use rdb pool size {}", rdbPoolSize); } @Override @@ -168,7 +178,6 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except } - @Override public Row fillData(Row input, Object line) { JsonArray jsonArray = (JsonArray) line; diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index cb4240ca6..8f02fbd63 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -51,7 +51,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); sqlserverClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", SQLSERVER_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbPoolSize) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -64,7 +64,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbPoolSize); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); From 7335489ad7e8f283a3c2ca6b29ae7ce2539d177a Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Tue, 4 Feb 2020 18:21:21 +0800 Subject: [PATCH 31/69] add oceanbase sink --- README.md | 3 +- docs/oceanbaseSink.md | 53 +++++++++++ oceanbase/oceanbase-sink/pom.xml | 92 +++++++++++++++++++ .../sql/sink/oceanbase/OceanbaseSink.java | 79 ++++++++++++++++ .../oceanbase/table/OceanbaseSinkParser.java | 43 +++++++++ oceanbase/pom.xml | 47 ++++++++++ pom.xml | 1 + 7 files changed, 317 insertions(+), 1 deletion(-) create mode 100644 docs/oceanbaseSink.md create mode 100644 oceanbase/oceanbase-sink/pom.xml create mode 100644 oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/OceanbaseSink.java create mode 100644 oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/table/OceanbaseSinkParser.java create mode 100644 oceanbase/pom.xml diff --git a/README.md b/README.md index 34e51cfd6..33d8a003f 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ # 已支持 * 源表：kafka 0.9、0.10、0.11、1.x版本 * 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, serversocket, kudu, postgresql, clickhouse - * 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse + * 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse, oceanbase # 后续开发计划 * 维表快照 @@ -190,6 +190,7 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * [kudu 结果表插件](docs/kuduSink.md) * [postgresql 结果表插件](docs/postgresqlSink.md) * [clickhouse 结果表插件](docs/clickhouseSink.md) +* [oceanbase 结果表插件](docs/oceanbaseSink.md) ### 2.3 维表插件 * [hbase 维表插件](docs/hbaseSide.md) diff --git a/docs/oceanbaseSink.md b/docs/oceanbaseSink.md new file mode 100644 index 000000000..bdd8502d8 --- /dev/null +++ b/docs/oceanbaseSink.md @@ -0,0 +1,53 @@ +## 1.格式： +``` +CREATE TABLE tableName( + colName colType, + ... + colNameX colType + )WITH( + type ='oceanbase', + url ='jdbcUrl', + userName ='userName', + password ='pwd', + tableName ='tableName', + parallelism ='parllNum' + ); + +``` + +## 2.支持版本 + oceanbase 1.+, 2.+ + +## 3.表结构定义 + +|参数名称|含义| +|----|---| +| tableName| oceanbase表名称| +| colName | 列名称| +| colType | 列类型 [colType支持的类型](colType.md)| + +## 4.参数： + +|参数名称|含义|是否必填|默认值| +|----|----|----|----| +|type |表名 输出表类型|是|| +|url | 连接oceanbase数据库 jdbcUrl |是|| +|userName | oceanbase连接用户名 |是|| +| password | oceanbase连接密码|是|| +| tableName | oceanbase表名称|是|| +| parallelism | 并行度设置|否|1| + +## 5.样例： +``` +CREATE TABLE MyResult( + channel VARCHAR, + pv VARCHAR + )WITH( + type ='oceanbase', + url ='jdbc:mysql://172.16.8.104:3306/test', + userName ='dtstack', + password ='abc123', + tableName ='pv2', + parallelism ='1' + ) + ``` \ No newline at end of file diff --git a/oceanbase/oceanbase-sink/pom.xml b/oceanbase/oceanbase-sink/pom.xml new file mode 100644 index 000000000..e45a511bb --- /dev/null +++ b/oceanbase/oceanbase-sink/pom.xml @@ -0,0 +1,92 @@ + + + + sql.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.sink.oceanbase + jar + + oceanbase-sink + http://maven.apache.org + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.sink.rdb + ${sql.sink.rdb.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/OceanbaseSink.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/OceanbaseSink.java new file mode 100644 index 000000000..13bbee17f --- /dev/null +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/OceanbaseSink.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.dtstack.flink.sql.sink.oceanbase; + + +import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; + +import java.util.List; +import java.util.Map; + +/** + * Date: 2020/2/03 + * Company: www.dtstack.com + * @author xiuzhu + */ + +public class OceanbaseSink extends RdbSink implements IStreamSinkGener { + + private static final String OB_DRIVER = "com.mysql.jdbc.Driver"; + + public OceanbaseSink() { + } + + @Override + public void buildSql(String schema, String tableName, List fields) { + buildInsertSql(tableName, fields); + } + + @Override + public String buildUpdateSql(String schema, String tableName, List fieldNames, Map> realIndexes, List fullField) { + return null; + } + + @Override + public String getDriverName() { + return OB_DRIVER; + } + + @Override + public RetractJDBCOutputFormat getOutputFormat() { + return new RetractJDBCOutputFormat(); + } + + public void buildInsertSql(String tableName, List fields){ + String sqlTmp = "replace into " + tableName + " (${fields}) values (${placeholder})"; + String fieldsStr = ""; + String placeholder = ""; + + for (String fieldName : fields) { + fieldsStr += ",`" + fieldName + "`"; + placeholder += ",?"; + } + + fieldsStr = fieldsStr.replaceFirst(",", ""); + placeholder = placeholder.replaceFirst(",", ""); + + sqlTmp = sqlTmp.replace("${fields}", fieldsStr).replace("${placeholder}", placeholder); + this.sql = sqlTmp; + } +} diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/table/OceanbaseSinkParser.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/table/OceanbaseSinkParser.java new file mode 100644 index 000000000..31882f4ba --- /dev/null +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/table/OceanbaseSinkParser.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.oceanbase.table; + +import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; +import com.dtstack.flink.sql.table.TableInfo; + +import java.util.Map; + +/** + * Date: 2020/2/3 + * Company: www.dtstack.com + * @author xiuzhu + */ + +public class OceanbaseSinkParser extends RdbSinkParser { + + private static final String CURR_TYPE = "oceanbase"; + + @Override + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo oceanBaseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + oceanBaseTableInfo.setType(CURR_TYPE); + return oceanBaseTableInfo; + } + +} diff --git a/oceanbase/pom.xml b/oceanbase/pom.xml new file mode 100644 index 000000000..abc90f28c --- /dev/null +++ b/oceanbase/pom.xml @@ -0,0 +1,47 @@ + + + + flink.sql + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.oceanbase + pom + + + oceanbase-sink + + + + 5.1.40 + 1.0-SNAPSHOT + + + + + junit + junit + 3.8.1 + test + + + + com.dtstack.flink + sql.core + ${sql.core.version} + provided + + + + mysql + mysql-connector-java + ${mysql.connector.version} + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 81cb6d2e1..95c632397 100644 --- a/pom.xml +++ b/pom.xml @@ -30,6 +30,7 @@ serversocket console clickhouse + oceanbase From e8ba8913ab613fb1c4b06c677b3e92ad31bb3dd9 Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 13 Feb 2020 17:56:39 +0800 Subject: [PATCH 32/69] hbase datatype convert bug fix --- .../com/dtstack/flink/sql/util/MathUtil.java | 82 ++++++++++++++++++- .../sql/side/hbase/utils/HbaseUtils.java | 27 +++--- 2 files changed, 95 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index 37b06c8b8..a846b9b19 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -16,7 +16,7 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.util; @@ -66,6 +66,58 @@ public static Integer getIntegerVal(Object obj){ throw new RuntimeException("not support type of " + obj.getClass() + " convert to Integer." ); } + public static Float getFloatVal(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Float.valueOf((String) obj); + } else if (obj instanceof Float) { + return (Float) obj; + } else if (obj instanceof BigDecimal) { + return ((BigDecimal) obj).floatValue(); + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Float."); + } + + public static Float getFloatVal(Object obj, float defaultVal) { + if (obj == null) { + return defaultVal; + } + + return getFloatVal(obj); + } + + public static Double getDoubleVal(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Double.valueOf((String) obj); + } else if (obj instanceof Float) { + return ((Float) obj).doubleValue(); + } else if (obj instanceof Double){ + return (Double) obj; + }else if (obj instanceof BigDecimal) { + return ((BigDecimal) obj).doubleValue(); + }else if (obj instanceof Integer){ + return ((Integer)obj).doubleValue(); + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Double."); + } + + public static Double getDoubleVal(Object obj, double defaultVal) { + if (obj == null) { + return defaultVal; + } + + return getDoubleVal(obj); + } + public static Boolean getBoolean(Object obj, boolean defaultVal){ if(obj == null){ return defaultVal; @@ -99,4 +151,32 @@ public static String getString(Object obj){ return obj.toString(); } } + + public static Byte getByte(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Byte.valueOf((String) obj); + } else if (obj instanceof Byte) { + return (Byte) obj; + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Byte."); + } + + public static Short getShort(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Short.valueOf((String) obj); + } else if (obj instanceof Short) { + return (Short) obj; + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Short."); + } } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java index 39df3c913..098d967d6 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java @@ -16,11 +16,11 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.side.hbase.utils; -import com.dtstack.flink.sql.util.ByteUtils; +import com.dtstack.flink.sql.util.MathUtil; import org.apache.hadoop.hbase.util.Bytes; /** @@ -34,41 +34,42 @@ public class HbaseUtils { public static Object convertByte(byte[] hbaseData, String type){ + String data = Bytes.toString(hbaseData); if(type == null){ - return new String(hbaseData); + return data; } switch (type.toLowerCase()) { case "boolean": - return Bytes.toBoolean(hbaseData); + return MathUtil.getBoolean(data); case "int": - return Bytes.toInt(hbaseData); + return MathUtil.getIntegerVal(data); case "bigint": - return Bytes.toLong(hbaseData); + return MathUtil.getLongVal(data); case "tinyint": case "byte": - return ByteUtils.bytes2Byte(hbaseData); + return MathUtil.getByte(data); case "short": case "smallint": - return Bytes.toShort(hbaseData); + return MathUtil.getShort(data); case "char": case "varchar": case "string": - return Bytes.toString(hbaseData); + return data; case "float": - return Bytes.toFloat(hbaseData); + return MathUtil.getFloatVal(data); case "double": - return Bytes.toDouble(hbaseData); + return MathUtil.getDoubleVal(data); + default: + throw new RuntimeException("not support type of " + type); } - - throw new RuntimeException("not support type of " + type); } } From 162a20794d23fd7a33daf4ed35d8230c032b00aa Mon Sep 17 00:00:00 2001 From: chuixue Date: Mon, 17 Feb 2020 11:33:58 +0800 Subject: [PATCH 33/69] =?UTF-8?q?oracle=20sink=E5=A2=9E=E5=8A=A0=E7=AC=AC?= =?UTF-8?q?=E4=B8=80=E6=9D=A1=E5=BC=82=E5=B8=B8=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java index 7653c847c..90ecce9b5 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java @@ -210,7 +210,7 @@ private void writeSingleRecord(Row row) { dbConn.commit(); } catch (SQLException e) { outDirtyRecords.inc(); - if (outDirtyRecords.getCount() % dirtyDataPrintFrequency == 0 || LOG.isDebugEnabled()) { + if (outDirtyRecords.getCount() == 1 || outDirtyRecords.getCount() % dirtyDataPrintFrequency == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ..", row.toString()); LOG.error("", e); } From 16f1415eb5073a25873c484a2fbf7f271c0634c6 Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Mon, 2 Mar 2020 11:57:08 +0800 Subject: [PATCH 34/69] add oceanbase async side --- .../oceanbase-async-side/pom.xml | 93 +++++++++++++++++++ .../side/oceanbase/OceanbaseAsyncReqRow.java | 80 ++++++++++++++++ .../oceanbase/OceanbaseAsyncSideInfo.java | 43 +++++++++ .../oceanbase-side-core/pom.xml | 20 ++++ .../oceanbase/table/OceanbaseSideParser.java | 46 +++++++++ oceanbase/oceanbase-side/pom.xml | 37 ++++++++ oceanbase/pom.xml | 1 + 7 files changed, 320 insertions(+) create mode 100644 oceanbase/oceanbase-side/oceanbase-async-side/pom.xml create mode 100644 oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncReqRow.java create mode 100644 oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncSideInfo.java create mode 100644 oceanbase/oceanbase-side/oceanbase-side-core/pom.xml create mode 100644 oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java create mode 100644 oceanbase/oceanbase-side/pom.xml diff --git a/oceanbase/oceanbase-side/oceanbase-async-side/pom.xml b/oceanbase/oceanbase-side/oceanbase-async-side/pom.xml new file mode 100644 index 000000000..a59cbc0e5 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-async-side/pom.xml @@ -0,0 +1,93 @@ + + + + sql.side.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + com.dtstack.flink + sql.side.async.oceanbase + + oceanbase-async-side + jar + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.oceanbase.core + ${sql.side.oceanbase.core.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncReqRow.java b/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncReqRow.java new file mode 100644 index 000000000..a2896ca1e --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncReqRow.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.dtstack.flink.sql.side.oceanbase; + +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import io.vertx.core.Vertx; +import io.vertx.core.VertxOptions; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.jdbc.JDBCClient; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +/** + * Date: 2020/02/29 + * Company: www.dtstack.com + * + * @author xiuzhu + */ + +public class OceanbaseAsyncReqRow extends RdbAsyncReqRow { + + private static final Logger LOG = LoggerFactory.getLogger(OceanbaseAsyncReqRow.class); + + private final static String OCEANBASE_DRIVER = "com.mysql.jdbc.Driver"; + + public OceanbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + super(new OceanbaseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + } + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + JsonObject oceanbaseClientConfig = new JsonObject(); + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + oceanbaseClientConfig.put("url", rdbSideTableInfo.getUrl()) + .put("driver_class", OCEANBASE_DRIVER) + .put("max_pool_size", rdbPoolSize) + .put("user", rdbSideTableInfo.getUserName()) + .put("password", rdbSideTableInfo.getPassword()) + .put("provider_class", DT_PROVIDER_CLASS) + .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) + .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); + + System.setProperty("vertx.disableFileCPResolving", "true"); + + VertxOptions vo = new VertxOptions(); + vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); + vo.setWorkerPoolSize(rdbPoolSize); + vo.setFileResolverCachingEnabled(false); + Vertx vertx = Vertx.vertx(vo); + setRdbSQLClient(JDBCClient.createNonShared(vertx, oceanbaseClientConfig)); + } + +} diff --git a/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncSideInfo.java b/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncSideInfo.java new file mode 100644 index 000000000..e8e3b288d --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncSideInfo.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.dtstack.flink.sql.side.oceanbase; + +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; + +import java.util.List; + +/** + * Reason: + * Date: 2020/02/29 + * Company: www.dtstack.com + * + * @author xiuzhu + */ + +public class OceanbaseAsyncSideInfo extends RdbAsyncSideInfo { + + public OceanbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + } +} diff --git a/oceanbase/oceanbase-side/oceanbase-side-core/pom.xml b/oceanbase/oceanbase-side/oceanbase-side-core/pom.xml new file mode 100644 index 000000000..024d92c86 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-side-core/pom.xml @@ -0,0 +1,20 @@ + + + + sql.side.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + com.dtstack.flink + sql.side.oceanbase.core + 1.0-SNAPSHOT + jar + oceanbase-side-core + + + \ No newline at end of file diff --git a/oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java b/oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java new file mode 100644 index 000000000..43f92fe29 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.dtstack.flink.sql.side.oceanbase.table; + +import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; +import com.dtstack.flink.sql.table.TableInfo; + +import java.util.Map; + +/** + * Reason: + * Date: 2020/02/29 + * Company: www.dtstack.com + * + * @author xiuzhu + */ + + +public class OceanbaseSideParser extends RdbSideParser { + + private static final String CURR_TYPE = "oceanbase"; + + @Override + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo oceanbaseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + oceanbaseTableInfo.setType(CURR_TYPE); + return oceanbaseTableInfo; + } +} diff --git a/oceanbase/oceanbase-side/pom.xml b/oceanbase/oceanbase-side/pom.xml new file mode 100644 index 000000000..f7a18532e --- /dev/null +++ b/oceanbase/oceanbase-side/pom.xml @@ -0,0 +1,37 @@ + + + + sql.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + com.dtstack.flink + sql.side.oceanbase + 1.0-SNAPSHOT + + + oceanbase-side-core + oceanbase-async-side + + + oceanbase-side + pom + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.rdb + ${rdb.side.version} + + + + \ No newline at end of file diff --git a/oceanbase/pom.xml b/oceanbase/pom.xml index abc90f28c..bcce9fc0a 100644 --- a/oceanbase/pom.xml +++ b/oceanbase/pom.xml @@ -15,6 +15,7 @@ oceanbase-sink + oceanbase-side From ec69b2c5eab61798802c478a752569650325744c Mon Sep 17 00:00:00 2001 From: chuixue Date: Sun, 8 Mar 2020 20:49:25 +0800 Subject: [PATCH 35/69] side table primary key trim --- .../com/dtstack/flink/sql/table/AbsTableParser.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java index aa0639c8a..39a1fbaaf 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java @@ -25,10 +25,12 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Reason: @@ -112,9 +114,13 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ public static void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ String primaryFields = matcher.group(1).trim(); - String[] splitArry = primaryFields.split(","); - List primaryKes = Lists.newArrayList(splitArry); - tableInfo.setPrimaryKeys(primaryKes); + if (primaryFields != null) { + String[] splitArry = primaryFields.split(","); + List primaryKes = Arrays.stream(splitArry).map(x -> x.trim()).collect(Collectors.toList()); + tableInfo.setPrimaryKeys(primaryKes); + return; + } + tableInfo.setPrimaryKeys(Lists.newArrayList()); } public Class dbTypeConvertToJavaType(String fieldType) { From 7e9af6b4bb26ca3c0f060cbbdae261776774ec83 Mon Sep 17 00:00:00 2001 From: chuixue Date: Mon, 9 Mar 2020 18:17:05 +0800 Subject: [PATCH 36/69] oracle trim --- core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java | 2 +- .../dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index a846b9b19..1a590d9f3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -146,7 +146,7 @@ public static String getString(Object obj){ } if(obj instanceof String){ - return (String) obj; + return ((String) obj).trim(); }else { return obj.toString(); } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index c9d2f98d9..6d0b8ac9d 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -60,7 +60,7 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { sqlCondition = "select ${selectField} from ${tableName} where "; for (int i = 0; i < equalFieldList.size(); i++) { String equalField = sideTableInfo.getPhysicalFields().getOrDefault(equalFieldList.get(i), equalFieldList.get(i)); - sqlCondition += dealLowerFiled(equalField) + " " + sqlJoinCompareOperate.get(i) + " " + " ?"; + sqlCondition += dealLowerFiledTrim(equalField) + " " + sqlJoinCompareOperate.get(i) + " " + " ?"; if (i != equalFieldList.size() - 1) { sqlCondition += " and "; } @@ -72,8 +72,8 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { - private String dealLowerFiled(String field) { - return "\"" + field + "\""; + private String dealLowerFiledTrim(String field) { + return "trim(\"" + field + "\")"; } private String dealLowerSelectFiled(String fieldsStr) { From 5e6bbe518accc7128ca5177f6a0b355323673af4 Mon Sep 17 00:00:00 2001 From: chuixue Date: Wed, 11 Mar 2020 16:25:16 +0800 Subject: [PATCH 37/69] join watermark not trigger --- .../java/com/dtstack/flink/sql/side/AsyncReqRow.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index 74356b9e6..5185de7a6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -30,17 +30,16 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; -import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.Date; import java.sql.Timestamp; +import java.time.LocalDate; import java.time.LocalDateTime; -import java.util.Collection; import java.util.Collections; -import java.util.concurrent.TimeoutException; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow @@ -105,6 +104,10 @@ protected Object convertTimeIndictorTypeInfo(Integer index, Object obj) { //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. if (obj instanceof LocalDateTime && isTimeIndicatorTypeInfo) { obj = Timestamp.valueOf(((LocalDateTime) obj)); + } else if (obj instanceof LocalDateTime) { + obj = Timestamp.valueOf(((LocalDateTime) obj)); + } else if (obj instanceof LocalDate) { + obj = Date.valueOf((LocalDate) obj); } return obj; } From 2249a7561928d92c69478a0531964c417f035b77 Mon Sep 17 00:00:00 2001 From: chuixue Date: Tue, 17 Mar 2020 20:44:34 +0800 Subject: [PATCH 38/69] oracle trim rollback --- .../dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index 6d0b8ac9d..c9d2f98d9 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -60,7 +60,7 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { sqlCondition = "select ${selectField} from ${tableName} where "; for (int i = 0; i < equalFieldList.size(); i++) { String equalField = sideTableInfo.getPhysicalFields().getOrDefault(equalFieldList.get(i), equalFieldList.get(i)); - sqlCondition += dealLowerFiledTrim(equalField) + " " + sqlJoinCompareOperate.get(i) + " " + " ?"; + sqlCondition += dealLowerFiled(equalField) + " " + sqlJoinCompareOperate.get(i) + " " + " ?"; if (i != equalFieldList.size() - 1) { sqlCondition += " and "; } @@ -72,8 +72,8 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { - private String dealLowerFiledTrim(String field) { - return "trim(\"" + field + "\")"; + private String dealLowerFiled(String field) { + return "\"" + field + "\""; } private String dealLowerSelectFiled(String fieldsStr) { From 7931ebcb38943e61831f31b58f534c6b6d69e392 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 30 Mar 2020 15:12:28 +0800 Subject: [PATCH 39/69] hbase sink kerberos --- .../com/dtstack/flink/sql/option/Options.java | 11 +++ docs/hbaseSink.md | 27 ++--- .../sql/sink/hbase/HbaseOutputFormat.java | 77 ++++++++++++--- .../flink/sql/sink/hbase/HbaseSink.java | 20 +++- .../sql/sink/hbase/table/HbaseSinkParser.java | 20 ++-- .../sql/sink/hbase/table/HbaseTableInfo.java | 51 ++++++++++ .../sink/hbase/utils/HbaseConfigUtils.java | 99 ++----------------- .../sql/launcher/perjob/ConfigParseUtil.java | 42 ++++++++ .../perjob/PerJobClusterClientBuilder.java | 11 ++- .../sql/launcher/perjob/PerJobSubmitter.java | 13 ++- 10 files changed, 237 insertions(+), 134 deletions(-) create mode 100644 launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/ConfigParseUtil.java diff --git a/core/src/main/java/com/dtstack/flink/sql/option/Options.java b/core/src/main/java/com/dtstack/flink/sql/option/Options.java index a653aa42e..9346fbb10 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/Options.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/Options.java @@ -54,6 +54,9 @@ public class Options { @OptionRequired(description = "sql ext jar,eg udf jar") private String addjar; + @OptionRequired(description = "file add to ship file") + private String addShipfile; + @OptionRequired(description = "sql ref prop,eg specify event time") private String confProp = "{}"; @@ -194,4 +197,12 @@ public String getPluginLoadMode() { public void setPluginLoadMode(String pluginLoadMode) { this.pluginLoadMode = pluginLoadMode; } + + public String getAddShipfile() { + return addShipfile; + } + + public void setAddShipfile(String addShipfile) { + this.addShipfile = addShipfile; + } } diff --git a/docs/hbaseSink.md b/docs/hbaseSink.md index 3926101ed..8045457a1 100644 --- a/docs/hbaseSink.md +++ b/docs/hbaseSink.md @@ -36,16 +36,23 @@ hbase2.0 |tableName | 关联的hbase表名称|是|| |rowKey | hbase的rowkey关联的列信息|是|| |parallelism | 并行度设置|否|1| +| kerberosAuthEnable | 是否开启kerberos认证|否|false| +| regionserverKeytabFile| regionserver的KeytabFile名称，yarnPer模式需通过addShipfile提前上传，本地模式会查找user.dir路径|否|| +| regionserverPrincipal | regionserver的principal|否|| +| zookeeperSaslClient | zookeeper.sasl.client值|否|true| +| securityKrb5Conf | java.security.krb5.conf值|否|| +> kerberos 配置 + > kerberos 配置 * hbase.security.authentication = 'kerberos', * hbase.security.authorization = 'true', * hbase.master.kerberos.principal = 'hbase/cdh01@DTSTACK.COM', - * hbase.master.keytab.file = '/Users/hbase.keytab', - * hbase.regionserver.keytab.file = '/Users/hbase.keytab', + * hbase.master.keytab.file = 'hbase.keytab', + * hbase.regionserver.keytab.file = 'hbase.keytab', * hbase.regionserver.kerberos.principal = 'hbase/cdh01@DTSTACK.COM' - * (非必选)java.security.krb5.conf = '/etc/krb5.conf' + * (非必选)java.security.krb5.conf = 'krb5.conf' ## 5.样例： ``` @@ -58,14 +65,12 @@ hbase2.0 zookeeperQuorum ='172.16.10.104:2181,172.16.10.224:2181,172.16.10.252:2181', zookeeperParent ='/hbase', tableName ='workerinfo01', - rowKey ='channel' - - ---hbase.security.authentication = 'kerberos', - ---hbase.security.authorization = 'true', - ---hbase.master.kerberos.principal = 'hbase/cdh01@DTSTACK.COM', - ---hbase.master.keytab.file = '/Users/chuixue/Desktop/hbase.keytab', - ---hbase.regionserver.keytab.file = '/Users/chuixue/Desktop/hbase.keytab', - ---hbase.regionserver.kerberos.principal = 'hbase/cdh01@DTSTACK.COM' + rowKey ='channel', + kerberosAuthEnable='true', + regionserverKeytabFile = 'hbase.keytab', + regionserverPrincipal = 'hbase/kerberos1@DTSTACK.COM', + zookeeperSaslClient='false', + securityKrb5Conf='/etc/krb5.conf' diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index b0aa10989..46b7b9d05 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -37,6 +37,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.security.PrivilegedAction; import java.text.SimpleDateFormat; @@ -59,8 +60,11 @@ public class HbaseOutputFormat extends MetricOutputFormat { private String[] columnNames; private String[] columnTypes; private Map columnNameFamily; - private Map hbaseConfig; - + private boolean kerberosAuthEnable; + private String regionserverKeytabFile; + private String regionserverPrincipal; + private String securityKrb5Conf; + private String zookeeperSaslClient; private String[] families; private String[] qualifiers; @@ -78,18 +82,14 @@ public class HbaseOutputFormat extends MetricOutputFormat { @Override public void configure(Configuration parameters) { LOG.warn("---configure---"); - boolean openKerberos = HbaseConfigUtils.openKerberos(hbaseConfig); try { - if (openKerberos) { - conf = HbaseConfigUtils.getHadoopConfiguration(hbaseConfig); + conf = HBaseConfiguration.create(); + if (kerberosAuthEnable) { conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); - String principal = HbaseConfigUtils.getPrincipal(hbaseConfig); - String keytab = HbaseConfigUtils.getKeytab(hbaseConfig); - - UserGroupInformation userGroupInformation = null; + fillSyncKerberosConfig(conf, regionserverKeytabFile, regionserverPrincipal, zookeeperSaslClient, securityKrb5Conf); - userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, principal, keytab); + UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, regionserverPrincipal, regionserverKeytabFile); org.apache.hadoop.conf.Configuration finalConf = conf; conn = userGroupInformation.doAs(new PrivilegedAction() { @Override @@ -103,13 +103,12 @@ public Connection run() { } }); } else { - conf = HbaseConfigUtils.getConfig(hbaseConfig); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); conn = ConnectionFactory.createConnection(conf); } } catch (IOException e) { - LOG.error("", e); + throw new RuntimeException(e); } } @@ -256,8 +255,28 @@ public HbaseOutputFormatBuilder setColumnNameFamily(Map columnNa return this; } - public HbaseOutputFormatBuilder setHbaseConfig(Map hbaseConfig) { - format.hbaseConfig = hbaseConfig; + public HbaseOutputFormatBuilder setKerberosAuthEnable(boolean kerberosAuthEnable) { + format.kerberosAuthEnable = kerberosAuthEnable; + return this; + } + + public HbaseOutputFormatBuilder setRegionserverKeytabFile(String regionserverKeytabFile) { + format.regionserverKeytabFile = regionserverKeytabFile; + return this; + } + + public HbaseOutputFormatBuilder setRegionserverPrincipal(String regionserverPrincipal) { + format.regionserverPrincipal = regionserverPrincipal; + return this; + } + + public HbaseOutputFormatBuilder setSecurityKrb5Conf(String securityKrb5Conf) { + format.securityKrb5Conf = securityKrb5Conf; + return this; + } + + public HbaseOutputFormatBuilder setZookeeperSaslClient(String zookeeperSaslClient) { + format.zookeeperSaslClient = zookeeperSaslClient; return this; } @@ -289,5 +308,35 @@ public HbaseOutputFormat finish() { } + private void fillSyncKerberosConfig(org.apache.hadoop.conf.Configuration config, String regionserverKeytabFile, String regionserverPrincipal, + String zookeeperSaslClient, String securityKrb5Conf) throws IOException { + if (StringUtils.isEmpty(regionserverKeytabFile)) { + throw new IllegalArgumentException("Must provide regionserverKeytabFile when authentication is Kerberos"); + } + String regionserverKeytabFilePath = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; + LOG.info("regionserverKeytabFilePath:{}",regionserverKeytabFilePath); + config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KEYTAB_FILE, regionserverKeytabFilePath); + config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KEYTAB_FILE, regionserverKeytabFilePath); + + if (StringUtils.isEmpty(regionserverPrincipal)) { + throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); + } + config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, regionserverPrincipal); + config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL, regionserverPrincipal); + config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHORIZATION, "true"); + config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHENTICATION, "kerberos"); + + + if (!StringUtils.isEmpty(zookeeperSaslClient)) { + System.setProperty(HbaseConfigUtils.KEY_ZOOKEEPER_SASL_CLIENT, zookeeperSaslClient); + } + + if (!StringUtils.isEmpty(securityKrb5Conf)) { + String krb5ConfPath = System.getProperty("user.dir") + File.separator + securityKrb5Conf; + LOG.info("krb5ConfPath:{}", krb5ConfPath); + System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_KRB5_CONF, krb5ConfPath); + } + } + } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index c918c60af..6bffca2f0 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -54,8 +54,12 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener< protected String tableName; protected String[] rowkey; protected String registerTabName; - protected Map hbaseConfig; + protected boolean kerberosAuthEnable; + protected String regionserverKeytabFile; + protected String regionserverPrincipal; + protected String securityKrb5Conf; + protected String zookeeperSaslClient; private int parallelism = -1; public HbaseSink() { @@ -72,7 +76,12 @@ public HbaseSink genStreamSink(TargetTableInfo targetTableInfo) { this.rowkey = hbaseTableInfo.getRowkey(); this.columnNameFamily = hbaseTableInfo.getColumnNameFamily(); this.registerTabName = hbaseTableInfo.getName(); - this.hbaseConfig = hbaseTableInfo.getHbaseConfig(); + + this.kerberosAuthEnable = hbaseTableInfo.isKerberosAuthEnable(); + this.regionserverKeytabFile = hbaseTableInfo.getRegionserverKeytabFile(); + this.regionserverPrincipal = hbaseTableInfo.getRegionserverPrincipal(); + this.securityKrb5Conf = hbaseTableInfo.getSecurityKrb5Conf(); + this.zookeeperSaslClient = hbaseTableInfo.getZookeeperSaslClient(); Integer tmpSinkParallelism = hbaseTableInfo.getParallelism(); if (tmpSinkParallelism != null) { @@ -121,7 +130,12 @@ public DataStreamSink> consumeDataStream(DataStream entity.getKey().contains(".")) - .map(entity -> hbaseTableInfo.getHbaseConfig().put(entity.getKey(), entity.getValue())) - .count(); + hbaseTableInfo.setKerberosAuthEnable(MathUtil.getBoolean(props.get(KERBEROS_AUTH_ENABLE_KEY.toLowerCase()), false)); + hbaseTableInfo.setRegionserverKeytabFile((String) props.get(REGIONSERVER_KEYTAB_FILE_KEY.toLowerCase())); + hbaseTableInfo.setRegionserverPrincipal((String) props.get(REGIONSERVER_PRINCIPAL_KEY.toLowerCase())); + hbaseTableInfo.setSecurityKrb5Conf((String) props.get(SECURITY_KRB5_CONF_KEY.toLowerCase())); + hbaseTableInfo.setZookeeperSaslClient((String) props.get(ZOOKEEPER_SASL_CLINT_KEY.toLowerCase())); return hbaseTableInfo; } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java index 9f5104e73..565ab6cb7 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java @@ -54,6 +54,16 @@ public class HbaseTableInfo extends TargetTableInfo { private String tableName; + private boolean kerberosAuthEnable; + + private String regionserverKeytabFile; + + private String regionserverPrincipal; + + private String securityKrb5Conf; + + private String zookeeperSaslClient; + private Map hbaseConfig = Maps.newHashMap(); public HbaseTableInfo(){ @@ -150,4 +160,45 @@ public Map getHbaseConfig() { public void setHbaseConfig(Map hbaseConfig) { this.hbaseConfig = hbaseConfig; } + + public boolean isKerberosAuthEnable() { + return kerberosAuthEnable; + } + + public void setKerberosAuthEnable(boolean kerberosAuthEnable) { + this.kerberosAuthEnable = kerberosAuthEnable; + } + + public String getRegionserverKeytabFile() { + return regionserverKeytabFile; + } + + public void setRegionserverKeytabFile(String regionserverKeytabFile) { + this.regionserverKeytabFile = regionserverKeytabFile; + } + + public String getRegionserverPrincipal() { + return regionserverPrincipal; + } + + public void setRegionserverPrincipal(String regionserverPrincipal) { + this.regionserverPrincipal = regionserverPrincipal; + } + + public String getSecurityKrb5Conf() { + return securityKrb5Conf; + } + + public void setSecurityKrb5Conf(String securityKrb5Conf) { + this.securityKrb5Conf = securityKrb5Conf; + } + + public String getZookeeperSaslClient() { + return zookeeperSaslClient; + } + + public void setZookeeperSaslClient(String zookeeperSaslClient) { + this.zookeeperSaslClient = zookeeperSaslClient; + } } + diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java index a463bb686..4b909dec5 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java @@ -18,18 +18,12 @@ package com.dtstack.flink.sql.sink.hbase.utils; -import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; /** * @@ -42,95 +36,20 @@ public class HbaseConfigUtils { private static final Logger LOG = LoggerFactory.getLogger(HbaseConfigUtils.class); - - private final static String AUTHENTICATION_TYPE = "Kerberos"; - private final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; - private final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; - private final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; - private final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; - private final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; - private final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; + // sync side kerberos + public final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; + public final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; + public final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; + public final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; + public final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; + public final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; - private static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; - - private static List KEYS_KERBEROS_REQUIRED = Arrays.asList( - KEY_HBASE_SECURITY_AUTHENTICATION, - KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, - KEY_HBASE_MASTER_KEYTAB_FILE, - KEY_HBASE_REGIONSERVER_KEYTAB_FILE, - KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL - ); - - - public static Configuration getConfig(Map hbaseConfigMap) { - Configuration hConfiguration = HBaseConfiguration.create(); - - for (Map.Entry entry : hbaseConfigMap.entrySet()) { - if (entry.getValue() != null && !(entry.getValue() instanceof Map)) { - hConfiguration.set(entry.getKey(), entry.getValue().toString()); - } - } - return hConfiguration; - } - - public static boolean openKerberos(Map hbaseConfigMap) { - if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHORIZATION)) { - return false; - } - return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); - } - - public static Configuration getHadoopConfiguration(Map hbaseConfigMap) { - for (String key : KEYS_KERBEROS_REQUIRED) { - if (StringUtils.isEmpty(MapUtils.getString(hbaseConfigMap, key))) { - throw new IllegalArgumentException(String.format("Must provide [%s] when authentication is Kerberos", key)); - } - } - loadKrb5Conf(hbaseConfigMap); - - Configuration conf = new Configuration(); - if (hbaseConfigMap == null) { - return conf; - } - - hbaseConfigMap.forEach((key, val) -> { - if (val != null) { - conf.set(key, val.toString()); - } - }); - - return conf; - } - - public static String getPrincipal(Map hbaseConfigMap) { - String principal = MapUtils.getString(hbaseConfigMap, KEY_HBASE_MASTER_KERBEROS_PRINCIPAL); - if (StringUtils.isNotEmpty(principal)) { - return principal; - } - - throw new IllegalArgumentException(""); - } - - public static String getKeytab(Map hbaseConfigMap) { - String keytab = MapUtils.getString(hbaseConfigMap, KEY_HBASE_MASTER_KEYTAB_FILE); - if (StringUtils.isNotEmpty(keytab)) { - return keytab; - } - - throw new IllegalArgumentException(""); - } - - public static void loadKrb5Conf(Map kerberosConfig) { - String krb5FilePath = MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); - if (org.apache.commons.lang.StringUtils.isEmpty(krb5FilePath)) { - return; - } - kerberosConfig.put(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); - } + public static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + public static final String KEY_ZOOKEEPER_SASL_CLIENT = "zookeeper.sasl.client"; public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { if (conf == null) { diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/ConfigParseUtil.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/ConfigParseUtil.java new file mode 100644 index 000000000..5acdb59a1 --- /dev/null +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/ConfigParseUtil.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.launcher.perjob; + +import org.apache.commons.io.Charsets; + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.util.Arrays; +import java.util.List; + +/** + * Date: 2019/12/28 + * Company: www.dtstack.com + * @author maqi + */ +public class ConfigParseUtil { + + public static List parsePathFromStr(String pathStr) throws UnsupportedEncodingException { + String addjarPath = URLDecoder.decode(pathStr, Charsets.UTF_8.toString()); + if (addjarPath.length() > 2) { + addjarPath = addjarPath.substring(1,addjarPath.length()-1).replace("\"",""); + } + return Arrays.asList(addjarPath.split(",")); + } +} diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index bad44bbc9..4c9ff1841 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -39,6 +39,7 @@ import java.io.File; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; @@ -85,7 +86,7 @@ public void init(String yarnConfDir, Configuration flinkConfig, Properties userC } public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJarPath, Options launcherOptions, JobGraph jobGraph) - throws MalformedURLException { + throws MalformedURLException, UnsupportedEncodingException { String flinkConf = StringUtils.isEmpty(launcherOptions.getFlinkconf()) ? DEFAULT_CONF_DIR : launcherOptions.getFlinkconf(); AbstractYarnClusterDescriptor clusterDescriptor = getClusterDescriptor(flinkConfig, yarnConf, flinkConf); @@ -121,6 +122,14 @@ public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJ + " Currently only classpath and shipfile are supported."); } + // add user customized file to shipfile + if (!StringUtils.isBlank(launcherOptions.getAddShipfile())) { + List paths = ConfigParseUtil.parsePathFromStr(launcherOptions.getAddShipfile()); + paths.forEach(path -> { + shipFiles.add(new File(path)); + }); + } + clusterDescriptor.addShipFiles(shipFiles); clusterDescriptor.setName(launcherOptions.getName()); String queue = launcherOptions.getQueue(); diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index 6a525a7e7..02cc60f5a 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -48,13 +48,12 @@ public class PerJobSubmitter { private static final Logger LOG = LoggerFactory.getLogger(PerJobSubmitter.class); public static String submit(Options launcherOptions, JobGraph jobGraph, Configuration flinkConfig) throws Exception { - if (!StringUtils.isBlank(launcherOptions.getAddjar())) { - String addjarPath = URLDecoder.decode(launcherOptions.getAddjar(), Charsets.UTF_8.toString()); - List paths = getJarPaths(addjarPath); - paths.forEach( path -> { - jobGraph.addJar(new Path("file://" + path)); - }); - } + if (!StringUtils.isBlank(launcherOptions.getAddjar())) { + List paths = ConfigParseUtil.parsePathFromStr(launcherOptions.getAddjar()); + paths.forEach(path -> { + jobGraph.addJar(new Path("file://" + path)); + }); + } String confProp = launcherOptions.getConfProp(); confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); From 0ee4a719de38731d515ad91d7f38cca7e6d8aa24 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 30 Mar 2020 19:36:44 +0800 Subject: [PATCH 40/69] rbdsink check --- .../dtstack/flink/sql/sink/rdb/RdbSink.java | 7 +++++- .../rdb/format/RetractJDBCOutputFormat.java | 23 ++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java index c7effea25..c3d96aa6e 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java @@ -50,7 +50,7 @@ * @author maqi */ public abstract class RdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { - + private final String DEFAULT_TEST_CHECK_CONNECTION_SQL = "select 1 from dual"; protected String driverName; protected String dbURL; @@ -101,6 +101,7 @@ public RichSinkFunction createJdbcSinkFunc() { outputFormat.setTableName(tableName); outputFormat.setDbType(dbType); outputFormat.setSchema(schema); + outputFormat.setTestConnectionSql(getCheckConnetionSql()); outputFormat.setDbSink(this); outputFormat.verifyField(); @@ -281,4 +282,8 @@ public DataStreamSink> consumeDataStream(DataStream rows = new ArrayList(); @@ -309,6 +312,7 @@ private void updatePreparedStmt(Row row, PreparedStatement pstmt) throws SQLExce private synchronized void submitExecuteBatch() { try { + triggerConnectionCheck(); this.upload.executeBatch(); dbConn.commit(); } catch (SQLException e) { @@ -324,11 +328,24 @@ private synchronized void submitExecuteBatch() { } } + private void triggerConnectionCheck() { + try { + upload.executeQuery(testConnectionSql); + } catch (SQLException e) { + LOG.error("triggerConnectionCheck failed..", e); + try { + dbConn.close(); + } catch (SQLException e1) { + LOG.error("dbConn close failed..", e); + } + } + } + private void checkConnectionOpen(Connection dbConn) { try { if (dbConn.isClosed()) { LOG.info("db connection reconnect.."); - dbConn= establishConnection(); + dbConn = establishConnection(); upload = dbConn.prepareStatement(insertQuery); this.dbConn = dbConn; } @@ -488,4 +505,8 @@ public List getFullField() { public void fullFieldAdd(String colName) { this.fullField.add(colName); } + + public void setTestConnectionSql(String testConnectionSql) { + this.testConnectionSql = testConnectionSql; + } } From 5e25aabec3d7833a26bf0737f556e701fbea1a73 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 30 Mar 2020 20:25:07 +0800 Subject: [PATCH 41/69] hbase sink doc --- docs/hbaseSink.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/hbaseSink.md b/docs/hbaseSink.md index 8045457a1..3dbd40fa4 100644 --- a/docs/hbaseSink.md +++ b/docs/hbaseSink.md @@ -42,17 +42,7 @@ hbase2.0 | zookeeperSaslClient | zookeeper.sasl.client值|否|true| | securityKrb5Conf | java.security.krb5.conf值|否|| -> kerberos 配置 -> kerberos 配置 - - * hbase.security.authentication = 'kerberos', - * hbase.security.authorization = 'true', - * hbase.master.kerberos.principal = 'hbase/cdh01@DTSTACK.COM', - * hbase.master.keytab.file = 'hbase.keytab', - * hbase.regionserver.keytab.file = 'hbase.keytab', - * hbase.regionserver.kerberos.principal = 'hbase/cdh01@DTSTACK.COM' - * (非必选)java.security.krb5.conf = 'krb5.conf' ## 5.样例： ``` From ebb84cb260bb4de9e6007ad155339d6c075098ef Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 30 Mar 2020 21:31:38 +0800 Subject: [PATCH 42/69] log --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 5 +++++ .../dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java | 2 ++ 2 files changed, 7 insertions(+) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 46b7b9d05..bc56f7d5d 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -87,6 +87,11 @@ public void configure(Configuration parameters) { if (kerberosAuthEnable) { conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); + LOG.info("regionserverKeytabFile: {}",regionserverKeytabFile); + LOG.info("regionserverPrincipal: {}",regionserverPrincipal); + LOG.info("zookeeperSaslClient: {}",zookeeperSaslClient); + LOG.info("zookeeperSaslClient: {}",securityKrb5Conf); + fillSyncKerberosConfig(conf, regionserverKeytabFile, regionserverPrincipal, zookeeperSaslClient, securityKrb5Conf); UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, regionserverPrincipal, regionserverKeytabFile); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java index 4b909dec5..624e3bfc8 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java @@ -52,6 +52,8 @@ public class HbaseConfigUtils { public static final String KEY_ZOOKEEPER_SASL_CLIENT = "zookeeper.sasl.client"; public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { + LOG.info("loginAndReturnUGI principal {}",principal); + LOG.info("loginAndReturnUGI keytab {}",keytab); if (conf == null) { throw new IllegalArgumentException("kerberos conf can not be null"); } From d93f386d579628d0f80f472fad134870b19b26d3 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 30 Mar 2020 21:34:13 +0800 Subject: [PATCH 43/69] modify log --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index bc56f7d5d..71125ed69 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -90,7 +90,7 @@ public void configure(Configuration parameters) { LOG.info("regionserverKeytabFile: {}",regionserverKeytabFile); LOG.info("regionserverPrincipal: {}",regionserverPrincipal); LOG.info("zookeeperSaslClient: {}",zookeeperSaslClient); - LOG.info("zookeeperSaslClient: {}",securityKrb5Conf); + LOG.info("securityKrb5Conf: {}",securityKrb5Conf); fillSyncKerberosConfig(conf, regionserverKeytabFile, regionserverPrincipal, zookeeperSaslClient, securityKrb5Conf); From c28af4b8a58c543b6c1a6075685c7ed31cbb7282 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 30 Mar 2020 21:44:27 +0800 Subject: [PATCH 44/69] add log --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 71125ed69..6d8bfc079 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -127,6 +127,7 @@ public void open(int taskNumber, int numTasks) throws IOException { @Override public void writeRecord(Tuple2 tuple2) { + LOG.info("receive data {},", tuple2.toString()); Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); @@ -158,19 +159,21 @@ public void writeRecord(Tuple2 tuple2) { put.addColumn(cf, qualifier, val); } + try { table.put(put); } catch (IOException e) { - outDirtyRecords.inc(); if (outDirtyRecords.getCount() % dirtyDataPrintFrequency == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ..", record.toString()); LOG.error("", e); } + outDirtyRecords.inc(); } if (outRecords.getCount() % rowLenth == 0) { LOG.info(record.toString()); } + outRecords.inc(); } From 49297c1a4cf3e79cbd431b2081cceed8d92a7dad Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 31 Mar 2020 10:45:29 +0800 Subject: [PATCH 45/69] fix params --- .../dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java index 624e3bfc8..949eefe8f 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java @@ -44,7 +44,7 @@ public class HbaseConfigUtils { public final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; public final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; - public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; + public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "zookeeper.quorum"; public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; From 35e6942372852899175d65d18c8cac275a395ea8 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 31 Mar 2020 14:06:03 +0800 Subject: [PATCH 46/69] hbase param --- .../dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java index 949eefe8f..0fefc638e 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java @@ -44,8 +44,8 @@ public class HbaseConfigUtils { public final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; public final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; - public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "zookeeper.quorum"; - public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; + public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; + public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "zookeeper.znode.parent"; public static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; From 34706d1f2809a81a268330c61203cd91dd0fbd78 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 31 Mar 2020 15:58:43 +0800 Subject: [PATCH 47/69] fix --- .../sql/sink/hbase/HbaseOutputFormat.java | 26 ++++++++++++++++++- .../flink/sql/sink/hbase/HbaseSink.java | 10 +++++++ .../sql/sink/hbase/table/HbaseSinkParser.java | 6 +++++ .../sql/sink/hbase/table/HbaseTableInfo.java | 20 ++++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 6d8bfc079..4dcace310 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -65,6 +65,8 @@ public class HbaseOutputFormat extends MetricOutputFormat { private String regionserverPrincipal; private String securityKrb5Conf; private String zookeeperSaslClient; + private String clientPrincipal; + private String clientKeytabFile; private String[] families; private String[] qualifiers; @@ -92,9 +94,21 @@ public void configure(Configuration parameters) { LOG.info("zookeeperSaslClient: {}",zookeeperSaslClient); LOG.info("securityKrb5Conf: {}",securityKrb5Conf); + LOG.info("clientPrincipal : {}", clientPrincipal); + LOG.info("clientKeytabFile : {}", clientKeytabFile); + fillSyncKerberosConfig(conf, regionserverKeytabFile, regionserverPrincipal, zookeeperSaslClient, securityKrb5Conf); - UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, regionserverPrincipal, regionserverKeytabFile); + if (!StringUtils.isEmpty(clientKeytabFile)) { + clientKeytabFile = System.getProperty("user.dir") + File.separator + clientKeytabFile; + } else { + clientKeytabFile = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; + } + + clientPrincipal = !StringUtils.isEmpty(clientPrincipal) ? clientPrincipal : regionserverPrincipal; + + + UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, clientPrincipal, clientKeytabFile); org.apache.hadoop.conf.Configuration finalConf = conf; conn = userGroupInformation.doAs(new PrivilegedAction() { @Override @@ -288,6 +302,16 @@ public HbaseOutputFormatBuilder setZookeeperSaslClient(String zookeeperSaslClien return this; } + public HbaseOutputFormatBuilder setClientPrincipal(String clientPrincipal) { + format.clientPrincipal = clientPrincipal; + return this; + } + + public HbaseOutputFormatBuilder setClientKeytabFile(String clientKeytabFile) { + format.clientKeytabFile = clientKeytabFile; + return this; + } + public HbaseOutputFormat finish() { Preconditions.checkNotNull(format.host, "zookeeperQuorum should be specified"); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 6bffca2f0..7f8e6e6da 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -60,6 +60,10 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener< protected String regionserverPrincipal; protected String securityKrb5Conf; protected String zookeeperSaslClient; + + private String clientPrincipal; + private String clientKeytabFile; + private int parallelism = -1; public HbaseSink() { @@ -83,6 +87,9 @@ public HbaseSink genStreamSink(TargetTableInfo targetTableInfo) { this.securityKrb5Conf = hbaseTableInfo.getSecurityKrb5Conf(); this.zookeeperSaslClient = hbaseTableInfo.getZookeeperSaslClient(); + this.clientKeytabFile = hbaseTableInfo.getClientKeytabFile(); + this.clientPrincipal = hbaseTableInfo.getClientPrincipal(); + Integer tmpSinkParallelism = hbaseTableInfo.getParallelism(); if (tmpSinkParallelism != null) { setParallelism(tmpSinkParallelism); @@ -137,6 +144,9 @@ public DataStreamSink> consumeDataStream(DataStream hbaseConfig = Maps.newHashMap(); public HbaseTableInfo(){ @@ -200,5 +204,21 @@ public String getZookeeperSaslClient() { public void setZookeeperSaslClient(String zookeeperSaslClient) { this.zookeeperSaslClient = zookeeperSaslClient; } + + public String getClientPrincipal() { + return clientPrincipal; + } + + public void setClientPrincipal(String clientPrincipal) { + this.clientPrincipal = clientPrincipal; + } + + public String getClientKeytabFile() { + return clientKeytabFile; + } + + public void setClientKeytabFile(String clientKeytabFile) { + this.clientKeytabFile = clientKeytabFile; + } } From 08f6bc0e3b5e5bcd8b48a0e4355c43c305d409a5 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 31 Mar 2020 17:24:37 +0800 Subject: [PATCH 48/69] remove principal --- .../sql/sink/hbase/HbaseOutputFormat.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 4dcace310..38b9a1550 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -97,14 +97,9 @@ public void configure(Configuration parameters) { LOG.info("clientPrincipal : {}", clientPrincipal); LOG.info("clientKeytabFile : {}", clientKeytabFile); - fillSyncKerberosConfig(conf, regionserverKeytabFile, regionserverPrincipal, zookeeperSaslClient, securityKrb5Conf); - - if (!StringUtils.isEmpty(clientKeytabFile)) { - clientKeytabFile = System.getProperty("user.dir") + File.separator + clientKeytabFile; - } else { - clientKeytabFile = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; - } + fillSyncKerberosConfig(conf, regionserverPrincipal, zookeeperSaslClient, securityKrb5Conf); + clientKeytabFile = System.getProperty("user.dir") + File.separator + clientKeytabFile; clientPrincipal = !StringUtils.isEmpty(clientPrincipal) ? clientPrincipal : regionserverPrincipal; @@ -340,15 +335,15 @@ public HbaseOutputFormat finish() { } - private void fillSyncKerberosConfig(org.apache.hadoop.conf.Configuration config, String regionserverKeytabFile, String regionserverPrincipal, + private void fillSyncKerberosConfig(org.apache.hadoop.conf.Configuration config, String regionserverPrincipal, String zookeeperSaslClient, String securityKrb5Conf) throws IOException { - if (StringUtils.isEmpty(regionserverKeytabFile)) { - throw new IllegalArgumentException("Must provide regionserverKeytabFile when authentication is Kerberos"); - } - String regionserverKeytabFilePath = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; - LOG.info("regionserverKeytabFilePath:{}",regionserverKeytabFilePath); - config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KEYTAB_FILE, regionserverKeytabFilePath); - config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KEYTAB_FILE, regionserverKeytabFilePath); +// if (StringUtils.isEmpty(regionserverKeytabFile)) { +// throw new IllegalArgumentException("Must provide regionserverKeytabFile when authentication is Kerberos"); +// } +// String regionserverKeytabFilePath = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; +// LOG.info("regionserverKeytabFilePath:{}",regionserverKeytabFilePath); +// config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KEYTAB_FILE, regionserverKeytabFilePath); +// config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KEYTAB_FILE, regionserverKeytabFilePath); if (StringUtils.isEmpty(regionserverPrincipal)) { throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); From aca4b78737b2891aca616ab597b44a42839a4889 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 31 Mar 2020 17:58:55 +0800 Subject: [PATCH 49/69] opt log --- .../sql/sink/hbase/HbaseOutputFormat.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 38b9a1550..c6322c5ce 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -89,13 +89,10 @@ public void configure(Configuration parameters) { if (kerberosAuthEnable) { conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); - LOG.info("regionserverKeytabFile: {}",regionserverKeytabFile); - LOG.info("regionserverPrincipal: {}",regionserverPrincipal); - LOG.info("zookeeperSaslClient: {}",zookeeperSaslClient); - LOG.info("securityKrb5Conf: {}",securityKrb5Conf); - LOG.info("clientPrincipal : {}", clientPrincipal); - LOG.info("clientKeytabFile : {}", clientKeytabFile); + LOG.info("kerberos config:{}", this.toString()); + Preconditions.checkArgument(!StringUtils.isEmpty(clientPrincipal), " clientPrincipal not null!"); + Preconditions.checkArgument(!StringUtils.isEmpty(clientKeytabFile), " clientKeytabFile not null!"); fillSyncKerberosConfig(conf, regionserverPrincipal, zookeeperSaslClient, securityKrb5Conf); @@ -337,14 +334,6 @@ public HbaseOutputFormat finish() { private void fillSyncKerberosConfig(org.apache.hadoop.conf.Configuration config, String regionserverPrincipal, String zookeeperSaslClient, String securityKrb5Conf) throws IOException { -// if (StringUtils.isEmpty(regionserverKeytabFile)) { -// throw new IllegalArgumentException("Must provide regionserverKeytabFile when authentication is Kerberos"); -// } -// String regionserverKeytabFilePath = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; -// LOG.info("regionserverKeytabFilePath:{}",regionserverKeytabFilePath); -// config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KEYTAB_FILE, regionserverKeytabFilePath); -// config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KEYTAB_FILE, regionserverKeytabFilePath); - if (StringUtils.isEmpty(regionserverPrincipal)) { throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); } @@ -365,5 +354,16 @@ private void fillSyncKerberosConfig(org.apache.hadoop.conf.Configuration config, } } - + @Override + public String toString() { + return "HbaseOutputFormat kerberos{" + + "kerberosAuthEnable=" + kerberosAuthEnable + + ", regionserverKeytabFile='" + regionserverKeytabFile + '\'' + + ", regionserverPrincipal='" + regionserverPrincipal + '\'' + + ", securityKrb5Conf='" + securityKrb5Conf + '\'' + + ", zookeeperSaslClient='" + zookeeperSaslClient + '\'' + + ", clientPrincipal='" + clientPrincipal + '\'' + + ", clientKeytabFile='" + clientKeytabFile + '\'' + + '}'; + } } From 1a4f788a82565658bfdd6c633a9ee6e1211dd52f Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 1 Apr 2020 12:35:41 +0800 Subject: [PATCH 50/69] fix pattern --- .../cassandra/table/CassandraSideParser.java | 7 ++- .../flink/sql/table/AbsSideTableParser.java | 2 +- .../flink/sql/table/AbsSourceParser.java | 21 ++++----- .../flink/sql/table/AbsTableParser.java | 45 ++++++++++++++++--- .../sql/side/hbase/table/HbaseSideParser.java | 10 ++--- .../source/kafka/table/KafkaSourceParser.java | 37 --------------- .../source/kafka/table/KafkaSourceParser.java | 36 --------------- .../source/kafka/table/KafkaSourceParser.java | 40 ----------------- .../source/kafka/table/KafkaSourceParser.java | 41 ----------------- 9 files changed, 55 insertions(+), 184 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java index 6403a225b..1727ae625 100644 --- a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java +++ b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java @@ -65,9 +65,8 @@ public class CassandraSideParser extends AbsSideTableParser { public static final String POOL_TIMEOUT_MILLIS_KEY = "poolTimeoutMillis"; - static { - keyPatternMap.put(SIDE_SIGN_KEY, SIDE_TABLE_SIGN); - keyHandlerMap.put(SIDE_SIGN_KEY, CassandraSideParser::dealSideSign); + public CassandraSideParser() { + addParserHandler(SIDE_SIGN_KEY, SIDE_TABLE_SIGN, this::dealSideSign); } @Override @@ -94,6 +93,6 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map keyPatternMap = Maps.newHashMap(); + public Map keyPatternMap = Maps.newHashMap(); - public static Map keyHandlerMap = Maps.newHashMap(); + public Map keyHandlerMap = Maps.newHashMap(); - static { - keyPatternMap.put(PRIMARY_KEY, primaryKeyPattern); - keyHandlerMap.put(PRIMARY_KEY, AbsTableParser::dealPrimaryKey); + public AbsTableParser() { + addParserHandler(PRIMARY_KEY, primaryKeyPattern, this::dealPrimaryKey); + addParserHandler(NEST_JSON_FIELD_KEY, nestJsonFieldKeyPattern, this::dealNestField); } protected boolean fieldNameNeedsUpperCase() { @@ -112,7 +116,7 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ tableInfo.finish(); } - public static void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ + public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ String primaryFields = matcher.group(1).trim(); if (primaryFields != null) { String[] splitArry = primaryFields.split(","); @@ -123,8 +127,37 @@ public static void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ tableInfo.setPrimaryKeys(Lists.newArrayList()); } + /** + * add parser for alias field + * @param matcher + * @param tableInfo + */ + protected void dealNestField(Matcher matcher, TableInfo tableInfo) { + String physicalField = matcher.group(1); + Preconditions.checkArgument(!physicalFieldFunPattern.matcher(physicalField).find(), + "No need to add data types when using functions, The correct way is : strLen(name) as nameSize, "); + + String fieldType = matcher.group(3); + String mappingField = matcher.group(4); + Class fieldClass = dbTypeConvertToJavaType(fieldType); + boolean notNull = matcher.group(5) != null; + TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); + fieldExtraInfo.setNotNull(notNull); + + tableInfo.addPhysicalMappings(mappingField, physicalField); + tableInfo.addField(mappingField); + tableInfo.addFieldClass(fieldClass); + tableInfo.addFieldType(fieldType); + tableInfo.addFieldExtraInfo(fieldExtraInfo); + } + public Class dbTypeConvertToJavaType(String fieldType) { return ClassUtil.stringConvertClass(fieldType); } + protected void addParserHandler(String parserName, Pattern pattern, ITableFieldDealHandler handler) { + keyPatternMap.put(parserName, pattern); + keyHandlerMap.put(parserName, handler); + } + } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 2b785c335..2046a94c1 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -24,14 +24,12 @@ import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.MathUtil; -import scala.collection.mutable.HashMap$; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; @@ -59,12 +57,12 @@ public class HbaseSideParser extends AbsSideTableParser { public static final String CACHE = "cache"; - static { - keyPatternMap.put(FIELD_KEY, FIELD_PATTERN); - keyHandlerMap.put(FIELD_KEY, HbaseSideParser::dealField); + public HbaseSideParser() { + addParserHandler(FIELD_KEY, FIELD_PATTERN, this::dealField); } + @Override public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseSideTableInfo hbaseTableInfo = new HbaseSideTableInfo(); @@ -91,7 +89,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map" + mappingField + " Class: " + fieldClass.toString()); - } - } - @Override public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { KafkaSourceTableInfo kafkaSourceTableInfo = new KafkaSourceTableInfo(); diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index aa02ebd30..71c103bd5 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -22,14 +22,9 @@ import com.dtstack.flink.sql.table.AbsSourceParser; import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.MathUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * Reason: @@ -40,37 +35,6 @@ public class KafkaSourceParser extends AbsSourceParser { - private static final Logger LOG = LoggerFactory.getLogger(KafkaSourceParser.class); - - private static final String KAFKA_NEST_FIELD_KEY = "nestFieldKey"; - - private static Pattern kafkaNestFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(\\w+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); - - static { - keyPatternMap.put(KAFKA_NEST_FIELD_KEY, kafkaNestFieldKeyPattern); - - keyHandlerMap.put(KAFKA_NEST_FIELD_KEY, KafkaSourceParser::dealNestField); - } - - static void dealNestField(Matcher matcher, TableInfo tableInfo) { - String physicalField = matcher.group(1); - String fieldType = matcher.group(3); - String mappingField = matcher.group(4); - Class fieldClass= ClassUtil.stringConvertClass(fieldType); - boolean notNull = matcher.group(5) != null; - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); - fieldExtraInfo.setNotNull(notNull); - - tableInfo.addPhysicalMappings(mappingField, physicalField); - tableInfo.addField(mappingField); - tableInfo.addFieldClass(fieldClass); - tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(fieldExtraInfo); - if(LOG.isInfoEnabled()){ - LOG.info(physicalField + "--->" + mappingField + " Class: " + fieldClass.toString()); - } - } - @Override public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index b5c733950..422981850 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -22,14 +22,9 @@ import com.dtstack.flink.sql.table.AbsSourceParser; import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.MathUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * Reason: @@ -40,41 +35,6 @@ public class KafkaSourceParser extends AbsSourceParser { - private static final Logger LOG = LoggerFactory.getLogger(KafkaSourceParser.class); - - private static final String KAFKA_NEST_FIELD_KEY = "nestFieldKey"; - - private static Pattern kafkaNestFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(\\w+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); - - static { - keyPatternMap.put(KAFKA_NEST_FIELD_KEY, kafkaNestFieldKeyPattern); - - keyHandlerMap.put(KAFKA_NEST_FIELD_KEY, KafkaSourceParser::dealNestField); - } - - /** - * add parser for alias field - * @param matcher - * @param tableInfo - */ - static void dealNestField(Matcher matcher, TableInfo tableInfo) { - String physicalField = matcher.group(1); - String fieldType = matcher.group(3); - String mappingField = matcher.group(4); - Class fieldClass= ClassUtil.stringConvertClass(fieldType); - boolean notNull = matcher.group(5) != null; - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); - fieldExtraInfo.setNotNull(notNull); - - tableInfo.addPhysicalMappings(mappingField, physicalField); - tableInfo.addField(mappingField); - tableInfo.addFieldClass(fieldClass); - tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(fieldExtraInfo); - if(LOG.isInfoEnabled()){ - LOG.info(physicalField + "--->" + mappingField + " Class: " + fieldClass.toString()); - } - } @Override public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 059c958d9..b5165dcb2 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -22,14 +22,9 @@ import com.dtstack.flink.sql.table.AbsSourceParser; import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.MathUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * Reason: @@ -40,42 +35,6 @@ public class KafkaSourceParser extends AbsSourceParser { - private static final Logger LOG = LoggerFactory.getLogger(KafkaSourceParser.class); - - private static final String KAFKA_NEST_FIELD_KEY = "nestFieldKey"; - - private static Pattern kafkaNestFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(\\w+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); - - static { - keyPatternMap.put(KAFKA_NEST_FIELD_KEY, kafkaNestFieldKeyPattern); - - keyHandlerMap.put(KAFKA_NEST_FIELD_KEY, KafkaSourceParser::dealNestField); - } - - /** - * add parser for alias field - * @param matcher - * @param tableInfo - */ - static void dealNestField(Matcher matcher, TableInfo tableInfo) { - String physicalField = matcher.group(1); - String fieldType = matcher.group(3); - String mappingField = matcher.group(4); - Class fieldClass= ClassUtil.stringConvertClass(fieldType); - boolean notNull = matcher.group(5) != null; - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); - fieldExtraInfo.setNotNull(notNull); - - tableInfo.addPhysicalMappings(mappingField, physicalField); - tableInfo.addField(mappingField); - tableInfo.addFieldClass(fieldClass); - tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(fieldExtraInfo); - if(LOG.isInfoEnabled()){ - LOG.info(physicalField + "--->" + mappingField + " Class: " + fieldClass.toString()); - } - } - @Override public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { From e4bd24c018420233139cbff8842ed041a855e509 Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Wed, 1 Apr 2020 16:40:01 +0800 Subject: [PATCH 51/69] delete conn check --- .../sink/rdb/format/RetractJDBCOutputFormat.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java index 1d7185a78..47a35a701 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java @@ -312,7 +312,6 @@ private void updatePreparedStmt(Row row, PreparedStatement pstmt) throws SQLExce private synchronized void submitExecuteBatch() { try { - triggerConnectionCheck(); this.upload.executeBatch(); dbConn.commit(); } catch (SQLException e) { @@ -328,19 +327,6 @@ private synchronized void submitExecuteBatch() { } } - private void triggerConnectionCheck() { - try { - upload.executeQuery(testConnectionSql); - } catch (SQLException e) { - LOG.error("triggerConnectionCheck failed..", e); - try { - dbConn.close(); - } catch (SQLException e1) { - LOG.error("dbConn close failed..", e); - } - } - } - private void checkConnectionOpen(Connection dbConn) { try { if (dbConn.isClosed()) { From 218dae01d8ebb161aa1a6e3e430fec962de06bf8 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 1 Apr 2020 17:05:42 +0800 Subject: [PATCH 52/69] connection check --- .../dtstack/flink/sql/sink/rdb/RdbSink.java | 5 --- .../rdb/format/RetractJDBCOutputFormat.java | 32 +++++++------------ 2 files changed, 11 insertions(+), 26 deletions(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java index c3d96aa6e..7d74792f9 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java @@ -50,7 +50,6 @@ * @author maqi */ public abstract class RdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { - private final String DEFAULT_TEST_CHECK_CONNECTION_SQL = "select 1 from dual"; protected String driverName; protected String dbURL; @@ -101,7 +100,6 @@ public RichSinkFunction createJdbcSinkFunc() { outputFormat.setTableName(tableName); outputFormat.setDbType(dbType); outputFormat.setSchema(schema); - outputFormat.setTestConnectionSql(getCheckConnetionSql()); outputFormat.setDbSink(this); outputFormat.verifyField(); @@ -282,8 +280,5 @@ public DataStreamSink> consumeDataStream(DataStream rows = new ArrayList(); @@ -188,7 +188,6 @@ public void writeRecord(Tuple2 tuple2) { private void insertWrite(Row row) { - checkConnectionOpen(dbConn); try { if (batchNum == 1) { writeSingleRecord(row); @@ -312,7 +311,7 @@ private void updatePreparedStmt(Row row, PreparedStatement pstmt) throws SQLExce private synchronized void submitExecuteBatch() { try { - triggerConnectionCheck(); + checkConnectionOpen(); this.upload.executeBatch(); dbConn.commit(); } catch (SQLException e) { @@ -328,22 +327,15 @@ private synchronized void submitExecuteBatch() { } } - private void triggerConnectionCheck() { - try { - upload.executeQuery(testConnectionSql); - } catch (SQLException e) { - LOG.error("triggerConnectionCheck failed..", e); - try { - dbConn.close(); - } catch (SQLException e1) { - LOG.error("dbConn close failed..", e); - } - } - } - private void checkConnectionOpen(Connection dbConn) { + private void checkConnectionOpen() { + checkTimes++; + if (checkTimes % connectionCheckFrequency != 0) { + return; + } + LOG.warn("db connection check.."); try { - if (dbConn.isClosed()) { + if (dbConn.isClosed() || !dbConn.isValid(60)) { LOG.info("db connection reconnect.."); dbConn = establishConnection(); upload = dbConn.prepareStatement(insertQuery); @@ -354,6 +346,7 @@ private void checkConnectionOpen(Connection dbConn) { } catch (ClassNotFoundException e) { LOG.error("load jdbc class error when reconnect db..", e); } + checkTimes = 0; } /** @@ -506,7 +499,4 @@ public void fullFieldAdd(String colName) { this.fullField.add(colName); } - public void setTestConnectionSql(String testConnectionSql) { - this.testConnectionSql = testConnectionSql; - } } From 34b76151cb544e0d55676018b848bed40212d263 Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 2 Apr 2020 15:49:16 +0800 Subject: [PATCH 53/69] remove log --- JAAS4352423491229344111.conf | 7 +++++++ .../dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 8 +++----- 2 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 JAAS4352423491229344111.conf diff --git a/JAAS4352423491229344111.conf b/JAAS4352423491229344111.conf new file mode 100644 index 000000000..535e689b0 --- /dev/null +++ b/JAAS4352423491229344111.conf @@ -0,0 +1,7 @@ +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + useTicketCache=false + keyTab="/xxx/hdfs.keytab" + principal="hdfs@TDH"; +}; diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index c6322c5ce..946dbdbaf 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -133,7 +133,9 @@ public void open(int taskNumber, int numTasks) throws IOException { @Override public void writeRecord(Tuple2 tuple2) { - LOG.info("receive data {},", tuple2.toString()); + if (outRecords.getCount() == 0 || outRecords.getCount() % rowLenth == 0) { + LOG.info(tuple2.toString()); + } Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); @@ -176,10 +178,6 @@ public void writeRecord(Tuple2 tuple2) { outDirtyRecords.inc(); } - if (outRecords.getCount() % rowLenth == 0) { - LOG.info(record.toString()); - } - outRecords.inc(); } From e3e81213657823120699e8ac96b2559cec0ff73c Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 2 Apr 2020 15:51:05 +0800 Subject: [PATCH 54/69] remove jaas.conf --- JAAS4352423491229344111.conf | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 JAAS4352423491229344111.conf diff --git a/JAAS4352423491229344111.conf b/JAAS4352423491229344111.conf deleted file mode 100644 index 535e689b0..000000000 --- a/JAAS4352423491229344111.conf +++ /dev/null @@ -1,7 +0,0 @@ -Client { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - useTicketCache=false - keyTab="/xxx/hdfs.keytab" - principal="hdfs@TDH"; -}; From fe5bc61de6977d1532a18e3ae140dfb5b999ddf3 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 20 Apr 2020 16:27:08 +0800 Subject: [PATCH 55/69] hbase kerberos renew --- .../flink/sql/sink/hbase/HbaseOutputFormat.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 946dbdbaf..4ffe1349a 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -27,7 +27,10 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; +import org.apache.hadoop.hbase.AuthUtil; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -80,6 +83,9 @@ public class HbaseOutputFormat extends MetricOutputFormat { private static int rowLenth = 1000; private static int dirtyDataPrintFrequency = 1000; + private transient ChoreService choreService; + + @Override public void configure(Configuration parameters) { @@ -106,6 +112,12 @@ public void configure(Configuration parameters) { @Override public Connection run() { try { + ScheduledChore authChore = AuthUtil.getAuthChore(finalConf); + if (authChore != null) { + ChoreService choreService = new ChoreService("hbaseKerberosSink"); + choreService.scheduleChore(authChore); + } + return ConnectionFactory.createConnection(finalConf); } catch (IOException e) { LOG.error("Get connection fail with config:{}", finalConf); @@ -214,6 +226,10 @@ public void close() throws IOException { conn.close(); conn = null; } + + if (choreService != null) { + choreService.shutdown(); + } } private HbaseOutputFormat() { From 03372508892d6c9e392183a26734d19281cef8c3 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 20 Apr 2020 21:11:33 +0800 Subject: [PATCH 56/69] add hbase config --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 2 ++ .../dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 4ffe1349a..cf11cd62d 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -105,6 +105,8 @@ public void configure(Configuration parameters) { clientKeytabFile = System.getProperty("user.dir") + File.separator + clientKeytabFile; clientPrincipal = !StringUtils.isEmpty(clientPrincipal) ? clientPrincipal : regionserverPrincipal; + conf.set(HbaseConfigUtils.KEY_HBASE_CLIENT_KEYTAB_FILE, clientKeytabFile); + conf.set(HbaseConfigUtils.KEY_HBASE_CLIENT_KERBEROS_PRINCIPAL, clientPrincipal); UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, clientPrincipal, clientKeytabFile); org.apache.hadoop.conf.Configuration finalConf = conf; diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java index 0fefc638e..0ec42f192 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/utils/HbaseConfigUtils.java @@ -47,6 +47,9 @@ public class HbaseConfigUtils { public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "zookeeper.znode.parent"; + public final static String KEY_HBASE_CLIENT_KEYTAB_FILE = "hbase.client.keytab.file"; + public final static String KEY_HBASE_CLIENT_KERBEROS_PRINCIPAL = "hbase.client.kerberos.principal"; + public static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; public static final String KEY_ZOOKEEPER_SASL_CLIENT = "zookeeper.sasl.client"; From b77c563399655663efa4bae219e8c75bfaaf74cc Mon Sep 17 00:00:00 2001 From: chuixue Date: Wed, 22 Apr 2020 20:43:19 +0800 Subject: [PATCH 57/69] hbase fix --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index cf11cd62d..f335d3bf8 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -116,7 +116,7 @@ public Connection run() { try { ScheduledChore authChore = AuthUtil.getAuthChore(finalConf); if (authChore != null) { - ChoreService choreService = new ChoreService("hbaseKerberosSink"); + choreService = new ChoreService("hbaseKerberosSink"); choreService.scheduleChore(authChore); } From 42f7e94faf28f4128e68d572db47a3c3712495bc Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 23 Apr 2020 13:54:05 +0800 Subject: [PATCH 58/69] hbase side empty string --- .../java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 378bc9d66..453da41a8 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -146,7 +146,7 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = input.getField(conValIndex); - if(equalObj == null){ + if(equalObj == null || "".equals(equalObj.toString())){ dealMissKey(input, resultFuture); return; } From b1ec3c170932989f6b2eda46fe519c667df5b460 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 28 Apr 2020 13:04:33 +0800 Subject: [PATCH 59/69] hbbase super open --- .../java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 453da41a8..61bf2a82d 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -90,6 +90,7 @@ public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List hbaseConfig = hbaseSideTableInfo.getHbaseConfig(); From 266b7c0162630ae2806c2a59417926484a18db47 Mon Sep 17 00:00:00 2001 From: chuixue Date: Tue, 28 Apr 2020 20:32:24 +0800 Subject: [PATCH 60/69] hbase async cache --- .../java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 2 +- .../sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 61bf2a82d..433115033 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -166,7 +166,7 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except return; }else if(ECacheContentType.SingleLine == val.getType()){ try { - Row row = fillData(input, val); + Row row = fillData(input, val.getContent()); resultFuture.complete(Collections.singleton(row)); } catch (Exception e) { dealFillDataError(resultFuture, e, input, val); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index 0cf9eb8c7..ea8af6ac9 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -94,7 +94,7 @@ public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFu Row row = fillData(input, sideVal); if(openCache){ - sideCache.putCache(rowKeyStr, CacheObj.buildCacheObj(ECacheContentType.SingleLine, row)); + sideCache.putCache(rowKeyStr, CacheObj.buildCacheObj(ECacheContentType.SingleLine, sideVal)); } resultFuture.complete(Collections.singleton(row)); From 0cc0a73100c18f99803a07845ade50eea1f349ed Mon Sep 17 00:00:00 2001 From: chuixue Date: Tue, 28 Apr 2020 20:40:56 +0800 Subject: [PATCH 61/69] hbase cache conf not parser --- .../com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 2046a94c1..3e8c1a082 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -67,6 +67,7 @@ public HbaseSideParser() { public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseSideTableInfo hbaseTableInfo = new HbaseSideTableInfo(); hbaseTableInfo.setName(tableName); + parseCacheProp(hbaseTableInfo, props); parseFieldsInfo(fieldsInfo, hbaseTableInfo); hbaseTableInfo.setTableName((String) props.get(TABLE_NAME_KEY.toLowerCase())); hbaseTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(PARALLELISM_KEY.toLowerCase()))); From 9d2f5a500cd28b1671bf4fe6b33c2d6db2d72c57 Mon Sep 17 00:00:00 2001 From: chuixue Date: Tue, 28 Apr 2020 20:58:25 +0800 Subject: [PATCH 62/69] hbase async miss rk not cache --- .../sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index ea8af6ac9..e44edf519 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -105,9 +105,9 @@ public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFu dealMissKey(input, resultFuture); - if(openCache){ - sideCache.putCache(rowKeyStr, CacheMissVal.getMissKeyObj()); - } + // if(openCache){ + // sideCache.putCache(rowKeyStr, CacheMissVal.getMissKeyObj()); + // } } }catch (Exception e){ resultFuture.completeExceptionally(e); From f2c1b93f43b7d9e8e6b9d9556eaa55c4de8d51f9 Mon Sep 17 00:00:00 2001 From: chuixue Date: Wed, 29 Apr 2020 13:55:38 +0800 Subject: [PATCH 63/69] drop log --- .../side/hbase/rowkeydealer/RowKeyEqualModeDealer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index e44edf519..19c039f08 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -84,10 +84,10 @@ public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFu List sideVal = Lists.newArrayList(); for(String key : colNames){ Object val = sideMap.get(key); - if(val == null){ - System.out.println("can't get data with column " + key); - LOG.error("can't get data with column " + key); - } + // if(val == null){ + // System.out.println("can't get data with column " + key); + // LOG.error("can't get data with column " + key); + // } sideVal.add(val); } From 911dc149f2717ff5b177bf92e91282965719598d Mon Sep 17 00:00:00 2001 From: chuixue Date: Wed, 29 Apr 2020 21:30:44 +0800 Subject: [PATCH 64/69] hbase decimal --- .../com/dtstack/flink/sql/util/MathUtil.java | 17 +++++++++++++++++ .../flink/sql/side/hbase/utils/HbaseUtils.java | 4 ++++ 2 files changed, 21 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index 1a590d9f3..5e19b6976 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -21,6 +21,7 @@ package com.dtstack.flink.sql.util; import java.math.BigDecimal; +import java.math.BigInteger; /** * Date: 2017/4/21 @@ -179,4 +180,20 @@ public static Short getShort(Object obj) { throw new RuntimeException("not support type of " + obj.getClass() + " convert to Short."); } + + public static BigDecimal getBigDecimal(Object obj) { + if (obj == null) { + return null; + } + if (obj instanceof String) { + return new BigDecimal((String) obj); + } else if (obj instanceof BigDecimal) { + return (BigDecimal) obj; + } else if (obj instanceof BigInteger) { + return new BigDecimal((BigInteger) obj); + } else if (obj instanceof Number) { + return BigDecimal.valueOf(((Number) obj).doubleValue()); + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to BigDecimal."); + } } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java index 098d967d6..ae7aa8517 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java @@ -68,6 +68,10 @@ public static Object convertByte(byte[] hbaseData, String type){ case "double": return MathUtil.getDoubleVal(data); + case "decimal": + case "decimalunsigned": + return MathUtil.getBigDecimal(data); + default: throw new RuntimeException("not support type of " + type); } From 6fbf4542085b66f1ff5f6ef952d79c3c37e0577d Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 30 Apr 2020 09:50:43 +0800 Subject: [PATCH 65/69] open miss rowkey cache --- .../sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index 19c039f08..04fb07bae 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -105,9 +105,9 @@ public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFu dealMissKey(input, resultFuture); - // if(openCache){ - // sideCache.putCache(rowKeyStr, CacheMissVal.getMissKeyObj()); - // } + if (openCache) { + sideCache.putCache(rowKeyStr, CacheMissVal.getMissKeyObj()); + } } }catch (Exception e){ resultFuture.completeExceptionally(e); From a62453ac55cffe95e70c015ab325a13fc4727f3d Mon Sep 17 00:00:00 2001 From: chuixue Date: Fri, 8 May 2020 14:40:26 +0800 Subject: [PATCH 66/69] log miss --- .../flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java index ee1d7ce0f..8fb2d5c6f 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java @@ -213,7 +213,7 @@ private void writeSingleRecord(Row row) { } catch (SQLException e) { outDirtyRecords.inc(); if (outDirtyRecords.getCount() == 1 || outDirtyRecords.getCount() % dirtyDataPrintFrequency == 0 || LOG.isDebugEnabled()) { - LOG.error("record insert failed ..", row.toString()); + LOG.error("record insert failed .. {}", row.toString()); LOG.error("", e); } } From 5f16fa73979f53dc071b93275c18bfbe9d938ec7 Mon Sep 17 00:00:00 2001 From: chuixue Date: Fri, 19 Jun 2020 12:32:53 +0800 Subject: [PATCH 67/69] add sink sql log --- .../flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java | 1 + 1 file changed, 1 insertion(+) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java index 8fb2d5c6f..610be83da 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java @@ -115,6 +115,7 @@ public void open(int taskNumber, int numTasks) throws IOException { insertQuery = dbSink.buildUpdateSql(schema , tableName, Arrays.asList(dbSink.getFieldNames()), realIndexes, fullField); } upload = dbConn.prepareStatement(insertQuery); + LOG.info("sink sql is {} ", insertQuery); } else { throw new SQLException("Table " + tableName + " doesn't exist"); } From ab2110f97068e8d500b9560ef3dda06446038acc Mon Sep 17 00:00:00 2001 From: chuixue Date: Wed, 8 Jul 2020 15:52:39 +0800 Subject: [PATCH 68/69] rdb sink recreate ps --- .../flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java index 610be83da..859bbdb9c 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java @@ -336,12 +336,16 @@ private void checkConnectionOpen() { } LOG.warn("db connection check.."); try { - if (dbConn.isClosed() || !dbConn.isValid(60)) { + if (dbConn == null || dbConn.isClosed() || !dbConn.isValid(60)) { LOG.info("db connection reconnect.."); dbConn = establishConnection(); upload = dbConn.prepareStatement(insertQuery); this.dbConn = dbConn; } + if(null == upload || upload.isClosed()){ + LOG.warn("prepareStatement recreate.."); + upload = dbConn.prepareStatement(insertQuery); + } } catch (SQLException e) { LOG.error("check connection open failed..", e); } catch (ClassNotFoundException e) { From ee97ece33663cf7ee4707af4fb9a5783b2595c86 Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 10 Sep 2020 15:26:41 +0800 Subject: [PATCH 69/69] Oceanbase Sink null cover bug fix --- launcher/src/main/resources/log4j.properties | 7 ++++ .../sql/sink/oceanbase/OceanbaseSink.java | 37 +++++++++++++------ 2 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 launcher/src/main/resources/log4j.properties diff --git a/launcher/src/main/resources/log4j.properties b/launcher/src/main/resources/log4j.properties new file mode 100644 index 000000000..44e3a5ef2 --- /dev/null +++ b/launcher/src/main/resources/log4j.properties @@ -0,0 +1,7 @@ +#控制台输出： +log4j.rootLogger = INFO,consoleAppender +log4j.appender.consoleAppender = org.apache.log4j.ConsoleAppender +log4j.appender.console.Target = System.out +log4j.appender.consoleAppender.layout = org.apache.log4j.PatternLayout +log4j.appender.consoleAppender.layout.ConversionPattern =%d %-5p %m %n +log4j.appender.consoleAppender.ImmediateFlush = true \ No newline at end of file diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/OceanbaseSink.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/OceanbaseSink.java index 13bbee17f..b4e741c68 100644 --- a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/OceanbaseSink.java +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/OceanbaseSink.java @@ -23,9 +23,12 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; +import org.apache.commons.lang3.StringUtils; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Date: 2020/2/03 @@ -60,20 +63,30 @@ public RetractJDBCOutputFormat getOutputFormat() { return new RetractJDBCOutputFormat(); } - public void buildInsertSql(String tableName, List fields){ - String sqlTmp = "replace into " + tableName + " (${fields}) values (${placeholder})"; - String fieldsStr = ""; - String placeholder = ""; + public void buildInsertSql(String tableName, List fields) { + String[] fieldsName = fields.toArray(new String[fields.size()]); - for (String fieldName : fields) { - fieldsStr += ",`" + fieldName + "`"; - placeholder += ",?"; - } + String updateClause = Arrays.stream(fieldsName).map(f -> quoteIdentifier(f) + + "=IFNULL(VALUES(" + quoteIdentifier(f) + ")," + quoteIdentifier(f) + ")") + .collect(Collectors.joining(",")); - fieldsStr = fieldsStr.replaceFirst(",", ""); - placeholder = placeholder.replaceFirst(",", ""); + this.sql = getInsertIntoStatement("", tableName, fieldsName, null) + + " ON DUPLICATE KEY UPDATE " + updateClause; + } + + private String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } - sqlTmp = sqlTmp.replace("${fields}", fieldsStr).replace("${placeholder}", placeholder); - this.sql = sqlTmp; + private String getInsertIntoStatement(String schema, String tableName, String[] fieldNames, String[] partitionFields) { + String schemaInfo = StringUtils.isEmpty(schema) ? "" : quoteIdentifier(schema) + "."; + String columns = Arrays.stream(fieldNames) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + String placeholders = Arrays.stream(fieldNames) + .map(f -> "?") + .collect(Collectors.joining(", ")); + return "INSERT INTO " + schemaInfo + quoteIdentifier(tableName) + + "(" + columns + ")" + " VALUES (" + placeholders + ")"; } }