From aeb51e8df03e84a15ba928d808b08c1d1f0a4e18 Mon Sep 17 00:00:00 2001 From: zhouxiongjia <719216473@qq.com> Date: Thu, 26 Aug 2021 16:18:44 +0800 Subject: [PATCH] fix some bugs --- src/main/java/com/gauss/GaussLauncher.java | 8 +++++ .../com/gauss/applier/CheckRecordApplier.java | 11 ++++++- .../MultiThreadCheckRecordApplier.java | 1 + .../com/gauss/common/db/sql/SqlTemplate.java | 32 ++++++++++++------- .../com/gauss/common/utils/GaussUtils.java | 2 ++ .../gauss/comparer/GaussRecordComparer.java | 8 +++-- .../com/gauss/controller/GaussInstance.java | 3 ++ .../gauss/preparer/GaussRecordPreparer.java | 9 +++--- 8 files changed, 54 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/gauss/GaussLauncher.java b/src/main/java/com/gauss/GaussLauncher.java index b409ced..93bbc76 100644 --- a/src/main/java/com/gauss/GaussLauncher.java +++ b/src/main/java/com/gauss/GaussLauncher.java @@ -1,6 +1,8 @@ package com.gauss; import java.io.FileInputStream; +import java.text.SimpleDateFormat; +import java.util.Date; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.lang.StringUtils; @@ -8,12 +10,14 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.gauss.common.utils.GaussUtils; import com.gauss.controller.GaussController; public class GaussLauncher { private static final String CLASSPATH_URL_PREFIX = "classpath:"; private static final Logger logger = LoggerFactory.getLogger(GaussLauncher.class); + private static final Logger summaryLogger = LoggerFactory.getLogger("summary"); public static void main(String[] args) throws Throwable { try { @@ -27,6 +31,7 @@ public class GaussLauncher { } logger.info("## start the DataChecker."); + summaryLogger.info("Start time : " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n"); final GaussController controller = new GaussController(config); controller.start(); logger.info("## the DataChecker is running now ......"); @@ -41,6 +46,7 @@ public class GaussLauncher { logger.warn("## something goes wrong when stopping DataChecker:\n{}", ExceptionUtils.getFullStackTrace(e)); } finally { + summaryLogger.info("End time : " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n"); logger.info("## DataChecker is down."); } } @@ -54,10 +60,12 @@ public class GaussLauncher { if (controller.isStart()) { controller.stop(); } + summaryLogger.info("End time : " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n"); logger.info("## DataChecker is down."); } catch (Throwable e) { logger.error("## Something goes wrong when starting up the DataChecker:\n{}", ExceptionUtils.getFullStackTrace(e)); + summaryLogger.info("End time : " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n"); System.exit(0); } } diff --git a/src/main/java/com/gauss/applier/CheckRecordApplier.java b/src/main/java/com/gauss/applier/CheckRecordApplier.java index eaedf1a..71d6197 100644 --- a/src/main/java/com/gauss/applier/CheckRecordApplier.java +++ b/src/main/java/com/gauss/applier/CheckRecordApplier.java @@ -15,6 +15,7 @@ import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.ConnectionCallback; import org.springframework.jdbc.core.JdbcTemplate; +import com.gauss.common.db.meta.Table; import com.gauss.common.lifecycle.AbstractGaussLifeCycle; import com.gauss.common.model.GaussContext; import com.gauss.common.utils.GaussUtils; @@ -34,6 +35,12 @@ public class CheckRecordApplier extends AbstractGaussLifeCycle implements Record public CheckRecordApplier(GaussContext context, int query_dop) { this.context = context; + this.query_dop = query_dop; + } + + @Override + public void start() { + super.start(); } public void apply(List records) throws GaussException { @@ -58,7 +65,9 @@ public class CheckRecordApplier extends AbstractGaussLifeCycle implements Record StringBuilder buffer = new StringBuilder(); records.stream().forEach((String record) -> {buffer.append(record).append(SEPARATOR);}); - String sql = "copy " + context.getTableMeta().getFullName() + "_dataCheckerA " + "from stdin"; + Table tableMeta = context.getTableMeta(); + String compareTableName = tableMeta.getSchema() + ".A" + tableMeta.getName(); + String sql = "copy " + compareTableName + "_dataCheckerA " + "from stdin"; BaseConnection baseConn = (BaseConnection) (connection.getMetaData().getConnection()); CopyManager cp = new CopyManager(baseConn); StringReader reader = new StringReader(buffer.toString()); diff --git a/src/main/java/com/gauss/applier/MultiThreadCheckRecordApplier.java b/src/main/java/com/gauss/applier/MultiThreadCheckRecordApplier.java index 1dd1ecf..5991343 100644 --- a/src/main/java/com/gauss/applier/MultiThreadCheckRecordApplier.java +++ b/src/main/java/com/gauss/applier/MultiThreadCheckRecordApplier.java @@ -33,6 +33,7 @@ public class MultiThreadCheckRecordApplier extends CheckRecordApplier { this.executor = executor; } + @Override public void start() { super.start(); diff --git a/src/main/java/com/gauss/common/db/sql/SqlTemplate.java b/src/main/java/com/gauss/common/db/sql/SqlTemplate.java index 013c752..b99dbe2 100644 --- a/src/main/java/com/gauss/common/db/sql/SqlTemplate.java +++ b/src/main/java/com/gauss/common/db/sql/SqlTemplate.java @@ -1,8 +1,10 @@ package com.gauss.common.db.sql; import com.gauss.common.db.meta.ColumnMeta; +import com.gauss.common.db.meta.Table; import com.gauss.common.model.DbType; import com.gauss.common.model.GaussContext; +import com.gauss.common.utils.GaussUtils; import java.sql.Types; import java.util.ArrayList; @@ -16,21 +18,22 @@ public class SqlTemplate { private String orinTableName; - private String CompareTableName; + private String compareTableName; private GaussContext context; public SqlTemplate(DbType dbType, GaussContext context) { this.context = context; this.dbType = dbType; - this.orinTableName = context.getTableMeta().getFullName(); - this.CompareTableName = orinTableName + "_dataChecker"; + Table tableMeta = context.getTableMeta(); + this.orinTableName = tableMeta.getFullName(); + this.compareTableName = tableMeta.getSchema() + ".A" + tableMeta.getName() + "_dataChecker"; } public String getOracleMd5() { StringBuilder sb = new StringBuilder(); sb.append("lower(utl_raw.cast_to_raw(dbms_obfuscation_toolkit.md5(input_string=>"); - List columns = context.getTableMeta().getColumns(); + List columns = context.getTableMeta().getColumns(); for (ColumnMeta meta : columns) { switch (meta.getType()) { case Types.FLOAT: @@ -43,20 +46,21 @@ public class SqlTemplate { sb.append(" || '_' || "); } int length = sb.length(); - sb.delete(length-11,length); + sb.delete(length - 11, length); sb.append("))) "); return sb.toString(); } public String getMysqlMd5() { StringBuilder sb = new StringBuilder(); - sb.append("md5(concat_ws('_',"); + sb.append("md5(concat_ws('',"); List columns = context.getTableMeta().getColumns(); for (ColumnMeta meta : columns) { switch (meta.getType()) { + case Types.REAL: case Types.FLOAT: case Types.DOUBLE: - sb.append("round(").append(meta.getName()).append(", 10)"); + sb.append("round(convert(").append(meta.getName()).append(",char), 10)"); break; case Types.BINARY: sb.append("lower(hex(").append(meta.getName()).append("))"); @@ -80,6 +84,8 @@ public class SqlTemplate { case Types.BOOLEAN: sb.append("cast(").append(meta.getName()).append(" as int"); break; + case Types.REAL: + //same as double case Types.FLOAT: //same as double case Types.DOUBLE: @@ -91,16 +97,16 @@ public class SqlTemplate { default: sb.append(meta.getName()); } - sb.append(" || '_' || "); + sb.append(" || "); } int length = sb.length(); - sb.delete(length-11,length); + sb.delete(length - 4, length); sb.append(")"); return sb.toString(); } public String getPrepareSql() { - String res = "insert into " + CompareTableName + "B select " + getOpgsMd5() + " from " + orinTableName + ";"; + String res = "insert into " + compareTableName + "B select " + getOpgsMd5() + " from " + orinTableName + ";"; return res; } @@ -119,7 +125,7 @@ public class SqlTemplate { } public String getCompareSql() { - return "select * from " + CompareTableName + "A t1 full join " + CompareTableName + return "select * from " + compareTableName + "A t1 full join " + compareTableName + "B t2 on t1.checksumA=t2.checksumB where t2.checksumB is null or t1.checksumA is null;"; } @@ -130,8 +136,10 @@ public class SqlTemplate { sb.append(getMysqlMd5()); } else if (dbType == DbType.ORACLE) { sb.append(getOracleMd5()); + } else if (dbType == DbType.OPGS){ + sb.append(getOpgsMd5()); } else { - //todo + // todo } sb.append(" in ("); for (String str : md5list) { diff --git a/src/main/java/com/gauss/common/utils/GaussUtils.java b/src/main/java/com/gauss/common/utils/GaussUtils.java index 67e8555..ced88a7 100644 --- a/src/main/java/com/gauss/common/utils/GaussUtils.java +++ b/src/main/java/com/gauss/common/utils/GaussUtils.java @@ -5,7 +5,9 @@ import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.SQLException; import java.sql.Types; +import java.text.SimpleDateFormat; import java.util.Collection; +import java.util.Date; import java.util.HashSet; import javax.sql.DataSource; diff --git a/src/main/java/com/gauss/comparer/GaussRecordComparer.java b/src/main/java/com/gauss/comparer/GaussRecordComparer.java index c43b3b6..88a9735 100644 --- a/src/main/java/com/gauss/comparer/GaussRecordComparer.java +++ b/src/main/java/com/gauss/comparer/GaussRecordComparer.java @@ -3,6 +3,7 @@ package com.gauss.comparer; import com.gauss.common.audit.RecordDiffer; import com.gauss.common.db.meta.ColumnMeta; import com.gauss.common.db.meta.ColumnValue; +import com.gauss.common.db.meta.Table; import com.gauss.common.db.sql.SqlTemplate; import com.gauss.common.model.DbType; import com.gauss.common.model.GaussContext; @@ -34,8 +35,9 @@ public class GaussRecordComparer extends AbstractRecordComparer { this.query_dop = query_dop; this.dbType = dbType; this.context = context; - this.orinTableName = context.getTableMeta().getFullName(); - this.compareTableName = orinTableName + "_dataChecker"; + Table tableMeta = context.getTableMeta(); + this.orinTableName = tableMeta.getFullName(); + this.compareTableName = tableMeta.getSchema() + ".A" + tableMeta.getName() + "_dataChecker"; } @Override @@ -70,7 +72,7 @@ public class GaussRecordComparer extends AbstractRecordComparer { if (!diffTarget.isEmpty()) { GaussUtils.outputUnnormal("Target table : " + orinTableName); - searchFromDb(sqlTemplate.getSearchSql(diffTarget),dbType); + searchFromDb(new SqlTemplate(DbType.OPGS, context).getSearchSql(diffTarget),DbType.OPGS); } jdbcTemplate.execute("drop table " + compareTableName + "A;"); diff --git a/src/main/java/com/gauss/controller/GaussInstance.java b/src/main/java/com/gauss/controller/GaussInstance.java index 965a2a4..1695588 100644 --- a/src/main/java/com/gauss/controller/GaussInstance.java +++ b/src/main/java/com/gauss/controller/GaussInstance.java @@ -105,6 +105,9 @@ public class GaussInstance extends AbstractGaussLifeCycle { if (!extractor.isStart()) { extractor.start(); } + if (!applier.isStart()) { + applier.start(); + } worker = new Thread(new Runnable() { diff --git a/src/main/java/com/gauss/preparer/GaussRecordPreparer.java b/src/main/java/com/gauss/preparer/GaussRecordPreparer.java index 5aec58c..b5889f7 100644 --- a/src/main/java/com/gauss/preparer/GaussRecordPreparer.java +++ b/src/main/java/com/gauss/preparer/GaussRecordPreparer.java @@ -1,5 +1,6 @@ package com.gauss.preparer; +import com.gauss.common.db.meta.Table; import com.gauss.common.db.sql.SqlTemplate; import com.gauss.common.model.DbType; import com.gauss.common.model.GaussContext; @@ -24,16 +25,15 @@ public class GaussRecordPreparer extends AbstractRecordPreparer { private Thread prepareThread = null; - String orinTableName; - String compareTableName; public GaussRecordPreparer(GaussContext context, int query_dop) { this.context = context; this.query_dop = query_dop; - orinTableName = context.getTableMeta().getFullName(); - compareTableName = orinTableName + "_dataChecker"; + Table tableMeta = context.getTableMeta(); + compareTableName = tableMeta.getSchema() + ".A" + tableMeta.getName() + "_dataChecker"; } + @Override public void start() { super.start(); @@ -53,6 +53,7 @@ public class GaussRecordPreparer extends AbstractRecordPreparer { new Preparer(context)); prepareThread.start(); } + @Override public void stop() { super.stop(); -- Gitee