diff --git a/src/main/java/com/gauss/GaussLauncher.java b/src/main/java/com/gauss/GaussLauncher.java index b409ced70b2c093419741c849a32ba6d57b2847d..93bbc761fda6191ef7ca35c71a5167327f4d69d0 100644 --- a/src/main/java/com/gauss/GaussLauncher.java +++ b/src/main/java/com/gauss/GaussLauncher.java @@ -1,6 +1,8 @@ package com.gauss; import java.io.FileInputStream; +import java.text.SimpleDateFormat; +import java.util.Date; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.lang.StringUtils; @@ -8,12 +10,14 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.gauss.common.utils.GaussUtils; import com.gauss.controller.GaussController; public class GaussLauncher { private static final String CLASSPATH_URL_PREFIX = "classpath:"; private static final Logger logger = LoggerFactory.getLogger(GaussLauncher.class); + private static final Logger summaryLogger = LoggerFactory.getLogger("summary"); public static void main(String[] args) throws Throwable { try { @@ -27,6 +31,7 @@ public class GaussLauncher { } logger.info("## start the DataChecker."); + summaryLogger.info("Start time : " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n"); final GaussController controller = new GaussController(config); controller.start(); logger.info("## the DataChecker is running now ......"); @@ -41,6 +46,7 @@ public class GaussLauncher { logger.warn("## something goes wrong when stopping DataChecker:\n{}", ExceptionUtils.getFullStackTrace(e)); } finally { + summaryLogger.info("End time : " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n"); logger.info("## DataChecker is down."); } } @@ -54,10 +60,12 @@ public class GaussLauncher { if (controller.isStart()) { controller.stop(); } + summaryLogger.info("End time : " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n"); logger.info("## DataChecker is down."); } catch (Throwable e) { logger.error("## Something goes wrong when starting up the DataChecker:\n{}", ExceptionUtils.getFullStackTrace(e)); + summaryLogger.info("End time : " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n"); System.exit(0); } } diff --git a/src/main/java/com/gauss/applier/CheckRecordApplier.java b/src/main/java/com/gauss/applier/CheckRecordApplier.java index eaedf1a5251b796cbfa740d205e09471548f71a8..71d6197c3db7be6eba43dc7eb7117de351d80d68 100644 --- a/src/main/java/com/gauss/applier/CheckRecordApplier.java +++ b/src/main/java/com/gauss/applier/CheckRecordApplier.java @@ -15,6 +15,7 @@ import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.ConnectionCallback; import org.springframework.jdbc.core.JdbcTemplate; +import com.gauss.common.db.meta.Table; import com.gauss.common.lifecycle.AbstractGaussLifeCycle; import com.gauss.common.model.GaussContext; import com.gauss.common.utils.GaussUtils; @@ -34,6 +35,12 @@ public class CheckRecordApplier extends AbstractGaussLifeCycle implements Record public CheckRecordApplier(GaussContext context, int query_dop) { this.context = context; + this.query_dop = query_dop; + } + + @Override + public void start() { + super.start(); } public void apply(List records) throws GaussException { @@ -58,7 +65,9 @@ public class CheckRecordApplier extends AbstractGaussLifeCycle implements Record StringBuilder buffer = new StringBuilder(); records.stream().forEach((String record) -> {buffer.append(record).append(SEPARATOR);}); - String sql = "copy " + context.getTableMeta().getFullName() + "_dataCheckerA " + "from stdin"; + Table tableMeta = context.getTableMeta(); + String compareTableName = tableMeta.getSchema() + ".A" + tableMeta.getName(); + String sql = "copy " + compareTableName + "_dataCheckerA " + "from stdin"; BaseConnection baseConn = (BaseConnection) (connection.getMetaData().getConnection()); CopyManager cp = new CopyManager(baseConn); StringReader reader = new StringReader(buffer.toString()); diff --git a/src/main/java/com/gauss/applier/MultiThreadCheckRecordApplier.java b/src/main/java/com/gauss/applier/MultiThreadCheckRecordApplier.java index 1dd1ecf716ee192aec4c13604c55e3dec309a936..5991343a3fe64072b71d6b91e8234226a2868e45 100644 --- a/src/main/java/com/gauss/applier/MultiThreadCheckRecordApplier.java +++ b/src/main/java/com/gauss/applier/MultiThreadCheckRecordApplier.java @@ -33,6 +33,7 @@ public class MultiThreadCheckRecordApplier extends CheckRecordApplier { this.executor = executor; } + @Override public void start() { super.start(); diff --git a/src/main/java/com/gauss/common/db/sql/SqlTemplate.java b/src/main/java/com/gauss/common/db/sql/SqlTemplate.java index 013c752bc1598c58cdcf3a1af5aa50f39873df7b..b99dbe2d2355e963a67448d3e8fefd6fb16fb0eb 100644 --- a/src/main/java/com/gauss/common/db/sql/SqlTemplate.java +++ b/src/main/java/com/gauss/common/db/sql/SqlTemplate.java @@ -1,8 +1,10 @@ package com.gauss.common.db.sql; import com.gauss.common.db.meta.ColumnMeta; +import com.gauss.common.db.meta.Table; import com.gauss.common.model.DbType; import com.gauss.common.model.GaussContext; +import com.gauss.common.utils.GaussUtils; import java.sql.Types; import java.util.ArrayList; @@ -16,21 +18,22 @@ public class SqlTemplate { private String orinTableName; - private String CompareTableName; + private String compareTableName; private GaussContext context; public SqlTemplate(DbType dbType, GaussContext context) { this.context = context; this.dbType = dbType; - this.orinTableName = context.getTableMeta().getFullName(); - this.CompareTableName = orinTableName + "_dataChecker"; + Table tableMeta = context.getTableMeta(); + this.orinTableName = tableMeta.getFullName(); + this.compareTableName = tableMeta.getSchema() + ".A" + tableMeta.getName() + "_dataChecker"; } public String getOracleMd5() { StringBuilder sb = new StringBuilder(); sb.append("lower(utl_raw.cast_to_raw(dbms_obfuscation_toolkit.md5(input_string=>"); - List columns = context.getTableMeta().getColumns(); + List columns = context.getTableMeta().getColumns(); for (ColumnMeta meta : columns) { switch (meta.getType()) { case Types.FLOAT: @@ -43,20 +46,21 @@ public class SqlTemplate { sb.append(" || '_' || "); } int length = sb.length(); - sb.delete(length-11,length); + sb.delete(length - 11, length); sb.append("))) "); return sb.toString(); } public String getMysqlMd5() { StringBuilder sb = new StringBuilder(); - sb.append("md5(concat_ws('_',"); + sb.append("md5(concat_ws('',"); List columns = context.getTableMeta().getColumns(); for (ColumnMeta meta : columns) { switch (meta.getType()) { + case Types.REAL: case Types.FLOAT: case Types.DOUBLE: - sb.append("round(").append(meta.getName()).append(", 10)"); + sb.append("round(convert(").append(meta.getName()).append(",char), 10)"); break; case Types.BINARY: sb.append("lower(hex(").append(meta.getName()).append("))"); @@ -80,6 +84,8 @@ public class SqlTemplate { case Types.BOOLEAN: sb.append("cast(").append(meta.getName()).append(" as int"); break; + case Types.REAL: + //same as double case Types.FLOAT: //same as double case Types.DOUBLE: @@ -91,16 +97,16 @@ public class SqlTemplate { default: sb.append(meta.getName()); } - sb.append(" || '_' || "); + sb.append(" || "); } int length = sb.length(); - sb.delete(length-11,length); + sb.delete(length - 4, length); sb.append(")"); return sb.toString(); } public String getPrepareSql() { - String res = "insert into " + CompareTableName + "B select " + getOpgsMd5() + " from " + orinTableName + ";"; + String res = "insert into " + compareTableName + "B select " + getOpgsMd5() + " from " + orinTableName + ";"; return res; } @@ -119,7 +125,7 @@ public class SqlTemplate { } public String getCompareSql() { - return "select * from " + CompareTableName + "A t1 full join " + CompareTableName + return "select * from " + compareTableName + "A t1 full join " + compareTableName + "B t2 on t1.checksumA=t2.checksumB where t2.checksumB is null or t1.checksumA is null;"; } @@ -130,8 +136,10 @@ public class SqlTemplate { sb.append(getMysqlMd5()); } else if (dbType == DbType.ORACLE) { sb.append(getOracleMd5()); + } else if (dbType == DbType.OPGS){ + sb.append(getOpgsMd5()); } else { - //todo + // todo } sb.append(" in ("); for (String str : md5list) { diff --git a/src/main/java/com/gauss/common/utils/GaussUtils.java b/src/main/java/com/gauss/common/utils/GaussUtils.java index 67e8555dd35b1caaefcbc55f1478084fc19f1c6b..ced88a7f060c0acea82e2c8c33b955c696f793ed 100644 --- a/src/main/java/com/gauss/common/utils/GaussUtils.java +++ b/src/main/java/com/gauss/common/utils/GaussUtils.java @@ -5,7 +5,9 @@ import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.SQLException; import java.sql.Types; +import java.text.SimpleDateFormat; import java.util.Collection; +import java.util.Date; import java.util.HashSet; import javax.sql.DataSource; diff --git a/src/main/java/com/gauss/comparer/GaussRecordComparer.java b/src/main/java/com/gauss/comparer/GaussRecordComparer.java index c43b3b6de398aa5e031f84be8acdec47effaa141..88a973532ec1bc0a454c1ecd7ad7fb4594e93750 100644 --- a/src/main/java/com/gauss/comparer/GaussRecordComparer.java +++ b/src/main/java/com/gauss/comparer/GaussRecordComparer.java @@ -3,6 +3,7 @@ package com.gauss.comparer; import com.gauss.common.audit.RecordDiffer; import com.gauss.common.db.meta.ColumnMeta; import com.gauss.common.db.meta.ColumnValue; +import com.gauss.common.db.meta.Table; import com.gauss.common.db.sql.SqlTemplate; import com.gauss.common.model.DbType; import com.gauss.common.model.GaussContext; @@ -34,8 +35,9 @@ public class GaussRecordComparer extends AbstractRecordComparer { this.query_dop = query_dop; this.dbType = dbType; this.context = context; - this.orinTableName = context.getTableMeta().getFullName(); - this.compareTableName = orinTableName + "_dataChecker"; + Table tableMeta = context.getTableMeta(); + this.orinTableName = tableMeta.getFullName(); + this.compareTableName = tableMeta.getSchema() + ".A" + tableMeta.getName() + "_dataChecker"; } @Override @@ -70,7 +72,7 @@ public class GaussRecordComparer extends AbstractRecordComparer { if (!diffTarget.isEmpty()) { GaussUtils.outputUnnormal("Target table : " + orinTableName); - searchFromDb(sqlTemplate.getSearchSql(diffTarget),dbType); + searchFromDb(new SqlTemplate(DbType.OPGS, context).getSearchSql(diffTarget),DbType.OPGS); } jdbcTemplate.execute("drop table " + compareTableName + "A;"); diff --git a/src/main/java/com/gauss/controller/GaussInstance.java b/src/main/java/com/gauss/controller/GaussInstance.java index 965a2a406a068b5164192c89d25a5d0c74b3b089..1695588516643d094d94b3d3426f952a3b8ea76a 100644 --- a/src/main/java/com/gauss/controller/GaussInstance.java +++ b/src/main/java/com/gauss/controller/GaussInstance.java @@ -105,6 +105,9 @@ public class GaussInstance extends AbstractGaussLifeCycle { if (!extractor.isStart()) { extractor.start(); } + if (!applier.isStart()) { + applier.start(); + } worker = new Thread(new Runnable() { diff --git a/src/main/java/com/gauss/preparer/GaussRecordPreparer.java b/src/main/java/com/gauss/preparer/GaussRecordPreparer.java index 5aec58c5f71ea9a974a4d03f135fca8ed6872978..b5889f70ec0d4f55378a2141e91729062b4a41b5 100644 --- a/src/main/java/com/gauss/preparer/GaussRecordPreparer.java +++ b/src/main/java/com/gauss/preparer/GaussRecordPreparer.java @@ -1,5 +1,6 @@ package com.gauss.preparer; +import com.gauss.common.db.meta.Table; import com.gauss.common.db.sql.SqlTemplate; import com.gauss.common.model.DbType; import com.gauss.common.model.GaussContext; @@ -24,16 +25,15 @@ public class GaussRecordPreparer extends AbstractRecordPreparer { private Thread prepareThread = null; - String orinTableName; - String compareTableName; public GaussRecordPreparer(GaussContext context, int query_dop) { this.context = context; this.query_dop = query_dop; - orinTableName = context.getTableMeta().getFullName(); - compareTableName = orinTableName + "_dataChecker"; + Table tableMeta = context.getTableMeta(); + compareTableName = tableMeta.getSchema() + ".A" + tableMeta.getName() + "_dataChecker"; } + @Override public void start() { super.start(); @@ -53,6 +53,7 @@ public class GaussRecordPreparer extends AbstractRecordPreparer { new Preparer(context)); prepareThread.start(); } + @Override public void stop() { super.stop();