From fe21bc9d9d5c75f2b8773a559a3e2c23d42ba2a2 Mon Sep 17 00:00:00 2001 From: zephyr Date: Thu, 7 Sep 2023 14:37:11 +0800 Subject: [PATCH 01/15] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96taskGroupCommu?= =?UTF-8?q?nicationMap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/alibaba/datax/core/taskgroup/TaskGroupContainer.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java index d163e99..849eba0 100644 --- a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java +++ b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java @@ -83,7 +83,10 @@ public class TaskGroupContainer extends AbstractContainer { private void initCommunicator(Configuration configuration) { // todo - taskGroupCommunicationMap.put(0, new Communication()); + if (taskGroupCommunicationMap.isEmpty()) { + int taskGroupId = configuration.getInt(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID); + taskGroupCommunicationMap.put(taskGroupId, new Communication()); + } super.setContainerCommunicator(new StandaloneTGContainerCommunicator(configuration, taskGroupCommunicationMap)); } -- Gitee From edd5516fd0c30df8b110a7bd95aeefc6971bd77a Mon Sep 17 00:00:00 2001 From: zephyr Date: Thu, 7 Sep 2023 18:09:28 +0800 Subject: [PATCH 02/15] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E8=B0=83=E6=95=B4?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datax/common/util/ConfigurationUtil.java | 1 - .../container/collector/AbstractCollector.java | 17 +++++++++-------- .../collector/ProcessInnerCollector.java | 6 ++---- .../AbstractContainerCommunicator.java | 13 +------------ .../job/StandAloneJobContainerCommunicator.java | 10 +++++++--- .../AbstractTGContainerCommunicator.java | 6 ++---- .../StandaloneTGContainerCommunicator.java | 12 +++++++++--- .../container/report/AbstractReporter.java | 7 ------- .../container/report/ProcessInnerReporter.java | 14 ++++++++++---- .../core/taskgroup/TaskGroupContainer.java | 2 +- .../com/gbase8c/dmt/service/TaskService.java | 1 + dmt/src/test/java/com/gbase8c/dmt/Test.java | 6 ++++-- 12 files changed, 46 insertions(+), 49 deletions(-) diff --git a/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java b/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java index 9e4a43d..9f93704 100644 --- a/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java +++ b/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java @@ -12,7 +12,6 @@ public class ConfigurationUtil { public static void persistence(Configuration configuration) throws IOException { String resume = System.getProperty("datax.persistence"); if ("true".equalsIgnoreCase(resume)) { - configuration.set("core.container.model", "taskGroup"); String json = configuration.toJSON(); FileUtils.writeStringToFile(file(configuration), json, StandardCharsets.UTF_8, false); } diff --git a/core/src/main/java/com/alibaba/datax/core/statistics/container/collector/AbstractCollector.java b/core/src/main/java/com/alibaba/datax/core/statistics/container/collector/AbstractCollector.java index fc84a3b..c39be47 100644 --- a/core/src/main/java/com/alibaba/datax/core/statistics/container/collector/AbstractCollector.java +++ b/core/src/main/java/com/alibaba/datax/core/statistics/container/collector/AbstractCollector.java @@ -2,7 +2,6 @@ package com.alibaba.datax.core.statistics.container.collector; import com.alibaba.datax.common.util.Configuration; import com.alibaba.datax.core.statistics.communication.Communication; -import com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator; import com.alibaba.datax.core.util.container.CoreConstant; import com.alibaba.datax.dataxservice.face.domain.enums.State; import org.apache.commons.lang3.Validate; @@ -13,18 +12,23 @@ import java.util.concurrent.ConcurrentHashMap; public abstract class AbstractCollector { - private Map taskCommunicationMap = new ConcurrentHashMap(); + protected Map taskCommunicationMap = new ConcurrentHashMap<>(); + + protected Map taskGroupCommunicationMap = new ConcurrentHashMap<>(); private Long jobId; - protected AbstractContainerCommunicator containerCommunicator; - public AbstractCollector(AbstractContainerCommunicator containerCommunicator) { - this.containerCommunicator = containerCommunicator; + public AbstractCollector(Map taskGroupCommunicationMap) { + this.taskGroupCommunicationMap = taskGroupCommunicationMap; } public Map getTaskCommunicationMap() { return taskCommunicationMap; } + public Map getTaskGroupCommunicationMap() { + return taskGroupCommunicationMap; + } + public Long getJobId() { return jobId; } @@ -34,7 +38,6 @@ public abstract class AbstractCollector { } public void registerTGCommunication(List taskGroupConfigurationList) { - Map taskGroupCommunicationMap = containerCommunicator.getTaskGroupCommunicationMap(); for (Configuration config : taskGroupConfigurationList) { int taskGroupId = config.getInt( CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID); @@ -69,13 +72,11 @@ public abstract class AbstractCollector { public abstract Communication collectFromTaskGroup(); public Map getTGCommunicationMap() { - Map taskGroupCommunicationMap = containerCommunicator.getTaskGroupCommunicationMap(); return taskGroupCommunicationMap; } public Communication getTGCommunication(Integer taskGroupId) { Validate.isTrue(taskGroupId >= 0, "taskGroupId不能小于0"); - Map taskGroupCommunicationMap = containerCommunicator.getTaskGroupCommunicationMap(); return taskGroupCommunicationMap.get(taskGroupId); } diff --git a/core/src/main/java/com/alibaba/datax/core/statistics/container/collector/ProcessInnerCollector.java b/core/src/main/java/com/alibaba/datax/core/statistics/container/collector/ProcessInnerCollector.java index 9655202..1ca9c4f 100644 --- a/core/src/main/java/com/alibaba/datax/core/statistics/container/collector/ProcessInnerCollector.java +++ b/core/src/main/java/com/alibaba/datax/core/statistics/container/collector/ProcessInnerCollector.java @@ -1,20 +1,18 @@ package com.alibaba.datax.core.statistics.container.collector; import com.alibaba.datax.core.statistics.communication.Communication; -import com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator; import com.alibaba.datax.dataxservice.face.domain.enums.State; import java.util.Map; public class ProcessInnerCollector extends AbstractCollector { - public ProcessInnerCollector(AbstractContainerCommunicator containerCommunicator) { - super(containerCommunicator); + public ProcessInnerCollector(Map taskGroupCommunicationMap) { + super(taskGroupCommunicationMap); } @Override public Communication collectFromTaskGroup() { - Map taskGroupCommunicationMap = containerCommunicator.getTaskGroupCommunicationMap(); Communication communication = new Communication(); communication.setState(State.SUCCEEDED); diff --git a/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/AbstractContainerCommunicator.java b/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/AbstractContainerCommunicator.java index 4c569e4..07b0400 100644 --- a/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/AbstractContainerCommunicator.java +++ b/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/AbstractContainerCommunicator.java @@ -19,16 +19,13 @@ public abstract class AbstractContainerCommunicator { protected AbstractReporter reporter; protected Long jobId; - protected Map taskGroupCommunicationMap; private VMInfo vmInfo = VMInfo.getVmInfo(); private long lastReportTime = System.currentTimeMillis(); - public AbstractContainerCommunicator(Configuration configuration, - Map taskGroupCommunicationMap) { + public AbstractContainerCommunicator(Configuration configuration) { this.configuration = configuration; this.jobId = configuration.getLong(CoreConstant.DATAX_CORE_CONTAINER_JOB_ID); - this.taskGroupCommunicationMap = taskGroupCommunicationMap; } public Configuration getConfiguration() { @@ -55,14 +52,6 @@ public abstract class AbstractContainerCommunicator { return jobId; } - public Map getTaskGroupCommunicationMap() { - return taskGroupCommunicationMap; - } - - public void setTaskGroupCommunicationMap(Map taskGroupCommunicationMap) { - this.taskGroupCommunicationMap = taskGroupCommunicationMap; - } - public abstract void registerCommunication(List configurationList); public abstract Communication collect(); diff --git a/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/job/StandAloneJobContainerCommunicator.java b/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/job/StandAloneJobContainerCommunicator.java index 0cfe1f2..59e691f 100644 --- a/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/job/StandAloneJobContainerCommunicator.java +++ b/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/job/StandAloneJobContainerCommunicator.java @@ -3,8 +3,10 @@ package com.alibaba.datax.core.statistics.container.communicator.job; import com.alibaba.datax.common.util.Configuration; import com.alibaba.datax.core.statistics.communication.Communication; import com.alibaba.datax.core.statistics.communication.CommunicationTool; +import com.alibaba.datax.core.statistics.container.collector.AbstractCollector; import com.alibaba.datax.core.statistics.container.collector.ProcessInnerCollector; import com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator; +import com.alibaba.datax.core.statistics.container.report.AbstractReporter; import com.alibaba.datax.core.statistics.container.report.ProcessInnerReporter; import com.alibaba.datax.dataxservice.face.domain.enums.State; import org.slf4j.Logger; @@ -20,9 +22,11 @@ public class StandAloneJobContainerCommunicator extends AbstractContainerCommuni public StandAloneJobContainerCommunicator(Configuration configuration, Map taskGroupCommunicationMap) { - super(configuration, taskGroupCommunicationMap); - super.setCollector(new ProcessInnerCollector(this)); - super.setReporter(new ProcessInnerReporter(this)); + super(configuration); + AbstractCollector collector = new ProcessInnerCollector(taskGroupCommunicationMap); + setCollector(collector); + AbstractReporter reporter = new ProcessInnerReporter(collector.getTaskCommunicationMap(), taskGroupCommunicationMap); + setReporter(reporter); } @Override diff --git a/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/AbstractTGContainerCommunicator.java b/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/AbstractTGContainerCommunicator.java index e42d6dc..41cddab 100644 --- a/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/AbstractTGContainerCommunicator.java +++ b/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/AbstractTGContainerCommunicator.java @@ -27,10 +27,8 @@ public abstract class AbstractTGContainerCommunicator extends AbstractContainerC */ protected int taskGroupId; - public AbstractTGContainerCommunicator(Configuration configuration, - Map taskGroupCommunicationMap) { - super(configuration, taskGroupCommunicationMap); - super.setCollector(new ProcessInnerCollector(this)); + public AbstractTGContainerCommunicator(Configuration configuration) { + super(configuration); this.taskGroupId = configuration.getInt(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID); } diff --git a/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/StandaloneTGContainerCommunicator.java b/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/StandaloneTGContainerCommunicator.java index 769780a..4f06fea 100644 --- a/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/StandaloneTGContainerCommunicator.java +++ b/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/StandaloneTGContainerCommunicator.java @@ -2,7 +2,9 @@ package com.alibaba.datax.core.statistics.container.communicator.taskgroup; import com.alibaba.datax.common.util.Configuration; import com.alibaba.datax.core.statistics.communication.CommunicationTool; -import com.alibaba.datax.core.statistics.container.communicator.job.StandAloneJobContainerCommunicator; +import com.alibaba.datax.core.statistics.container.collector.AbstractCollector; +import com.alibaba.datax.core.statistics.container.collector.ProcessInnerCollector; +import com.alibaba.datax.core.statistics.container.report.AbstractReporter; import com.alibaba.datax.core.statistics.container.report.ProcessInnerReporter; import com.alibaba.datax.core.statistics.communication.Communication; import com.alibaba.datax.core.util.container.CoreConstant; @@ -19,8 +21,12 @@ public class StandaloneTGContainerCommunicator extends AbstractTGContainerCommun public StandaloneTGContainerCommunicator(Configuration configuration, Map taskGroupCommunicationMap) { - super(configuration, taskGroupCommunicationMap); - super.setReporter(new ProcessInnerReporter(this)); + super(configuration); + AbstractCollector collector = new ProcessInnerCollector(taskGroupCommunicationMap); + setCollector(collector); + AbstractReporter reporter = new ProcessInnerReporter(collector.getTaskCommunicationMap(), taskGroupCommunicationMap); + setReporter(reporter); + String jobName = configuration.getString("job.name"); if (null != jobName) { LOG = LoggerFactory.getLogger(jobName); diff --git a/core/src/main/java/com/alibaba/datax/core/statistics/container/report/AbstractReporter.java b/core/src/main/java/com/alibaba/datax/core/statistics/container/report/AbstractReporter.java index 99c82f0..57f9858 100644 --- a/core/src/main/java/com/alibaba/datax/core/statistics/container/report/AbstractReporter.java +++ b/core/src/main/java/com/alibaba/datax/core/statistics/container/report/AbstractReporter.java @@ -1,16 +1,9 @@ package com.alibaba.datax.core.statistics.container.report; import com.alibaba.datax.core.statistics.communication.Communication; -import com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator; public abstract class AbstractReporter { - protected AbstractContainerCommunicator containerCommunicator; - - public AbstractReporter(AbstractContainerCommunicator containerCommunicator) { - this.containerCommunicator = containerCommunicator; - } - public abstract void reportJobCommunication(Long jobId, Communication communication); public abstract void reportTGCommunication(Integer taskGroupId, Communication communication); diff --git a/core/src/main/java/com/alibaba/datax/core/statistics/container/report/ProcessInnerReporter.java b/core/src/main/java/com/alibaba/datax/core/statistics/container/report/ProcessInnerReporter.java index 15df137..2962716 100644 --- a/core/src/main/java/com/alibaba/datax/core/statistics/container/report/ProcessInnerReporter.java +++ b/core/src/main/java/com/alibaba/datax/core/statistics/container/report/ProcessInnerReporter.java @@ -5,10 +5,18 @@ import com.alibaba.datax.core.statistics.container.communicator.AbstractContaine import org.apache.commons.lang3.Validate; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; public class ProcessInnerReporter extends AbstractReporter { - public ProcessInnerReporter(AbstractContainerCommunicator containerCommunicator) { - super(containerCommunicator); + + protected Map taskCommunicationMap; + + protected Map taskGroupCommunicationMap; + + public ProcessInnerReporter(Map taskCommunicationMap, + Map taskGroupCommunicationMap) { + this.taskCommunicationMap = taskCommunicationMap; + this.taskGroupCommunicationMap = taskGroupCommunicationMap; } @Override @@ -18,8 +26,6 @@ public class ProcessInnerReporter extends AbstractReporter { @Override public void reportTGCommunication(Integer taskGroupId, Communication communication) { - - Map taskGroupCommunicationMap = containerCommunicator.getTaskGroupCommunicationMap(); Validate.isTrue(taskGroupCommunicationMap.containsKey( taskGroupId), String.format("taskGroupCommunicationMap中没有注册taskGroupId[%d]的Communication，" + "无法更新该taskGroup的信息", taskGroupId)); diff --git a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java index 849eba0..c9bce21 100644 --- a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java +++ b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java @@ -516,7 +516,7 @@ public class TaskGroupContainer extends AbstractContainer { PluginType.READER); RecordSender recordSender; - if (transformerInfoExecs != null && transformerInfoExecs.size() > 0) { + if (transformerInfoExecs != null && !transformerInfoExecs.isEmpty()) { recordSender = new BufferedRecordTransformerExchanger(taskGroupId, this.taskId, this.channel,this.taskCommunication ,pluginCollector, transformerInfoExecs); } else { recordSender = new BufferedRecordExchanger(this.channel, pluginCollector); diff --git a/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java b/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java index d758a0d..7283cee 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java +++ b/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java @@ -112,6 +112,7 @@ public class TaskService { if (ts != null) { configurations = Arrays.stream(ts).map(this::file) .map(ConfigParser::parse) + .peek(conf -> conf.set("core.container.model", "taskGroup")) .collect(Collectors.toList()); } diff --git a/dmt/src/test/java/com/gbase8c/dmt/Test.java b/dmt/src/test/java/com/gbase8c/dmt/Test.java index 2e246d4..0e071f4 100644 --- a/dmt/src/test/java/com/gbase8c/dmt/Test.java +++ b/dmt/src/test/java/com/gbase8c/dmt/Test.java @@ -10,9 +10,11 @@ public class Test { System.setProperty("datax.home", "D:/ora-migration-tool/target/ora-migration-tool/ora-migration-tool"); System.setProperty("datax.persistence", "true"); -// Configuration configuration = ConfigParser.parse("D:/workdir/1/configuration/TEST.T_TABLE.json"); - Configuration configuration = ConfigParser.parse("D:/workdir/test.json"); +// Configuration configuration = ConfigParser.parse("D:/workdir/test.json"); // configuration.set("core.container.model", "taskGroup"); + + Configuration configuration = ConfigParser.parse("D:/workdir/ori.json"); + Engine engine = new Engine(); engine.start(configuration); } -- Gitee From 21e19e8f6f6fa1af1989589b1a12e56559628b80 Mon Sep 17 00:00:00 2001 From: zephyr Date: Fri, 8 Sep 2023 16:17:57 +0800 Subject: [PATCH 03/15] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=96=AD=E7=82=B9=E7=BB=AD=E4=BC=A0=E8=87=AA=E5=8A=A8=E6=B8=85?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../alibaba/datax/core/job/JobContainer.java | 35 ++-- .../StandAloneJobContainerCommunicator.java | 3 +- .../StandaloneTGContainerCommunicator.java | 15 +- .../core/taskgroup/TaskGroupContainer.java | 188 +++++++++--------- .../dmt/migration/MigrationObjectService.java | 16 +- .../com/gbase8c/dmt/service/TaskService.java | 42 ++++ dmt/src/test/java/com/gbase8c/dmt/Test.java | 7 +- 7 files changed, 166 insertions(+), 140 deletions(-) diff --git a/core/src/main/java/com/alibaba/datax/core/job/JobContainer.java b/core/src/main/java/com/alibaba/datax/core/job/JobContainer.java index ee5d887..e492b4e 100644 --- a/core/src/main/java/com/alibaba/datax/core/job/JobContainer.java +++ b/core/src/main/java/com/alibaba/datax/core/job/JobContainer.java @@ -46,14 +46,11 @@ import java.util.concurrent.ConcurrentHashMap; * 但它并不做实际的数据同步操作 */ public class JobContainer extends AbstractContainer { - private static final Logger LOG = LoggerFactory - .getLogger(JobContainer.class); + private static final Logger LOG = LoggerFactory.getLogger(JobContainer.class); - private static final SimpleDateFormat dateFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); + private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - private ClassLoaderSwapper classLoaderSwapper = ClassLoaderSwapper - .newCurrentThreadClassLoaderSwapper(); + private ClassLoaderSwapper classLoaderSwapper = ClassLoaderSwapper.newCurrentThreadClassLoaderSwapper(); private long jobId; private String jobName; @@ -85,8 +82,7 @@ public class JobContainer extends AbstractContainer { private ErrorRecordChecker errorLimit; - public JobContainer(Configuration configuration, - Map taskGroupCommunicationMap) { + public JobContainer(Configuration configuration, Map taskGroupCommunicationMap) { super(configuration); this.taskGroupCommunicationMap = taskGroupCommunicationMap; errorLimit = new ErrorRecordChecker(configuration); @@ -105,7 +101,7 @@ public class JobContainer extends AbstractContainer { try { this.startTimeStamp = System.currentTimeMillis(); isDryRun = configuration.getBool(CoreConstant.DATAX_JOB_SETTING_DRYRUN, false); - if(isDryRun) { + if (isDryRun) { LOG.info("jobContainer starts to do preCheck ..."); this.preCheck(); } else { @@ -118,7 +114,6 @@ public class JobContainer extends AbstractContainer { LOG.info("jobContainer starts to do prepare ..."); this.prepare(); LOG.info("jobContainer starts to do split ..."); - // todo 如果是断点续传，直接读取分块信息和分块数 this.totalStage = this.split(); LOG.info("jobContainer starts to do schedule ..."); this.schedule(); @@ -143,7 +138,6 @@ public class JobContainer extends AbstractContainer { if (super.getContainerCommunicator() == null) { // 由于 containerCollector 是在 scheduler() 中初始化的，所以当在 scheduler() 之前出现异常时，需要在此处对 containerCollector 进行初始化 - AbstractContainerCommunicator tempContainerCollector; // standalone tempContainerCollector = new StandAloneJobContainerCommunicator(configuration, new ConcurrentHashMap<>()); @@ -166,7 +160,7 @@ public class JobContainer extends AbstractContainer { throw DataXException.asDataXException( FrameworkErrorCode.RUNTIME_ERROR, e); } finally { - if(!isDryRun) { + if (!isDryRun) { this.destroy(); this.endTimeStamp = System.currentTimeMillis(); @@ -245,7 +239,6 @@ public class JobContainer extends AbstractContainer { return jobReader; } - private Writer.Job preCheckWriterInit(JobPluginCollector jobPluginCollector) { this.writerPluginName = this.configuration.getString( CoreConstant.DATAX_JOB_CONTENT_WRITER_NAME); @@ -326,7 +319,7 @@ public class JobContainer extends AbstractContainer { private void preHandle() { String handlerPluginTypeStr = this.configuration.getString( CoreConstant.DATAX_JOB_PREHANDLER_PLUGINTYPE); - if(!StringUtils.isNotEmpty(handlerPluginTypeStr)){ + if (!StringUtils.isNotEmpty(handlerPluginTypeStr)) { return; } PluginType handlerPluginType; @@ -362,7 +355,7 @@ public class JobContainer extends AbstractContainer { String handlerPluginTypeStr = this.configuration.getString( CoreConstant.DATAX_JOB_POSTHANDLER_PLUGINTYPE); - if(!StringUtils.isNotEmpty(handlerPluginTypeStr)){ + if (!StringUtils.isNotEmpty(handlerPluginTypeStr)) { return; } PluginType handlerPluginType; @@ -411,14 +404,14 @@ public class JobContainer extends AbstractContainer { List transformerList = this.configuration.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT_TRANSFORMER); - LOG.debug("transformer configuration: "+ JSON.toJSONString(transformerList)); + LOG.debug("transformer configuration: " + JSON.toJSONString(transformerList)); /** * 输入是reader和writer的parameter list，输出是content下面元素的list */ List contentConfig = mergeReaderAndWriterTaskConfigs( readerTaskConfigs, writerTaskConfigs, transformerList); - LOG.debug("contentConfig configuration: "+ JSON.toJSONString(contentConfig)); + LOG.debug("contentConfig configuration: " + JSON.toJSONString(contentConfig)); // todo 如果是断点续传，参考这句设置 this.configuration.set(CoreConstant.DATAX_JOB_CONTENT, contentConfig); @@ -472,8 +465,7 @@ public class JobContainer extends AbstractContainer { } // 取较小值 - this.needChannelNumber = needChannelNumberByByte < needChannelNumberByRecord ? - needChannelNumberByByte : needChannelNumberByRecord; + this.needChannelNumber = Math.min(needChannelNumberByByte, needChannelNumberByRecord); // 如果从byte或record上设置了needChannelNumber则退出 if (this.needChannelNumber < Integer.MAX_VALUE) { @@ -516,7 +508,6 @@ public class JobContainer extends AbstractContainer { /** * 通过获取配置信息得到每个taskGroup需要运行哪些tasks任务 */ - // todo 如果是断点续传, 需要从文件里读取并做记录 List taskGroupConfigs = JobAssignUtil.assignFairly(this.configuration, this.needChannelNumber, channelsPerTaskGroup); @@ -525,7 +516,7 @@ public class JobContainer extends AbstractContainer { ExecuteMode executeMode = null; AbstractScheduler scheduler; try { - executeMode = ExecuteMode.STANDALONE; + executeMode = ExecuteMode.STANDALONE; scheduler = initStandaloneScheduler(this.configuration); //设置 executeMode @@ -820,7 +811,7 @@ public class JobContainer extends AbstractContainer { taskConfig.set(CoreConstant.JOB_WRITER_PARAMETER, writerTasksConfigs.get(i)); - if(transformerConfigs!=null && transformerConfigs.size()>0){ + if (transformerConfigs != null && transformerConfigs.size() > 0) { taskConfig.set(CoreConstant.JOB_TRANSFORMER, transformerConfigs); } diff --git a/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/job/StandAloneJobContainerCommunicator.java b/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/job/StandAloneJobContainerCommunicator.java index 59e691f..71b724a 100644 --- a/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/job/StandAloneJobContainerCommunicator.java +++ b/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/job/StandAloneJobContainerCommunicator.java @@ -51,7 +51,8 @@ public class StandAloneJobContainerCommunicator extends AbstractContainerCommuni public void report(Communication communication) { super.getReporter().reportJobCommunication(super.getJobId(), communication); - LOG.info(CommunicationTool.Stringify.getSnapshot(communication)); + String jobName = configuration.getString("job.name"); + LOG.info(jobName + ": " + CommunicationTool.Stringify.getSnapshot(communication)); reportVmInfo(); } diff --git a/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/StandaloneTGContainerCommunicator.java b/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/StandaloneTGContainerCommunicator.java index 4f06fea..76144e1 100644 --- a/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/StandaloneTGContainerCommunicator.java +++ b/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/StandaloneTGContainerCommunicator.java @@ -15,9 +15,8 @@ import java.util.Map; public class StandaloneTGContainerCommunicator extends AbstractTGContainerCommunicator { -// private static final Logger LOG = LoggerFactory -// .getLogger(StandaloneTGContainerCommunicator.class); - private Logger LOG = null; + private static final Logger LOG = LoggerFactory + .getLogger(StandaloneTGContainerCommunicator.class); public StandaloneTGContainerCommunicator(Configuration configuration, Map taskGroupCommunicationMap) { @@ -26,13 +25,6 @@ public class StandaloneTGContainerCommunicator extends AbstractTGContainerCommun setCollector(collector); AbstractReporter reporter = new ProcessInnerReporter(collector.getTaskCommunicationMap(), taskGroupCommunicationMap); setReporter(reporter); - - String jobName = configuration.getString("job.name"); - if (null != jobName) { - LOG = LoggerFactory.getLogger(jobName); - } else { - LOG = LoggerFactory.getLogger(StandaloneTGContainerCommunicator.class); - } } @Override @@ -41,7 +33,8 @@ public class StandaloneTGContainerCommunicator extends AbstractTGContainerCommun // todo if ("taskGroup".equalsIgnoreCase(configuration .getString(CoreConstant.DATAX_CORE_CONTAINER_MODEL))) { - LOG.info(CommunicationTool.Stringify.getSnapshot(communication)); + String jobName = configuration.getString("job.name"); + LOG.info(jobName + ": " + CommunicationTool.Stringify.getSnapshot(communication)); reportVmInfo(); } } diff --git a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java index c9bce21..0bbb284 100644 --- a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java +++ b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java @@ -82,7 +82,7 @@ public class TaskGroupContainer extends AbstractContainer { } private void initCommunicator(Configuration configuration) { - // todo + // todo 如果taskGroupCommunicationMap为空, 尝试从configuration中读取相关属性进行初始化 if (taskGroupCommunicationMap.isEmpty()) { int taskGroupId = configuration.getInt(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID); taskGroupCommunicationMap.put(taskGroupId, new Communication()); @@ -134,11 +134,11 @@ public class TaskGroupContainer extends AbstractContainer { List taskConfigs = this.configuration .getListConfiguration(CoreConstant.DATAX_JOB_CONTENT); - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("taskGroup[{}]'s task configs[{}]", this.taskGroupId, JSON.toJSONString(taskConfigs)); } - + int taskCountInThisTaskGroup = taskConfigs.size(); LOG.info(String.format( "taskGroupId=[%d] start [%d] channels for [%d] tasks.", @@ -158,13 +158,13 @@ public class TaskGroupContainer extends AbstractContainer { Communication lastTaskGroupContainerCommunication = new Communication(); while (true) { - //1.判断task状态 - boolean failedOrKilled = false; - Map communicationMap = containerCommunicator.getCommunicationMap(); - for (Map.Entry entry : communicationMap.entrySet()){ - Integer taskId = entry.getKey(); - Communication taskCommunication = entry.getValue(); - if (!taskCommunication.isFinished()){ + //1.判断task状态 + boolean failedOrKilled = false; + Map communicationMap = containerCommunicator.getCommunicationMap(); + for (Map.Entry entry : communicationMap.entrySet()) { + Integer taskId = entry.getKey(); + Communication taskCommunication = entry.getValue(); + if (!taskCommunication.isFinished()) { continue; } @@ -174,25 +174,25 @@ public class TaskGroupContainer extends AbstractContainer { String path = "job.content[" + taskId + "].state"; // 失败，看task是否支持failover，重试次数未超过最大限制 - if (taskCommunication.getState() == State.FAILED){ + if (taskCommunication.getState() == State.FAILED) { // todo 记录状态为State.FAILED configuration.set(path, State.FAILED); ConfigurationUtil.persistence(configuration); taskFailedExecutorMap.put(taskId, taskExecutor); - if(taskExecutor.supportFailOver() && taskExecutor.getAttemptCount() < taskMaxRetryTimes){ + if (taskExecutor.supportFailOver() && taskExecutor.getAttemptCount() < taskMaxRetryTimes) { taskExecutor.shutdown(); //关闭老的executor containerCommunicator.resetCommunication(taskId); //将task的状态重置 - Configuration taskConfig = taskConfigMap.get(taskId); - taskQueue.add(taskConfig); //重新加入任务列表 - }else{ - failedOrKilled = true; - break; - } - } else if(taskCommunication.getState() == State.KILLED){ - failedOrKilled = true; - break; - } else if(taskCommunication.getState() == State.SUCCEEDED){ + Configuration taskConfig = taskConfigMap.get(taskId); + taskQueue.add(taskConfig); //重新加入任务列表 + } else { + failedOrKilled = true; + break; + } + } else if (taskCommunication.getState() == State.KILLED) { + failedOrKilled = true; + break; + } else if (taskCommunication.getState() == State.SUCCEEDED) { // todo 记录状态为State.SUCCEEDED Configuration taskConfig = taskConfigMap.get(taskId); if (taskConfig != null) { @@ -201,18 +201,17 @@ public class TaskGroupContainer extends AbstractContainer { } Long taskStartTime = taskStartTimeMap.get(taskId); - if(taskStartTime != null){ - Long usedTime = System.currentTimeMillis() - taskStartTime; - LOG.info("taskGroup[{}] taskId[{}] is successed, used[{}]ms", - this.taskGroupId, taskId, usedTime); + if (taskStartTime != null) { + long usedTime = System.currentTimeMillis() - taskStartTime; + LOG.info("taskGroup[{}] taskId[{}] is successed, used[{}]ms", this.taskGroupId, taskId, usedTime); //usedTime*1000*1000 转换成PerfRecord记录的ns，这里主要是简单登记，进行最长任务的打印。因此增加特定静态方法 - PerfRecord.addPerfRecord(taskGroupId, taskId, PerfRecord.PHASE.TASK_TOTAL,taskStartTime, usedTime * 1000L * 1000L); + PerfRecord.addPerfRecord(taskGroupId, taskId, PerfRecord.PHASE.TASK_TOTAL, taskStartTime, usedTime * 1000L * 1000L); taskStartTimeMap.remove(taskId); taskConfigMap.remove(taskId); } } - } - + } + // 2.发现该taskGroup下taskExecutor的总状态失败则汇报错误 if (failedOrKilled) { lastTaskGroupContainerCommunication = reportTaskGroupCommunication( @@ -221,7 +220,7 @@ public class TaskGroupContainer extends AbstractContainer { throw DataXException.asDataXException( FrameworkErrorCode.PLUGIN_RUNTIME_ERROR, lastTaskGroupContainerCommunication.getThrowable()); } - + //3.有任务未执行，且正在运行的任务数小于最大通道限制(任务并发数) Iterator iterator = taskQueue.iterator(); while (iterator.hasNext() && runTasks.size() < channelNumber) { @@ -229,7 +228,7 @@ public class TaskGroupContainer extends AbstractContainer { Integer taskId = taskConfig.getInt(CoreConstant.TASK_ID); int attemptCount = 1; TaskExecutor lastExecutor = taskFailedExecutorMap.get(taskId); - if (lastExecutor!=null) { + if (lastExecutor != null) { attemptCount = lastExecutor.getAttemptCount() + 1; long now = System.currentTimeMillis(); long failedTime = lastExecutor.getTimeStamp(); @@ -251,9 +250,9 @@ public class TaskGroupContainer extends AbstractContainer { } } Configuration taskConfigForRun = taskMaxRetryTimes > 1 ? taskConfig.clone() : taskConfig; - TaskExecutor taskExecutor = new TaskExecutor(taskConfigForRun, attemptCount); + TaskExecutor taskExecutor = new TaskExecutor(taskConfigForRun, attemptCount); taskStartTimeMap.put(taskId, System.currentTimeMillis()); - // todo 启动任务, 将分块状态置为State.RUNNING + // todo 启动任务, 将分块状态置为State.RUNNING String path = "job.content[" + taskId + "].state"; configuration.set(path, State.RUNNING); ConfigurationUtil.persistence(configuration); @@ -273,12 +272,12 @@ public class TaskGroupContainer extends AbstractContainer { //4.任务列表为空，executor已结束, 搜集状态为success--->成功 if (taskQueue.isEmpty() && isAllTaskDone(runTasks) && containerCommunicator.collectState() == State.SUCCEEDED) { - // 成功的情况下，也需要汇报一次。否则在任务结束非常快的情况下，采集的信息将会不准确 + // 成功的情况下，也需要汇报一次。否则在任务结束非常快的情况下，采集的信息将会不准确 lastTaskGroupContainerCommunication = reportTaskGroupCommunication( lastTaskGroupContainerCommunication, taskCountInThisTaskGroup); LOG.info("taskGroup[{}] completed it's tasks.", this.taskGroupId); - // todo remove + // todo 任务执行完成, remove配置文件 ConfigurationUtil.remove(configuration); break; } @@ -292,8 +291,8 @@ public class TaskGroupContainer extends AbstractContainer { lastReportTimeStamp = now; //taskMonitor对于正在运行的task，每reportIntervalInMillSec进行检查 - for(TaskExecutor taskExecutor:runTasks){ - taskMonitor.report(taskExecutor.getTaskId(),this.containerCommunicator.getCommunication(taskExecutor.getTaskId())); + for (TaskExecutor taskExecutor : runTasks) { + taskMonitor.report(taskExecutor.getTaskId(), this.containerCommunicator.getCommunication(taskExecutor.getTaskId())); } } @@ -317,8 +316,8 @@ public class TaskGroupContainer extends AbstractContainer { throw DataXException.asDataXException( FrameworkErrorCode.RUNTIME_ERROR, e); - }finally { - if(!PerfTrace.getInstance().isJob()){ + } finally { + if (!PerfTrace.getInstance().isJob()) { //最后打印cpu的平均消耗，GC的统计 VMInfo vmInfo = VMInfo.getVmInfo(); if (vmInfo != null) { @@ -330,49 +329,49 @@ public class TaskGroupContainer extends AbstractContainer { } } } - - private Map buildTaskConfigMap(List configurations){ - Map map = new HashMap(); - for(Configuration taskConfig : configurations){ - int taskId = taskConfig.getInt(CoreConstant.TASK_ID); - map.put(taskId, taskConfig); - } - return map; + + private Map buildTaskConfigMap(List configurations) { + Map map = new HashMap(); + for (Configuration taskConfig : configurations) { + int taskId = taskConfig.getInt(CoreConstant.TASK_ID); + map.put(taskId, taskConfig); + } + return map; } - private List buildRemainTasks(List configurations){ - List remainTasks = new LinkedList(); - for(Configuration taskConfig : configurations){ + private List buildRemainTasks(List configurations) { + List remainTasks = new LinkedList(); + for (Configuration taskConfig : configurations) { String state = taskConfig.getString("state"); if (!State.SUCCEEDED.name().equals(state)) { remainTasks.add(taskConfig); } - } - return remainTasks; + } + return remainTasks; } - - private TaskExecutor removeTask(List taskList, int taskId){ - Iterator iterator = taskList.iterator(); - while(iterator.hasNext()){ - TaskExecutor taskExecutor = iterator.next(); - if(taskExecutor.getTaskId() == taskId){ - iterator.remove(); - return taskExecutor; - } - } - return null; + + private TaskExecutor removeTask(List taskList, int taskId) { + Iterator iterator = taskList.iterator(); + while (iterator.hasNext()) { + TaskExecutor taskExecutor = iterator.next(); + if (taskExecutor.getTaskId() == taskId) { + iterator.remove(); + return taskExecutor; + } + } + return null; } - - private boolean isAllTaskDone(List taskList){ - for(TaskExecutor taskExecutor : taskList){ - if(!taskExecutor.isTaskFinished()){ - return false; - } - } - return true; + + private boolean isAllTaskDone(List taskList) { + for (TaskExecutor taskExecutor : taskList) { + if (!taskExecutor.isTaskFinished()) { + return false; + } + } + return true; } - private Communication reportTaskGroupCommunication(Communication lastTaskGroupContainerCommunication, int taskCount){ + private Communication reportTaskGroupCommunication(Communication lastTaskGroupContainerCommunication, int taskCount) { Communication nowTaskGroupContainerCommunication = this.containerCommunicator.collect(); nowTaskGroupContainerCommunication.setTimestamp(System.currentTimeMillis()); Communication reportCommunication = CommunicationTool.getReportCommunication(nowTaskGroupContainerCommunication, @@ -381,7 +380,7 @@ public class TaskGroupContainer extends AbstractContainer { return reportCommunication; } - private void markCommunicationFailed(Integer taskId){ + private void markCommunicationFailed(Integer taskId) { Communication communication = containerCommunicator.getCommunication(taskId); communication.setState(State.FAILED); } @@ -402,9 +401,9 @@ public class TaskGroupContainer extends AbstractContainer { private Thread readerThread; private Thread writerThread; - + private ReaderRunner readerRunner; - + private WriterRunner writerRunner; /** @@ -451,7 +450,7 @@ public class TaskGroupContainer extends AbstractContainer { writerRunner = (WriterRunner) generateRunner(PluginType.WRITER); this.writerThread = new Thread(writerRunner, String.format("%s-%d-%d-writer", - StringUtils.isNotBlank(jobName)? jobName : jobId, taskGroupId, this.taskId)); + StringUtils.isNotBlank(jobName) ? jobName : jobId, taskGroupId, this.taskId)); //通过设置thread的contextClassLoader，即可实现同步和主程序不通的加载器 this.writerThread.setContextClassLoader(LoadUtil.getJarLoader( PluginType.WRITER, this.taskConfig.getString( @@ -460,10 +459,10 @@ public class TaskGroupContainer extends AbstractContainer { /** * 生成readerThread */ - readerRunner = (ReaderRunner) generateRunner(PluginType.READER,transformerInfoExecs); + readerRunner = (ReaderRunner) generateRunner(PluginType.READER, transformerInfoExecs); this.readerThread = new Thread(readerRunner, String.format("%s-%d-%d-reader", - StringUtils.isNotBlank(jobName)? jobName : jobId, taskGroupId, this.taskId)); + StringUtils.isNotBlank(jobName) ? jobName : jobId, taskGroupId, this.taskId)); /** * 通过设置thread的contextClassLoader，即可实现同步和主程序不通的加载器 */ @@ -494,7 +493,6 @@ public class TaskGroupContainer extends AbstractContainer { } - private AbstractRunner generateRunner(PluginType pluginType) { return generateRunner(pluginType, null); } @@ -517,7 +515,7 @@ public class TaskGroupContainer extends AbstractContainer { RecordSender recordSender; if (transformerInfoExecs != null && !transformerInfoExecs.isEmpty()) { - recordSender = new BufferedRecordTransformerExchanger(taskGroupId, this.taskId, this.channel,this.taskCommunication ,pluginCollector, transformerInfoExecs); + recordSender = new BufferedRecordTransformerExchanger(taskGroupId, this.taskId, this.channel, this.taskCommunication, pluginCollector, transformerInfoExecs); } else { recordSender = new BufferedRecordExchanger(this.channel, pluginCollector); } @@ -564,41 +562,41 @@ public class TaskGroupContainer extends AbstractContainer { return false; } - if(taskCommunication==null || !taskCommunication.isFinished()){ - return false; - } + if (taskCommunication == null || !taskCommunication.isFinished()) { + return false; + } return true; } - - private int getTaskId(){ - return taskId; + + private int getTaskId() { + return taskId; } - private long getTimeStamp(){ + private long getTimeStamp() { return taskCommunication.getTimestamp(); } - private int getAttemptCount(){ + private int getAttemptCount() { return attemptCount; } - - private boolean supportFailOver(){ - return writerRunner.supportFailOver(); + + private boolean supportFailOver() { + return writerRunner.supportFailOver(); } - private void shutdown(){ + private void shutdown() { writerRunner.shutdown(); readerRunner.shutdown(); - if(writerThread.isAlive()){ + if (writerThread.isAlive()) { writerThread.interrupt(); } - if(readerThread.isAlive()){ + if (readerThread.isAlive()) { readerThread.interrupt(); } } - private boolean isShutdown(){ + private boolean isShutdown() { return !readerThread.isAlive() && !writerThread.isAlive(); } } diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java index cf09b92..bca253a 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java @@ -310,14 +310,14 @@ public class MigrationObjectService { } } -// // todo just for test - if (src.getDbType().equals(DbType.Oracle)) { - String partitionSql = "SELECT partition_name as partitionName FROM ALL_TAB_PARTITIONS WHERE table_owner = ? AND table_name = ? order by partition_position"; - List partitions = metadata.query(partitionSql, new ColumnListHandler(), tableDto.getSchema(), tableDto.getName()); - if (CollectionUtils.isNotEmpty(partitions)) { - paraMap.put("partition", partitions); - } - } +// // todo 支持从partition读, 暂时关闭, 原因是数据断点续传如果是从partition读的话, 无法程序生成删除目标端的sql +// if (src.getDbType().equals(DbType.Oracle)) { +// String partitionSql = "SELECT partition_name as partitionName FROM ALL_TAB_PARTITIONS WHERE table_owner = ? AND table_name = ? order by partition_position"; +// List partitions = metadata.query(partitionSql, new ColumnListHandler(), tableDto.getSchema(), tableDto.getName()); +// if (CollectionUtils.isNotEmpty(partitions)) { +// paraMap.put("partition", partitions); +// } +// } HashMap connMap = new HashMap<>(); connMap.put("jdbcUrl", Lists.newArrayList(src.getJdbcUrl())); diff --git a/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java b/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java index 7283cee..8490086 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java +++ b/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java @@ -3,6 +3,7 @@ package com.gbase8c.dmt.service; import com.alibaba.datax.common.util.Configuration; import com.alibaba.datax.core.Engine; import com.alibaba.datax.core.util.ConfigParser; +import com.alibaba.datax.core.util.container.CoreConstant; import com.gbase8c.dmt.config.DmtConfig; import com.gbase8c.dmt.dao.*; import com.gbase8c.dmt.dao.file.*; @@ -22,10 +23,13 @@ import io.reactivex.rxjava3.schedulers.Schedulers; import lombok.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.dbutils.QueryRunner; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import javax.sql.DataSource; import java.io.File; +import java.sql.SQLException; import java.util.*; import java.util.stream.Collectors; @@ -120,6 +124,7 @@ public class TaskService { .observeOn(Schedulers.io()) .parallel() .runOn(Schedulers.io()) + .doOnNext(this::clean) .doOnNext(this::start) .sequential() .collect(Collectors.toList()) @@ -134,6 +139,43 @@ public class TaskService { } + public void clean(Configuration configuration) throws SQLException { + List taskConfigs = configuration.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT); + Task task = taskDao.get(dmtConfig.getTaskId()); + DataSourceDto tarDataSourceDto = dataSourceDao.get(task.getTar()); + DataSource dataSource = DataSourceFactory.getDataSource(tarDataSourceDto); + for (Configuration taskConfig : taskConfigs) { + String state = taskConfig.getString("state"); + if ("FAILED".equalsIgnoreCase(state) || "RUNNING".equalsIgnoreCase(state)) { + String tarTable = taskConfig.getString("writer.parameter.table"); + String cleanSql; + String splitPk = taskConfig.getString("reader.parameter.splitPk"); + if (StringUtils.isNotBlank(splitPk)) { + String querySql = taskConfig.getString("reader.parameter.querySql"); + List columnList = taskConfig.getList("reader.parameter.columnList", String.class); + List tarColumnList = taskConfig.getList("writer.parameter.column", String.class); + String whereSql; + String whereKey = " where "; + int index = querySql.toLowerCase().indexOf(whereKey); + whereSql = querySql.substring(index); + + String tarColumn = tarColumnList.get(columnList.indexOf(splitPk)); + whereSql = whereSql.replaceAll(splitPk, tarColumn); + + cleanSql = "delete from " + tarTable + whereSql; + } else { + // truncate table xxx + cleanSql = "truncate table " + tarTable; + } + // 执行cleanSql + log.info("开始表 {} 分块清理, sql为: {}", tarTable, cleanSql); + QueryRunner queryRunner = new QueryRunner(dataSource); + queryRunner.execute(cleanSql); + log.info("完成表 {} 分块清理, sql为: {}", tarTable, cleanSql); + } + } + } + public void start(Configuration configuration) { Engine engine = new Engine(); engine.start(configuration); diff --git a/dmt/src/test/java/com/gbase8c/dmt/Test.java b/dmt/src/test/java/com/gbase8c/dmt/Test.java index 0e071f4..1be4974 100644 --- a/dmt/src/test/java/com/gbase8c/dmt/Test.java +++ b/dmt/src/test/java/com/gbase8c/dmt/Test.java @@ -10,13 +10,14 @@ public class Test { System.setProperty("datax.home", "D:/ora-migration-tool/target/ora-migration-tool/ora-migration-tool"); System.setProperty("datax.persistence", "true"); -// Configuration configuration = ConfigParser.parse("D:/workdir/test.json"); -// configuration.set("core.container.model", "taskGroup"); + Configuration configuration = ConfigParser.parse("D:/workdir/test.json"); + configuration.set("core.container.model", "taskGroup"); - Configuration configuration = ConfigParser.parse("D:/workdir/ori.json"); +// Configuration configuration = ConfigParser.parse("D:/workdir/ori.json"); Engine engine = new Engine(); engine.start(configuration); + } } -- Gitee From 5a13fc929cbad139e2c548e21d79760846fa8489 Mon Sep 17 00:00:00 2001 From: zephyr Date: Thu, 14 Sep 2023 13:32:02 +0800 Subject: [PATCH 04/15] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E5=AF=B9?= =?UTF-8?q?=E8=B1=A1=E8=BF=81=E7=A7=BB=E7=8A=B6=E6=80=81=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datax/common/util/ConfigurationUtil.java | 84 +++++++++++++++++++ .../core/taskgroup/TaskGroupContainer.java | 17 ++-- .../com/gbase8c/dmt/common/util/MoUtils.java | 16 ++++ .../dmt/migration/MigrationObjectService.java | 19 +++++ .../dmt/migration/MigrationThread.java | 15 ++++ .../dmt/model/migration/dto/DboDto.java | 31 ------- .../dmt/model/migration/record/Mo.java | 16 ++++ .../com/gbase8c/dmt/service/TaskService.java | 3 + 8 files changed, 164 insertions(+), 37 deletions(-) create mode 100644 dmt/src/main/java/com/gbase8c/dmt/common/util/MoUtils.java create mode 100644 dmt/src/main/java/com/gbase8c/dmt/model/migration/record/Mo.java diff --git a/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java b/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java index 9f93704..154f61e 100644 --- a/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java +++ b/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java @@ -1,14 +1,58 @@ package com.alibaba.datax.common.util; +import com.alibaba.fastjson.JSON; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; public class ConfigurationUtil { + public static void jobState(Configuration configuration, String state, String msg) throws IOException { + String resume = System.getProperty("datax.persistence"); + if ("true".equalsIgnoreCase(resume)) { + String jobStatePath = "job.state"; + configuration.set(jobStatePath, state); + String json = configuration.toJSON(); + FileUtils.writeStringToFile(file(configuration), json, StandardCharsets.UTF_8, false); + + String jobName = configuration.getString("job.name"); + String workdir = configuration.getString("job.workdir"); + String id = configuration.getString("job._id"); + String filePath = StringUtils.joinWith(File.separator, workdir, id, "status", "data.json"); + + List mos = JSON.parseArray(FileUtils.readFileToString(new File(filePath), StandardCharsets.UTF_8), Mo.class); + Map map = new HashMap<>(); + for (Mo mo : mos) { + map.put(mo.getName(), mo); + } + + if (map.containsKey(jobName)) { + Mo mo = map.get(jobName); + //UnStart, InProgress, Finished, Failure; + // RUNNING, FAILED, SUCCEEDED + if ("RUNNING".equalsIgnoreCase(state)) { + mo.setStatus("InProgress"); + } + if ("FAILED".equalsIgnoreCase(state)) { + mo.setStatus("Failure"); + mo.setMsg(msg); + }if ("SUCCEEDED".equalsIgnoreCase(state)) { + mo.setStatus("Finished"); + } + String jsonStr = JSON.toJSONString(mos); + FileUtils.writeStringToFile(new File(filePath), jsonStr, StandardCharsets.UTF_8, false); + } + } + } + public static void persistence(Configuration configuration) throws IOException { String resume = System.getProperty("datax.persistence"); if ("true".equalsIgnoreCase(resume)) { @@ -34,4 +78,44 @@ public class ConfigurationUtil { return new File(StringUtils.joinWith(File.separator, workdir, id, "configuration", name + ".json")); } + public static class Mo { + private String type; + private String name; + private String status; + private String msg; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } + + } + } diff --git a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java index 0bbb284..3c71b4b 100644 --- a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java +++ b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java @@ -102,7 +102,8 @@ public class TaskGroupContainer extends AbstractContainer { public void start() { try { // todo 在这里存文件 - ConfigurationUtil.persistence(configuration); + String jobStatePath = "job.state"; + ConfigurationUtil.jobState(configuration, State.RUNNING.name(), null); /** * 状态check时间间隔，较短，可以把任务及时分发到对应channel中 */ @@ -175,9 +176,7 @@ public class TaskGroupContainer extends AbstractContainer { String path = "job.content[" + taskId + "].state"; // 失败，看task是否支持failover，重试次数未超过最大限制 if (taskCommunication.getState() == State.FAILED) { - // todo 记录状态为State.FAILED - configuration.set(path, State.FAILED); - ConfigurationUtil.persistence(configuration); + taskFailedExecutorMap.put(taskId, taskExecutor); if (taskExecutor.supportFailOver() && taskExecutor.getAttemptCount() < taskMaxRetryTimes) { @@ -187,6 +186,10 @@ public class TaskGroupContainer extends AbstractContainer { taskQueue.add(taskConfig); //重新加入任务列表 } else { failedOrKilled = true; + // todo 记录状态为State.FAILED + configuration.set(path, State.FAILED); + ConfigurationUtil.persistence(configuration); + break; } } else if (taskCommunication.getState() == State.KILLED) { @@ -216,6 +219,8 @@ public class TaskGroupContainer extends AbstractContainer { if (failedOrKilled) { lastTaskGroupContainerCommunication = reportTaskGroupCommunication( lastTaskGroupContainerCommunication, taskCountInThisTaskGroup); + // todo + ConfigurationUtil.jobState(configuration, State.FAILED.name(), lastTaskGroupContainerCommunication.getThrowable().getMessage()); throw DataXException.asDataXException( FrameworkErrorCode.PLUGIN_RUNTIME_ERROR, lastTaskGroupContainerCommunication.getThrowable()); @@ -277,8 +282,8 @@ public class TaskGroupContainer extends AbstractContainer { lastTaskGroupContainerCommunication, taskCountInThisTaskGroup); LOG.info("taskGroup[{}] completed it's tasks.", this.taskGroupId); - // todo 任务执行完成, remove配置文件 - ConfigurationUtil.remove(configuration); + // todo 任务执行完成 + ConfigurationUtil.jobState(configuration, State.SUCCEEDED.name(), null); break; } diff --git a/dmt/src/main/java/com/gbase8c/dmt/common/util/MoUtils.java b/dmt/src/main/java/com/gbase8c/dmt/common/util/MoUtils.java new file mode 100644 index 0000000..07a25e2 --- /dev/null +++ b/dmt/src/main/java/com/gbase8c/dmt/common/util/MoUtils.java @@ -0,0 +1,16 @@ +package com.gbase8c.dmt.common.util; + +import com.gbase8c.dmt.config.DmtConfig; +import com.gbase8c.dmt.model.enums.MigrationObjectType; +import com.gbase8c.dmt.model.migration.record.Mo; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.util.List; + +public class MoUtils { + public static void persistent(DmtConfig dmtConfig, MigrationObjectType mot, List mos) { + String dbObjectFile = StringUtils.joinWith(File.separator, dmtConfig.getWorkDir(), dmtConfig.getTaskId(), "status", mot.getName() + ".json"); + JsonMapper.nonEmptyMapper().toFile(new File(dbObjectFile), mos); + } +} diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java index bca253a..bdd062a 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java @@ -3,6 +3,7 @@ package com.gbase8c.dmt.migration; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.gbase8c.dmt.common.util.JsonMapper; +import com.gbase8c.dmt.common.util.MoUtils; import com.gbase8c.dmt.config.DmtConfig; import com.gbase8c.dmt.dao.DataSourceDao; import com.gbase8c.dmt.dao.SnapshotDao; @@ -10,6 +11,7 @@ import com.gbase8c.dmt.db.metadata.Metadata; import com.gbase8c.dmt.db.metadata.MetadataFactory; import com.gbase8c.dmt.model.enums.MigrationObjectType; import com.gbase8c.dmt.model.enums.DbType; +import com.gbase8c.dmt.model.enums.Status; import com.gbase8c.dmt.model.migration.config.MigrateConfig; import com.gbase8c.dmt.model.migration.config.Snapshot; import com.gbase8c.dmt.model.migration.config.Task; @@ -19,6 +21,7 @@ import com.gbase8c.dmt.model.migration.dto.MigrationObject; import com.gbase8c.dmt.model.migration.dto.TableDto; import com.gbase8c.dmt.model.migration.job.DataxJob; import com.gbase8c.dmt.model.enums.MigrationTaskStatus; +import com.gbase8c.dmt.model.migration.record.Mo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import lombok.extern.slf4j.Slf4j; @@ -112,6 +115,22 @@ public class MigrationObjectService { orderedDbObjects.put(migrationObjectType, migrationObjects(dbObjects.get(migrationObjectType))); } } + + for (Map.Entry> entry : orderedDbObjects.entrySet()) { + MigrationObjectType migrationObjectType = entry.getKey(); + List migrationObjects = entry.getValue(); + List mos = migrationObjects.stream() + .map(migrationObject -> Mo.builder() + .type(migrationObjectType.getName()) + .name((MigrationObjectType.TABLESPACE.equals(migrationObjectType) + || MigrationObjectType.SCHEMA.equals(migrationObjectType)) + ? migrationObject.getName() : migrationObject.getSchema() + "." + migrationObject.getName()) + .status(Status.UnStart.name()) + .build()) + .collect(Collectors.toList()); + MoUtils.persistent(dmtConfig, migrationObjectType, mos); + } + return orderedDbObjects; } diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java index 59e3cd3..ed94124 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java @@ -1,5 +1,6 @@ package com.gbase8c.dmt.migration; +import com.gbase8c.dmt.common.util.MoUtils; import com.gbase8c.dmt.config.DmtConfig; import com.gbase8c.dmt.dao.RecordDao; import com.gbase8c.dmt.dao.TaskDao; @@ -16,6 +17,7 @@ import com.gbase8c.dmt.model.migration.record.JobThreadRecord; import com.gbase8c.dmt.model.migration.record.MigrationRecordDto; import com.gbase8c.dmt.model.enums.MigrationTaskStatus; import com.gbase8c.dmt.model.enums.Status; +import com.gbase8c.dmt.model.migration.record.Mo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import lombok.Getter; @@ -305,6 +307,19 @@ public class MigrationThread extends Thread implements Closeable { .filter(m -> MigrationObjectType.TABLE.equals(m.getMigrationObjectType())) .map(MigrationObject::getTableMapper) .collect(Collectors.toList()); + if (!type.equals(MigrationObjectType.DATA)) { + List mos = jobThreadRecords.stream() + .map(record -> Mo.builder() + .type(type.getName()) + .name((MigrationObjectType.TABLESPACE.equals(type) + || MigrationObjectType.SCHEMA.equals(type)) + ? record.getDbObject().getName() : record.getDbObject().getSchema() + "." + record.getDbObject().getName()) + .status(record.getFlag() ? Status.Finished.name() : Status.Failure.name()) + .msg(record.getFlag() ? null : record.getErrorLog()) + .build()) + .collect(Collectors.toList()); + MoUtils.persistent(dmtConfig, type, mos); + } this.tableMappers.addAll(tableMappers); Map> jobListMap = jobThreadRecords .stream() diff --git a/dmt/src/main/java/com/gbase8c/dmt/model/migration/dto/DboDto.java b/dmt/src/main/java/com/gbase8c/dmt/model/migration/dto/DboDto.java index 88b5e2d..b910a3c 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/model/migration/dto/DboDto.java +++ b/dmt/src/main/java/com/gbase8c/dmt/model/migration/dto/DboDto.java @@ -58,37 +58,6 @@ public class DboDto { dbObjects.put(MigrationObjectType.TABLE, tableDtos); } -// // index -// if (migrateConfig.getMigrateIndex()) { -// List tableIndexDtos = Lists.newArrayList(); -// Map> tableNameIndexDtos = getSchemaDtos().stream() -// .flatMap(schemaDto -> schemaDto.getIndexDtos().stream()) -// .collect(Collectors.groupingBy(IndexDto::getTableName)); -// for (Map.Entry> entry :tableNameIndexDtos.entrySet()) { -// List indexDtos = entry.getValue(); -// List tableIndexContents = indexDtos.stream() -// .flatMap(indexDto -> indexDto.getContents().stream()) -// .collect(Collectors.toList()); -// IndexDto tableIndexDto = IndexDto.builder() -// .migrationObjectType(indexDtos.get(0).getMigrationObjectType()) -// .schema(indexDtos.get(0).getSchema()) -// .tarSchema(indexDtos.get(0).getTarSchema()) -// .name(indexDtos.get(0).getMigrationObjectType().getName()) -// .tarName(indexDtos.get(0).getMigrationObjectType().getName()) -// .contents(tableIndexContents) -// .status(indexDtos.get(0).getStatus()) -// .initializable(indexDtos.get(0).getInitializable()) -// .convertible(indexDtos.get(0).getConvertible()) -// .initMsg(indexDtos.get(0).getInitMsg()) -// .convertMsg(indexDtos.get(0).getConvertMsg()) -// .noteMsg(indexDtos.get(0).getNoteMsg()) -// .msg(indexDtos.get(0).getMsg()) -// .build(); -// tableIndexDtos.add(tableIndexDto); -// } -// dbObjects.put(MigrationObjectType.INDEX, tableIndexDtos); -// } - // index if (migrateConfig.getMigrateIndex()) { List indexDtos = getSchemaDtos().stream() diff --git a/dmt/src/main/java/com/gbase8c/dmt/model/migration/record/Mo.java b/dmt/src/main/java/com/gbase8c/dmt/model/migration/record/Mo.java new file mode 100644 index 0000000..1d3321a --- /dev/null +++ b/dmt/src/main/java/com/gbase8c/dmt/model/migration/record/Mo.java @@ -0,0 +1,16 @@ +package com.gbase8c.dmt.model.migration.record; + +import lombok.*; + +@Getter +@Setter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Mo { + private String type; + private String name; + private String status; + private String msg; + +} diff --git a/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java b/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java index 8490086..bd6c8b6 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java +++ b/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java @@ -389,6 +389,9 @@ public class TaskService { private void convert(MigrationTask mt) { Task task = mt.getTask(); + MigrateConfig migrateConfig = task.getMigrateConfig(); + Boolean migrateData = migrateConfig.getMigrateData(); + DboDto dboDto = mt.getDboDto(); MigrationTaskStatus status = MigrationTaskStatus.CONVERTING; -- Gitee From d2e0b701828ecfa09c31089bd4c233107ec9f707 Mon Sep 17 00:00:00 2001 From: zephyr Date: Thu, 14 Sep 2023 14:05:48 +0800 Subject: [PATCH 05/15] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E8=BF=81=E7=A7=BB?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E5=88=A0=E9=99=A4=E5=88=86=E5=9D=97=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/alibaba/datax/core/taskgroup/TaskGroupContainer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java index 3c71b4b..9adb58b 100644 --- a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java +++ b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java @@ -284,6 +284,7 @@ public class TaskGroupContainer extends AbstractContainer { LOG.info("taskGroup[{}] completed it's tasks.", this.taskGroupId); // todo 任务执行完成 ConfigurationUtil.jobState(configuration, State.SUCCEEDED.name(), null); + ConfigurationUtil.remove(configuration); break; } -- Gitee From cba6ee8e0c94cde134eaa5128c2d6942160dca3f Mon Sep 17 00:00:00 2001 From: zephyr Date: Fri, 15 Sep 2023 11:19:48 +0800 Subject: [PATCH 06/15] =?UTF-8?q?=E6=9B=B4=E6=94=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E5=AF=B9=E8=B1=A1=E6=89=A7=E8=A1=8C=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datax/common/util/ConfigurationUtil.java | 54 ++-- .../dmt/migration/MigrationObjectService.java | 1 - .../dmt/migration/MigrationThread.java | 258 ++++++------------ .../com/gbase8c/dmt/migration/MoLogger.java | 115 ++++++++ .../dmt/migration/job/AbstractJobThread.java | 3 + .../migration/job/ConstraintJobThread.java | 5 +- .../dmt/migration/job/DataXJobThread.java | 17 +- .../dmt/migration/job/FunctionJobThread.java | 5 +- .../dmt/migration/job/IndexJobThread.java | 5 +- .../dmt/migration/job/SchemaJobThread.java | 5 +- .../dmt/migration/job/SqlJobThread.java | 33 ++- .../dmt/migration/job/SynonymJobThread.java | 5 +- .../dmt/migration/job/TableJobThread.java | 5 +- .../migration/job/TableSpaceJobThread.java | 5 +- .../dmt/migration/job/ViewJobThread.java | 5 +- 15 files changed, 288 insertions(+), 233 deletions(-) create mode 100644 dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java diff --git a/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java b/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java index 154f61e..02df8da 100644 --- a/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java +++ b/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java @@ -23,33 +23,33 @@ public class ConfigurationUtil { String json = configuration.toJSON(); FileUtils.writeStringToFile(file(configuration), json, StandardCharsets.UTF_8, false); - String jobName = configuration.getString("job.name"); - String workdir = configuration.getString("job.workdir"); - String id = configuration.getString("job._id"); - String filePath = StringUtils.joinWith(File.separator, workdir, id, "status", "data.json"); - - List mos = JSON.parseArray(FileUtils.readFileToString(new File(filePath), StandardCharsets.UTF_8), Mo.class); - Map map = new HashMap<>(); - for (Mo mo : mos) { - map.put(mo.getName(), mo); - } - - if (map.containsKey(jobName)) { - Mo mo = map.get(jobName); - //UnStart, InProgress, Finished, Failure; - // RUNNING, FAILED, SUCCEEDED - if ("RUNNING".equalsIgnoreCase(state)) { - mo.setStatus("InProgress"); - } - if ("FAILED".equalsIgnoreCase(state)) { - mo.setStatus("Failure"); - mo.setMsg(msg); - }if ("SUCCEEDED".equalsIgnoreCase(state)) { - mo.setStatus("Finished"); - } - String jsonStr = JSON.toJSONString(mos); - FileUtils.writeStringToFile(new File(filePath), jsonStr, StandardCharsets.UTF_8, false); - } +// String jobName = configuration.getString("job.name"); +// String workdir = configuration.getString("job.workdir"); +// String id = configuration.getString("job._id"); +// String filePath = StringUtils.joinWith(File.separator, workdir, id, "status", "data.json"); +// +// List mos = JSON.parseArray(FileUtils.readFileToString(new File(filePath), StandardCharsets.UTF_8), Mo.class); +// Map map = new HashMap<>(); +// for (Mo mo : mos) { +// map.put(mo.getName(), mo); +// } +// +// if (map.containsKey(jobName)) { +// Mo mo = map.get(jobName); +// //UnStart, InProgress, Finished, Failure; +// // RUNNING, FAILED, SUCCEEDED +// if ("RUNNING".equalsIgnoreCase(state)) { +// mo.setStatus("InProgress"); +// } +// if ("FAILED".equalsIgnoreCase(state)) { +// mo.setStatus("Failure"); +// mo.setMsg(msg); +// }if ("SUCCEEDED".equalsIgnoreCase(state)) { +// mo.setStatus("Finished"); +// } +// String jsonStr = JSON.toJSONString(mos); +// FileUtils.writeStringToFile(new File(filePath), jsonStr, StandardCharsets.UTF_8, false); +// } } } diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java index bdd062a..7731a52 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java @@ -167,7 +167,6 @@ public class MigrationObjectService { .jobId(jobId) .src(src) .tar(tar) -// .tableDto(tableDto) .contents(Lists.newArrayList(JSON.toJSONString(dataxMap))) .json(JSON.toJSONString(dataxMap)) .build()); diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java index ed94124..a0b48c5 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java @@ -35,57 +35,27 @@ import java.util.concurrent.*; import java.util.stream.Collectors; @Slf4j +@Getter +@Setter public class MigrationThread extends Thread implements Closeable { - @Getter - @Setter - private String taskId; - @Getter - @Setter - private MigrationRecordDto recordDto ; - @Getter - @Setter - DmtConfig dmtConfig; - @Getter - @Setter + private String taskId; + private MigrationRecordDto recordDto; + private DmtConfig dmtConfig; private List recordList = new CopyOnWriteArrayList<>(); - @Getter - @Setter private List tableMappers = new CopyOnWriteArrayList<>(); - @Getter - @Setter private RecordDao recordDao; - @Getter - @Setter private TaskDao taskDao; - - @Getter - @Setter private volatile MigrationTaskStatus migrationTaskStatus = MigrationTaskStatus.RUNNING; - private ThreadPoolTaskExecutor threadPoolTaskExecutor = null; - @Getter - @Setter private DataSource tarDataSource; - - @Setter - @Getter protected volatile boolean suspend = false; - @Setter - @Getter protected volatile boolean stop = false; - - final BlockingQueue> queue - = new LinkedBlockingQueue>(); - - CompletionService completionService = null; - - @Getter - @Setter + private final BlockingQueue> queue = new LinkedBlockingQueue>(); + private CompletionService completionService = null; private Map> migrationObjectListMap; - @Getter - @Setter - private Map> jobListMap = new LinkedHashMap<>(); + private Map> jobListMap = new LinkedHashMap<>(); + private Map moLoggerMap = new HashMap<>(); public static MigrationThread build(DboDto dboDto, Map> orderedDbObject, @@ -99,7 +69,7 @@ public class MigrationThread extends Thread implements Closeable { return migrationThread; } - public void init(DmtConfig dmtConfig){ + public void init(DmtConfig dmtConfig) { this.migrationTaskStatus = MigrationTaskStatus.RUNNING; this.setDmtConfig(dmtConfig); initThreadPool(); @@ -110,13 +80,13 @@ public class MigrationThread extends Thread implements Closeable { /** * 初始化线程池 */ - private void initThreadPool(){ + private void initThreadPool() { //todo:初始化线程池,前期默认，后续换成配置中取 threadPoolTaskExecutor = new ThreadPoolTaskExecutor(); - if (null!=dmtConfig){ + if (null != dmtConfig) { threadPoolTaskExecutor.setCorePoolSize(dmtConfig.getTaskPoolCoreSize()); threadPoolTaskExecutor.setMaxPoolSize(dmtConfig.getTaskPoolMaxSize()); - }else { + } else { threadPoolTaskExecutor.setCorePoolSize(10); threadPoolTaskExecutor.setMaxPoolSize(15); } @@ -127,95 +97,48 @@ public class MigrationThread extends Thread implements Closeable { threadPoolTaskExecutor.setAwaitTerminationSeconds(60); threadPoolTaskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); threadPoolTaskExecutor.initialize(); - this.completionService = new ExecutorCompletionService<>(threadPoolTaskExecutor,queue); + this.completionService = new ExecutorCompletionService<>(threadPoolTaskExecutor, queue); log.info("\n maxPoolSize : {} " + - "\n corePoolSize : {} ",threadPoolTaskExecutor.getMaxPoolSize(),threadPoolTaskExecutor.getCorePoolSize()); + "\n corePoolSize : {} ", threadPoolTaskExecutor.getMaxPoolSize(), threadPoolTaskExecutor.getCorePoolSize()); } - private void initTaskList(){ - Iterator>> iterator = migrationObjectListMap.entrySet().iterator(); - while (iterator.hasNext()){ - Map.Entry> taskMap = iterator.next(); + private void initTaskList() { + for (Map.Entry> entry : migrationObjectListMap.entrySet()) { List jobList = new ArrayList<>(); - MigrationObjectType type = taskMap.getKey(); - for (MigrationObject obj:taskMap.getValue()) { + MigrationObjectType type = entry.getKey(); + MoLogger moLogger = new MoLogger(dmtConfig.getWorkDir(), dmtConfig.getTaskId(), type.getName()); + moLoggerMap.put(type, moLogger); + for (MigrationObject obj : entry.getValue()) { AbstractJobThread jobThread = null; - switch (type){ -// case SCHEMA: -// jobThread = new SchemaJobThread(type,obj,tarDataSource); -// break; -// case TABLE: -// jobThread = new TableJobThread(type,obj,tarDataSource); -// break; -// case TABLESPACE: -// jobThread = new TableSpaceJobThread(type,obj,tarDataSource); -// break; - case DATA: - jobThread = new DataXJobThread(type,obj,tarDataSource); - break; - default: - jobThread = new SqlJobThread(type, obj, tarDataSource); - break; + if (type == MigrationObjectType.DATA) { + jobThread = new DataXJobThread(type, obj, tarDataSource, moLogger); + } else { + jobThread = new SqlJobThread(type, obj, tarDataSource, moLogger); } jobList.add(jobThread); } - this.jobListMap.put(taskMap.getKey(),jobList); + this.jobListMap.put(type, jobList); } } - - private void initMigrationRecord(){ + private void initMigrationRecord() { this.recordDto = new MigrationRecordDto(); this.recordDto.setTaskId(taskId); this.recordDto.setStartTime(new Date()); -// //todo:重构结果返回值 -// List recordList = new ArrayList<>(); -// for (Map.Entry> dbObjectTypeListEntry : migrationObjectListMap.entrySet()) { -// ExecuteRecord record = new ExecuteRecord<>(); -// record.setType(dbObjectTypeListEntry.getKey()); -//// System.out.println(dbObjectTypeListEntry.getValue()); -// if (!dbObjectTypeListEntry.getValue().isEmpty()){ -// record.setTotal(dbObjectTypeListEntry.getValue().size()); -// }else { -// continue; -// } -// recordList.add(record); -// } -// this.setRecordList(recordList); } /** * 转换结果值 - * @param recordList - * @return */ - private Map> convertRecords(List recordList) { + private Map> convertRecords(List recordList) { Map> schemaMap = Maps.newHashMap(); - if (!CollectionUtils.isEmpty(recordList)){ + if (!CollectionUtils.isEmpty(recordList)) { schemaMap = recordList.stream().collect(Collectors.groupingBy(ExecuteRecord::getSchemaName)); } - return schemaMap; + return schemaMap; } - /** - * Causes this thread to begin execution; the Java Virtual Machine - * calls the run method of this thread. - *

- * The result is that two threads are running concurrently: the - * current thread (which returns from the call to the - * start method) and the other thread (which executes its - * run method). - *

- * It is never legal to start a thread more than once. - * In particular, a thread may not be restarted once it has completed - * execution. - * - * @throws IllegalThreadStateException if the thread was already - * started. - * @see #run() - * @see #stop() - */ @Override public synchronized void start() { this.migrationTaskStatus = MigrationTaskStatus.RUNNING; @@ -223,44 +146,49 @@ public class MigrationThread extends Thread implements Closeable { task.setStatus(MigrationTaskStatus.RUNNING); MigrationObjectService.updateTaskStatus(taskId, MigrationTaskStatus.RUNNING); taskDao.update(task); - log.info("[MigrationThread] mission : "+taskId + "， started !!! "); - jobListMap.entrySet().forEach(entry ->{ - long begin = System.currentTimeMillis(); - MigrationObjectType type = entry.getKey(); - List jobList = entry.getValue(); - //分批执行任务 - for (int i = 0; i < jobList.size(); i++) { - completionService.submit(jobList.get(i)); - } - //阻塞获取值 - futureTake(jobList.size(), type); - long end = System.currentTimeMillis(); - log.info("\n********************************************************************************************************"+ - "\n* type : {}" + - "\n* job size : {} " + - "\n* start time : {} " + - "\n* end time : {} " + - "\n* spend total time : {}ms " + - "\n********************************************************************************************************" + log.info("[MigrationThread] mission : " + taskId + "， started !!! "); + jobListMap.forEach((type, jobList) -> { + MoLogger moLogger = moLoggerMap.get(type); + if (moLogger != null) { + moLogger.start(); + } + long begin = System.currentTimeMillis(); + //分批执行任务 + for (AbstractJobThread abstractJobThread : jobList) { + completionService.submit(abstractJobThread); + } + //阻塞获取值 + futureTake(jobList.size(), type); + long end = System.currentTimeMillis(); + log.info("\n********************************************************************************************************" + + "\n* type : {}" + + "\n* job size : {} " + + "\n* start time : {} " + + "\n* end time : {} " + + "\n* spend total time : {}ms " + + "\n********************************************************************************************************" , type - ,jobList.size() - , DateFormatUtils.format(begin,"yyyy-MM-dd HH:mm:ss.SSS") - , DateFormatUtils.format(end,"yyyy-MM-dd HH:mm:ss.SSS") + , jobList.size() + , DateFormatUtils.format(begin, "yyyy-MM-dd HH:mm:ss.SSS") + , DateFormatUtils.format(end, "yyyy-MM-dd HH:mm:ss.SSS") , (end - begin)); - }); + if (moLogger != null) { + moLogger.stop(); + } + }); this.recordDto.setEndTime(new Date()); this.recordDto.setStatus(1); int count = this.recordList.stream().mapToInt(ExecuteRecord::getFailed).sum(); MigrationTaskStatus status = MigrationTaskStatus.SUCCESS; - if (count>0){ + if (count > 0) { status = MigrationTaskStatus.FAILED; } this.recordDto.setMigrationTaskStatus(status); try { this.recordDto.setSchemaRecordsMap(convertRecords(this.recordList)); this.recordDto.setTableMappers(this.tableMappers); - }catch (Exception e){ - log.error(e.getMessage(),e); + } catch (Exception e) { + log.error(e.getMessage(), e); status = MigrationTaskStatus.FAILED; } this.setMigrationTaskStatus(status); @@ -268,28 +196,19 @@ public class MigrationThread extends Thread implements Closeable { MigrationObjectService.updateTaskStatus(taskId, status); MigrationObjectService.removeTaskStatus(taskId); threadPoolTaskExecutor.shutdown(); -// try { -// this.close(); -// } catch (IOException exception) { -// exception.printStackTrace(); -// } } - public MigrationRecordDto getRunningRecord(){ + public MigrationRecordDto getRunningRecord() { MigrationRecordDto runningRecord = this.recordDto; Map> convertRecords = convertRecords(this.recordList); runningRecord.setSchemaRecordsMap(convertRecords); return runningRecord; } - - /** * 获取任务执行的值 - * @param jobSize - * @param type */ - private void futureTake(int jobSize, MigrationObjectType type){ + private void futureTake(int jobSize, MigrationObjectType type) { List jobThreadRecords = new ArrayList<>(); for (int i = 0; i < jobSize; i++) { try { @@ -307,25 +226,25 @@ public class MigrationThread extends Thread implements Closeable { .filter(m -> MigrationObjectType.TABLE.equals(m.getMigrationObjectType())) .map(MigrationObject::getTableMapper) .collect(Collectors.toList()); - if (!type.equals(MigrationObjectType.DATA)) { - List mos = jobThreadRecords.stream() - .map(record -> Mo.builder() - .type(type.getName()) - .name((MigrationObjectType.TABLESPACE.equals(type) - || MigrationObjectType.SCHEMA.equals(type)) - ? record.getDbObject().getName() : record.getDbObject().getSchema() + "." + record.getDbObject().getName()) - .status(record.getFlag() ? Status.Finished.name() : Status.Failure.name()) - .msg(record.getFlag() ? null : record.getErrorLog()) - .build()) - .collect(Collectors.toList()); - MoUtils.persistent(dmtConfig, type, mos); - } +// if (!type.equals(MigrationObjectType.DATA)) { +// List mos = jobThreadRecords.stream() +// .map(record -> Mo.builder() +// .type(type.getName()) +// .name((MigrationObjectType.TABLESPACE.equals(type) +// || MigrationObjectType.SCHEMA.equals(type)) +// ? record.getDbObject().getName() : record.getDbObject().getSchema() + "." + record.getDbObject().getName()) +// .status(record.getFlag() ? Status.Finished.name() : Status.Failure.name()) +// .msg(record.getFlag() ? null : record.getErrorLog()) +// .build()) +// .collect(Collectors.toList()); +// MoUtils.persistent(dmtConfig, type, mos); +// } this.tableMappers.addAll(tableMappers); Map> jobListMap = jobThreadRecords .stream() .collect(Collectors.groupingBy(JobThreadRecord::getSchemaName)); //任务结果转换 - jobListMap.entrySet().stream().forEach(entry ->{ + jobListMap.entrySet().stream().forEach(entry -> { int success = entry.getValue().stream().mapToInt(jobRecord -> jobRecord.getFlag() ? 1 : 0).sum(); int failed = entry.getValue().stream().mapToInt(jobRecord -> jobRecord.getFlag() ? 0 : 1).sum(); List errorObjs = Lists.newArrayList(); @@ -333,12 +252,12 @@ public class MigrationThread extends Thread implements Closeable { List errorLogList = Lists.newArrayList(); successObjs = entry.getValue().stream() .map(JobThreadRecord::getDbObject) - .filter(errorObj -> Status.Finished==errorObj.getStatus()) + .filter(errorObj -> Status.Finished == errorObj.getStatus()) .collect(Collectors.toList()); - if (failed != 0 ){ + if (failed != 0) { errorObjs = entry.getValue().stream() .map(JobThreadRecord::getDbObject) - .filter(errorObj -> Status.Failure==errorObj.getStatus()) + .filter(errorObj -> Status.Failure == errorObj.getStatus()) .collect(Collectors.toList()); errorLogList = entry.getValue().stream().map(JobThreadRecord::getErrorLog).filter(obj -> null != obj).collect(Collectors.toList()); } @@ -357,23 +276,6 @@ public class MigrationThread extends Thread implements Closeable { this.recordList.addAll(executeRecords); } - - - - - /** - * Closes this stream and releases any system resources associated - * with it. If the stream is already closed then invoking this - * method has no effect. - * - *

As noted in {@link AutoCloseable#close()}, cases where the - * close may fail require careful attention. It is strongly advised - * to relinquish the underlying resources and to internally - * mark the {@code Closeable} as closed, prior to throwing - * the {@code IOException}. - * - * @throws IOException if an I/O error occurs - */ @Override public void close() throws IOException { this.stop = true; diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java new file mode 100644 index 0000000..5577de4 --- /dev/null +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java @@ -0,0 +1,115 @@ +package com.gbase8c.dmt.migration; + +import com.alibaba.fastjson.JSON; +import com.gbase8c.dmt.model.migration.record.Mo; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +public class MoLogger { + private final ConcurrentHashMap map = new ConcurrentHashMap<>(); + private final Logger logger; + private Thread thread; + + public MoLogger(String workDir, String taskId, String type) { + File file = file(workDir, taskId, type); + logger = new Logger(file, map); + } + + public MoLogger(String workDir, String taskId, String type, long wait) { + File file = file(workDir, taskId, type); + logger = new Logger(file, map, wait); + } + + private File file(String workDir, String taskId, String type) { + String filePath = StringUtils.joinWith(File.separator, workDir, taskId, "status", type + ".json"); + return new File(filePath); + } + + public void start() { + thread = new Thread(logger); + thread.start(); + } + + public void stop() { + try { + thread.join(); + logger.setStop(true); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + public void log(Mo mo) { + map.put(mo.getName(), mo); + } + + @Getter + @Setter + @Slf4j + public static class Logger implements Runnable { + + private ConcurrentHashMap map; + private long wait = 10000L; + + private File file; + private boolean stop = false; + + public Logger(File file, ConcurrentHashMap map) { + this.file = file; + this.map = map; + } + + public Logger(File file, ConcurrentHashMap map, long wait) { + this.file = file; + this.map = map; + this.wait = wait; + } + + @Override + public void run() { + log.info("开始数据库对象状态更新: " + file.getPath()); + initMap(); + String lastJson = JSON.toJSONString(map.values()); + while (!stop) { + String newJson = JSON.toJSONString(map.values()); + if (!newJson.equalsIgnoreCase(lastJson)) { + lastJson = newJson; + try { + FileUtils.writeStringToFile(file, newJson, StandardCharsets.UTF_8); + Thread.sleep(wait); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + log.info("停止数据库对象状态更新: " + file.getPath()); + String json = JSON.toJSONString(map.values()); + try { + FileUtils.writeStringToFile(file, json, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void initMap() { + try { + List mos = JSON.parseArray(FileUtils.readFileToString(file, StandardCharsets.UTF_8), Mo.class); + for (Mo mo : mos) { + map.put(mo.getName(), mo); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + +} diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/job/AbstractJobThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/job/AbstractJobThread.java index 74acf17..1f6920e 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/job/AbstractJobThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/job/AbstractJobThread.java @@ -1,5 +1,6 @@ package com.gbase8c.dmt.migration.job; +import com.gbase8c.dmt.migration.MoLogger; import com.gbase8c.dmt.model.enums.MigrationObjectType; import com.gbase8c.dmt.model.migration.dto.MigrationObject; import com.gbase8c.dmt.model.migration.record.JobThreadRecord; @@ -18,6 +19,8 @@ public abstract class AbstractJobThread implements Callable { protected DataSource tarDataSource; + protected MoLogger moLogger; + /** * Computes a result, or throws an exception if unable to do so. * diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/job/ConstraintJobThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/job/ConstraintJobThread.java index 3795ea9..50aea00 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/job/ConstraintJobThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/job/ConstraintJobThread.java @@ -1,5 +1,6 @@ package com.gbase8c.dmt.migration.job; +import com.gbase8c.dmt.migration.MoLogger; import com.gbase8c.dmt.model.enums.MigrationObjectType; import com.gbase8c.dmt.model.migration.dto.MigrationObject; import com.gbase8c.dmt.model.migration.record.JobThreadRecord; @@ -9,8 +10,8 @@ import java.util.concurrent.Callable; public class ConstraintJobThread extends AbstractJobThread implements Callable { - public ConstraintJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource) { - super(type, dbObject, tarDataSource); + public ConstraintJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource, MoLogger moLogger) { + super(type, dbObject, tarDataSource, moLogger); } /** diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/job/DataXJobThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/job/DataXJobThread.java index 95742e0..edfaaca 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/job/DataXJobThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/job/DataXJobThread.java @@ -1,14 +1,17 @@ package com.gbase8c.dmt.migration.job; + import com.alibaba.datax.common.util.Configuration; import com.alibaba.datax.core.Engine; import com.alibaba.datax.core.util.ConfigParser; import com.gbase8c.dmt.common.util.Exceptions; import com.gbase8c.dmt.common.util.JsonMapper; +import com.gbase8c.dmt.migration.MoLogger; import com.gbase8c.dmt.model.enums.MigrationObjectType; import com.gbase8c.dmt.model.migration.dto.MigrationObject; import com.gbase8c.dmt.model.migration.record.JobThreadRecord; import com.gbase8c.dmt.model.enums.Status; import com.gbase8c.dmt.migration.MigrationObjectService; +import com.gbase8c.dmt.model.migration.record.Mo; import lombok.extern.slf4j.Slf4j; import javax.sql.DataSource; @@ -18,8 +21,8 @@ import java.util.Map; @Slf4j public class DataXJobThread extends AbstractJobThread { - public DataXJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource) { - super(type, dbObject, tarDataSource); + public DataXJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource, MoLogger moLogger) { + super(type, dbObject, tarDataSource, moLogger); } /** @@ -30,6 +33,12 @@ public class DataXJobThread extends AbstractJobThread { */ @Override public JobThreadRecord call() throws Exception { + Mo mo = Mo.builder() +// .type(type.getName()) + .name(dbObject.getSchema() + "." + dbObject.getName()) + .status(Status.InProgress.name()) + .build(); + moLogger.log(mo); JobThreadRecord jobThreadRecord = new JobThreadRecord(); jobThreadRecord.setFlag(true); List contents = dbObject.getContents(); @@ -38,7 +47,6 @@ public class DataXJobThread extends AbstractJobThread { dbObject.setStatus(Status.InProgress); String dataXConfig = contents.get(0); Configuration configuration = ConfigParser.parseJson(dataXConfig); -// String jobName = dataxJob.getTableDto().getSchema() + "." + dataxJob.getTableDto().getTableName(); long jobId = MigrationObjectService.toLong(dbObject.getSchema()+"."+dbObject.getName()); configuration.set("core.container.job.id", jobId); Map dataxMap = JsonMapper.nonEmptyMapper().fromJson(dataXConfig, Map.class); @@ -47,12 +55,15 @@ public class DataXJobThread extends AbstractJobThread { Engine engine = new Engine(); engine.start(configuration); dbObject.setStatus(Status.Finished); + mo.setName(Status.Finished.name()); } catch (Exception e) { log.error("datax error!", e); dbObject.setStatus(Status.Failure); dbObject.setMsg(e.getMessage()); jobThreadRecord.setErrorLog(Exceptions.getStackTraceAsString(e)); jobThreadRecord.setFlag(false); + mo.setStatus(Status.Failure.name()); + mo.setMsg(e.getMessage()); } jobThreadRecord.setDbObject(dbObject); return jobThreadRecord; diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/job/FunctionJobThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/job/FunctionJobThread.java index 00f4a7a..b155ebf 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/job/FunctionJobThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/job/FunctionJobThread.java @@ -1,5 +1,6 @@ package com.gbase8c.dmt.migration.job; +import com.gbase8c.dmt.migration.MoLogger; import com.gbase8c.dmt.model.enums.MigrationObjectType; import com.gbase8c.dmt.model.migration.dto.MigrationObject; import com.gbase8c.dmt.model.migration.record.JobThreadRecord; @@ -8,8 +9,8 @@ import javax.sql.DataSource; public class FunctionJobThread extends AbstractJobThread{ - public FunctionJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource) { - super(type, dbObject, tarDataSource); + public FunctionJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource, MoLogger moLogger) { + super(type, dbObject, tarDataSource, moLogger); } /** diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/job/IndexJobThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/job/IndexJobThread.java index ee4af33..460b4de 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/job/IndexJobThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/job/IndexJobThread.java @@ -1,5 +1,6 @@ package com.gbase8c.dmt.migration.job; +import com.gbase8c.dmt.migration.MoLogger; import com.gbase8c.dmt.model.enums.MigrationObjectType; import com.gbase8c.dmt.model.migration.dto.MigrationObject; import com.gbase8c.dmt.model.migration.record.JobThreadRecord; @@ -8,8 +9,8 @@ import javax.sql.DataSource; public class IndexJobThread extends AbstractJobThread{ - public IndexJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource) { - super(type, dbObject, tarDataSource); + public IndexJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource, MoLogger moLogger) { + super(type, dbObject, tarDataSource, moLogger); } /** diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/job/SchemaJobThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/job/SchemaJobThread.java index f87646d..bc009c2 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/job/SchemaJobThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/job/SchemaJobThread.java @@ -2,6 +2,7 @@ package com.gbase8c.dmt.migration.job; import com.gbase8c.dmt.common.util.Exceptions; import com.gbase8c.dmt.common.util.Reflections; +import com.gbase8c.dmt.migration.MoLogger; import com.gbase8c.dmt.model.enums.MigrationObjectType; import com.gbase8c.dmt.model.migration.dto.MigrationObject; import com.gbase8c.dmt.model.migration.record.JobThreadRecord; @@ -15,8 +16,8 @@ import java.sql.SQLException; @Slf4j public class SchemaJobThread extends AbstractJobThread{ - public SchemaJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource) { - super(type, dbObject, tarDataSource); + public SchemaJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource, MoLogger moLogger) { + super(type, dbObject, tarDataSource, moLogger); } /** diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/job/SqlJobThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/job/SqlJobThread.java index 843ae0d..4cad6f8 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/job/SqlJobThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/job/SqlJobThread.java @@ -1,10 +1,12 @@ package com.gbase8c.dmt.migration.job; import com.gbase8c.dmt.common.util.Exceptions; +import com.gbase8c.dmt.migration.MoLogger; import com.gbase8c.dmt.model.enums.MigrationObjectType; import com.gbase8c.dmt.model.migration.dto.MigrationObject; import com.gbase8c.dmt.model.migration.record.JobThreadRecord; import com.gbase8c.dmt.model.enums.Status; +import com.gbase8c.dmt.model.migration.record.Mo; import lombok.extern.slf4j.Slf4j; import org.apache.commons.dbutils.QueryRunner; import org.apache.commons.lang3.time.DateFormatUtils; @@ -16,8 +18,8 @@ import java.util.List; @Slf4j public class SqlJobThread extends AbstractJobThread{ - public SqlJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource) { - super(type, dbObject, tarDataSource); + public SqlJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource, MoLogger moLogger) { + super(type, dbObject, tarDataSource, moLogger); } /** @@ -28,6 +30,14 @@ public class SqlJobThread extends AbstractJobThread{ */ @Override public JobThreadRecord call() throws Exception { + Mo mo = Mo.builder() +// .type(type.getName()) + .name((MigrationObjectType.TABLESPACE.equals(type) + || MigrationObjectType.SCHEMA.equals(type)) + ? dbObject.getName() : dbObject.getSchema() + "." + dbObject.getName()) + .status(Status.InProgress.name()) + .build(); + moLogger.log(mo); long beginTime = System.currentTimeMillis(); JobThreadRecord jobThreadRecord = new JobThreadRecord(); List contents = dbObject.getContents(); @@ -36,7 +46,7 @@ public class SqlJobThread extends AbstractJobThread{ try { //todo:添加convert转换成功与否判断 if (dbObject.getConvertible()){ - for (String sql:contents){ + for (String sql : contents){ int execute = queryRunner.execute(sql); long endTime = System.currentTimeMillis(); @@ -48,34 +58,41 @@ public class SqlJobThread extends AbstractJobThread{ "\n end time : {}" , dbObject.getMigrationObjectType() ,sql.replaceAll(System.getProperty("line.separator"),"") - ,execute == 0?"success":"failed" + ,execute == 0 ? "success" : "failed" ,(endTime - beginTime) ,DateFormatUtils.format(beginTime,"yyyy-MM-dd HH:mm:ss.SSS") ,DateFormatUtils.format(endTime,"yyyy-MM-dd HH:mm:ss.SSS")); } jobThreadRecord.setFlag(true); dbObject.setStatus(Status.Finished); - }else{ + mo.setStatus(Status.Finished.name()); + } else { dbObject.setMsg(dbObject.getConvertMsg()); jobThreadRecord.setFlag(false); dbObject.setStatus(Status.Failure); jobThreadRecord.setErrorLog(dbObject.getConvertMsg()); + mo.setStatus(Status.Failure.name()); + mo.setMsg(dbObject.getConvertMsg()); } - }catch (SQLException exception) { + } catch (SQLException exception) { String sqlState = exception.getSQLState(); // 42P06 OpenGauss数据库标准错误码：模式重复，由于设计原因，重复的模式不显示报错。 if ("42P06".equals(sqlState)){ jobThreadRecord.setFlag(true); dbObject.setStatus(Status.Finished); - }else { + mo.setStatus(Status.Finished.name()); + } else { dbObject.setMsg(exception.getMessage()); dbObject.setStatus(Status.Failure); jobThreadRecord.setFlag(false); jobThreadRecord.setErrorLog(Exceptions.getStackTraceAsString(exception)); + mo.setStatus(Status.Failure.name()); + mo.setMsg(exception.getMessage()); } - log.error("execute sql job error error,cause", exception.getMessage(),exception); + log.error("execute sql job error error,cause: {}", exception.getMessage(),exception); } jobThreadRecord.setDbObject(dbObject); + moLogger.log(mo); return jobThreadRecord; } } diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/job/SynonymJobThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/job/SynonymJobThread.java index b203c3d..7a8615b 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/job/SynonymJobThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/job/SynonymJobThread.java @@ -1,5 +1,6 @@ package com.gbase8c.dmt.migration.job; +import com.gbase8c.dmt.migration.MoLogger; import com.gbase8c.dmt.model.enums.MigrationObjectType; import com.gbase8c.dmt.model.migration.dto.MigrationObject; import com.gbase8c.dmt.model.migration.record.JobThreadRecord; @@ -8,8 +9,8 @@ import javax.sql.DataSource; public class SynonymJobThread extends AbstractJobThread{ - public SynonymJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource) { - super(type, dbObject, tarDataSource); + public SynonymJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource, MoLogger moLogger) { + super(type, dbObject, tarDataSource, moLogger); } /** diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/job/TableJobThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/job/TableJobThread.java index 83abeab..c53d47e 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/job/TableJobThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/job/TableJobThread.java @@ -1,6 +1,7 @@ package com.gbase8c.dmt.migration.job; import com.gbase8c.dmt.common.util.Exceptions; +import com.gbase8c.dmt.migration.MoLogger; import com.gbase8c.dmt.model.enums.MigrationObjectType; import com.gbase8c.dmt.model.migration.dto.MigrationObject; import com.gbase8c.dmt.model.migration.record.JobThreadRecord; @@ -15,8 +16,8 @@ import java.util.List; public class TableJobThread extends AbstractJobThread { - public TableJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource) { - super(type, dbObject, tarDataSource); + public TableJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource, MoLogger moLogger) { + super(type, dbObject, tarDataSource, moLogger); } /** diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/job/TableSpaceJobThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/job/TableSpaceJobThread.java index f8ec29b..99e90d0 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/job/TableSpaceJobThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/job/TableSpaceJobThread.java @@ -1,5 +1,6 @@ package com.gbase8c.dmt.migration.job; +import com.gbase8c.dmt.migration.MoLogger; import com.gbase8c.dmt.model.enums.MigrationObjectType; import com.gbase8c.dmt.model.migration.dto.MigrationObject; import com.gbase8c.dmt.model.migration.record.JobThreadRecord; @@ -14,8 +15,8 @@ import java.util.List; @Slf4j public class TableSpaceJobThread extends AbstractJobThread{ - public TableSpaceJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource) { - super(type, dbObject, tarDataSource); + public TableSpaceJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource, MoLogger moLogger) { + super(type, dbObject, tarDataSource, moLogger); } /** diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/job/ViewJobThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/job/ViewJobThread.java index 7d5db09..a676a24 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/job/ViewJobThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/job/ViewJobThread.java @@ -1,5 +1,6 @@ package com.gbase8c.dmt.migration.job; +import com.gbase8c.dmt.migration.MoLogger; import com.gbase8c.dmt.model.enums.MigrationObjectType; import com.gbase8c.dmt.model.migration.dto.MigrationObject; import com.gbase8c.dmt.model.migration.record.JobThreadRecord; @@ -9,8 +10,8 @@ import javax.sql.DataSource; public class ViewJobThread extends AbstractJobThread{ - public ViewJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource) { - super(type, dbObject, tarDataSource); + public ViewJobThread(MigrationObjectType type, MigrationObject dbObject, DataSource tarDataSource, MoLogger moLogger) { + super(type, dbObject, tarDataSource, moLogger); } /** -- Gitee From 5b0a0230c8475f2d230024ea6afea376b565105f Mon Sep 17 00:00:00 2001 From: zephyr Date: Fri, 15 Sep 2023 11:36:22 +0800 Subject: [PATCH 07/15] =?UTF-8?q?=E6=9B=B4=E6=94=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E5=AF=B9=E8=B1=A1=E6=89=A7=E8=A1=8C=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datax/common/util/ConfigurationUtil.java | 84 ------------------- .../core/taskgroup/TaskGroupContainer.java | 9 +- .../com/gbase8c/dmt/migration/MoLogger.java | 22 +++-- 3 files changed, 16 insertions(+), 99 deletions(-) diff --git a/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java b/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java index 02df8da..9f93704 100644 --- a/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java +++ b/common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java @@ -1,58 +1,14 @@ package com.alibaba.datax.common.util; -import com.alibaba.fastjson.JSON; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; public class ConfigurationUtil { - public static void jobState(Configuration configuration, String state, String msg) throws IOException { - String resume = System.getProperty("datax.persistence"); - if ("true".equalsIgnoreCase(resume)) { - String jobStatePath = "job.state"; - configuration.set(jobStatePath, state); - String json = configuration.toJSON(); - FileUtils.writeStringToFile(file(configuration), json, StandardCharsets.UTF_8, false); - -// String jobName = configuration.getString("job.name"); -// String workdir = configuration.getString("job.workdir"); -// String id = configuration.getString("job._id"); -// String filePath = StringUtils.joinWith(File.separator, workdir, id, "status", "data.json"); -// -// List mos = JSON.parseArray(FileUtils.readFileToString(new File(filePath), StandardCharsets.UTF_8), Mo.class); -// Map map = new HashMap<>(); -// for (Mo mo : mos) { -// map.put(mo.getName(), mo); -// } -// -// if (map.containsKey(jobName)) { -// Mo mo = map.get(jobName); -// //UnStart, InProgress, Finished, Failure; -// // RUNNING, FAILED, SUCCEEDED -// if ("RUNNING".equalsIgnoreCase(state)) { -// mo.setStatus("InProgress"); -// } -// if ("FAILED".equalsIgnoreCase(state)) { -// mo.setStatus("Failure"); -// mo.setMsg(msg); -// }if ("SUCCEEDED".equalsIgnoreCase(state)) { -// mo.setStatus("Finished"); -// } -// String jsonStr = JSON.toJSONString(mos); -// FileUtils.writeStringToFile(new File(filePath), jsonStr, StandardCharsets.UTF_8, false); -// } - } - } - public static void persistence(Configuration configuration) throws IOException { String resume = System.getProperty("datax.persistence"); if ("true".equalsIgnoreCase(resume)) { @@ -78,44 +34,4 @@ public class ConfigurationUtil { return new File(StringUtils.joinWith(File.separator, workdir, id, "configuration", name + ".json")); } - public static class Mo { - private String type; - private String name; - private String status; - private String msg; - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } - - public String getMsg() { - return msg; - } - - public void setMsg(String msg) { - this.msg = msg; - } - - } - } diff --git a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java index 9adb58b..32bb1a3 100644 --- a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java +++ b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java @@ -103,7 +103,8 @@ public class TaskGroupContainer extends AbstractContainer { try { // todo 在这里存文件 String jobStatePath = "job.state"; - ConfigurationUtil.jobState(configuration, State.RUNNING.name(), null); + configuration.set(jobStatePath, State.RUNNING); + ConfigurationUtil.persistence(configuration); /** * 状态check时间间隔，较短，可以把任务及时分发到对应channel中 */ @@ -220,7 +221,8 @@ public class TaskGroupContainer extends AbstractContainer { lastTaskGroupContainerCommunication = reportTaskGroupCommunication( lastTaskGroupContainerCommunication, taskCountInThisTaskGroup); // todo - ConfigurationUtil.jobState(configuration, State.FAILED.name(), lastTaskGroupContainerCommunication.getThrowable().getMessage()); + configuration.set(jobStatePath, State.FAILED); + ConfigurationUtil.persistence(configuration); throw DataXException.asDataXException( FrameworkErrorCode.PLUGIN_RUNTIME_ERROR, lastTaskGroupContainerCommunication.getThrowable()); @@ -283,7 +285,8 @@ public class TaskGroupContainer extends AbstractContainer { LOG.info("taskGroup[{}] completed it's tasks.", this.taskGroupId); // todo 任务执行完成 - ConfigurationUtil.jobState(configuration, State.SUCCEEDED.name(), null); + configuration.set(jobStatePath, State.SUCCEEDED); + ConfigurationUtil.persistence(configuration); ConfigurationUtil.remove(configuration); break; } diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java index 5577de4..fb2180a 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java @@ -14,18 +14,20 @@ import java.nio.charset.StandardCharsets; import java.util.List; import java.util.concurrent.ConcurrentHashMap; +@Slf4j public class MoLogger { private final ConcurrentHashMap map = new ConcurrentHashMap<>(); + private final File file; private final Logger logger; private Thread thread; public MoLogger(String workDir, String taskId, String type) { - File file = file(workDir, taskId, type); + file = file(workDir, taskId, type); logger = new Logger(file, map); } public MoLogger(String workDir, String taskId, String type, long wait) { - File file = file(workDir, taskId, type); + file = file(workDir, taskId, type); logger = new Logger(file, map, wait); } @@ -41,9 +43,12 @@ public class MoLogger { public void stop() { try { - thread.join(); logger.setStop(true); - } catch (InterruptedException e) { + thread.join(); + log.info("停止数据库对象状态更新: " + file.getPath()); + String json = JSON.toJSONString(map.values()); + FileUtils.writeStringToFile(file, json, StandardCharsets.UTF_8); + } catch (Exception e) { throw new RuntimeException(e); } } @@ -61,7 +66,7 @@ public class MoLogger { private long wait = 10000L; private File file; - private boolean stop = false; + private volatile boolean stop = false; public Logger(File file, ConcurrentHashMap map) { this.file = file; @@ -91,13 +96,6 @@ public class MoLogger { } } } - log.info("停止数据库对象状态更新: " + file.getPath()); - String json = JSON.toJSONString(map.values()); - try { - FileUtils.writeStringToFile(file, json, StandardCharsets.UTF_8); - } catch (IOException e) { - throw new RuntimeException(e); - } } private void initMap() { -- Gitee From a5a27a11f800b23f3571543a49a8d10e9ccb06e0 Mon Sep 17 00:00:00 2001 From: zephyr Date: Fri, 15 Sep 2023 12:04:46 +0800 Subject: [PATCH 08/15] =?UTF-8?q?=E6=9B=B4=E6=94=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E5=AF=B9=E8=B1=A1=E6=89=A7=E8=A1=8C=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/gbase8c/dmt/migration/MoLogger.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java index fb2180a..45824b4 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java @@ -45,9 +45,6 @@ public class MoLogger { try { logger.setStop(true); thread.join(); - log.info("停止数据库对象状态更新: " + file.getPath()); - String json = JSON.toJSONString(map.values()); - FileUtils.writeStringToFile(file, json, StandardCharsets.UTF_8); } catch (Exception e) { throw new RuntimeException(e); } @@ -96,6 +93,17 @@ public class MoLogger { } } } + if (stop) { + log.info("停止数据库对象状态更新: " + file.getPath()); + String newJson = JSON.toJSONString(map.values()); + if (!newJson.equalsIgnoreCase(lastJson)) { + try { + FileUtils.writeStringToFile(file, newJson, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } } private void initMap() { -- Gitee From 2cb425994164ccfe7f87bd743fa9a7d2867e2a99 Mon Sep 17 00:00:00 2001 From: zephyr Date: Fri, 15 Sep 2023 12:05:30 +0800 Subject: [PATCH 09/15] =?UTF-8?q?=E6=9B=B4=E6=94=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E5=AF=B9=E8=B1=A1=E6=89=A7=E8=A1=8C=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java index 45824b4..3757e49 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java @@ -17,17 +17,16 @@ import java.util.concurrent.ConcurrentHashMap; @Slf4j public class MoLogger { private final ConcurrentHashMap map = new ConcurrentHashMap<>(); - private final File file; private final Logger logger; private Thread thread; public MoLogger(String workDir, String taskId, String type) { - file = file(workDir, taskId, type); + File file = file(workDir, taskId, type); logger = new Logger(file, map); } public MoLogger(String workDir, String taskId, String type, long wait) { - file = file(workDir, taskId, type); + File file = file(workDir, taskId, type); logger = new Logger(file, map, wait); } -- Gitee From 8fcba663f05f6cd647755ce3c48f3a835f025022 Mon Sep 17 00:00:00 2001 From: zephyr Date: Fri, 15 Sep 2023 14:05:55 +0800 Subject: [PATCH 10/15] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/gbase8c/dmt/common/util/MoUtils.java | 16 ------- .../com/gbase8c/dmt/common/util/SQLUtil.java | 8 ---- .../migration/MigrationExecutionService.java | 34 +++++++-------- .../dmt/migration/MigrationObjectService.java | 25 +++++------ .../dmt/migration/MigrationThread.java | 42 ++++++------------- .../dmt/migration/job/AbstractJobThread.java | 10 +---- .../migration/job/ConstraintJobThread.java | 6 --- .../dmt/migration/job/DataXJobThread.java | 7 ---- .../dmt/migration/job/FunctionJobThread.java | 6 --- .../dmt/migration/job/IndexJobThread.java | 6 --- .../dmt/migration/job/SchemaJobThread.java | 6 --- .../dmt/migration/job/SqlJobThread.java | 7 ---- .../dmt/migration/job/SynonymJobThread.java | 6 --- .../dmt/migration/job/TableJobThread.java | 7 ---- .../migration/job/TableSpaceJobThread.java | 6 --- .../dmt/migration/job/ViewJobThread.java | 7 ---- .../model/migration/record/ExecuteRecord.java | 2 +- .../com/gbase8c/dmt/service/TaskService.java | 1 - 18 files changed, 41 insertions(+), 161 deletions(-) delete mode 100644 dmt/src/main/java/com/gbase8c/dmt/common/util/MoUtils.java delete mode 100644 dmt/src/main/java/com/gbase8c/dmt/common/util/SQLUtil.java diff --git a/dmt/src/main/java/com/gbase8c/dmt/common/util/MoUtils.java b/dmt/src/main/java/com/gbase8c/dmt/common/util/MoUtils.java deleted file mode 100644 index 07a25e2..0000000 --- a/dmt/src/main/java/com/gbase8c/dmt/common/util/MoUtils.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.gbase8c.dmt.common.util; - -import com.gbase8c.dmt.config.DmtConfig; -import com.gbase8c.dmt.model.enums.MigrationObjectType; -import com.gbase8c.dmt.model.migration.record.Mo; -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.util.List; - -public class MoUtils { - public static void persistent(DmtConfig dmtConfig, MigrationObjectType mot, List mos) { - String dbObjectFile = StringUtils.joinWith(File.separator, dmtConfig.getWorkDir(), dmtConfig.getTaskId(), "status", mot.getName() + ".json"); - JsonMapper.nonEmptyMapper().toFile(new File(dbObjectFile), mos); - } -} diff --git a/dmt/src/main/java/com/gbase8c/dmt/common/util/SQLUtil.java b/dmt/src/main/java/com/gbase8c/dmt/common/util/SQLUtil.java deleted file mode 100644 index 40cf345..0000000 --- a/dmt/src/main/java/com/gbase8c/dmt/common/util/SQLUtil.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.gbase8c.dmt.common.util; - -public class SQLUtil { - public static String MYSQL_QUOT = "`"; - public static String ORACLE_QUOT = "\""; - public static String PG_QUOT = ""; - public static String SQLSERVER_QUOT = ""; -} diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationExecutionService.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationExecutionService.java index cacaf6b..67e32dd 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationExecutionService.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationExecutionService.java @@ -65,22 +65,22 @@ public class MigrationExecutionService { DataSourceDto dataSourceDto = dataSourceDao.get(tar); DataSource dataSource = DataSourceFactory.getDataSource(dataSourceDto); Map> orderedDbObjects = migrationObjectService.getMigrationObjects(dboDto); - if(migrationThread ==null){ + if (migrationThread ==null) { migrationThread = MigrationThread.build(dboDto, orderedDbObjects, dataSource,recordDao, taskDao); migrationThread.init(dmtConfig); _threadTable.put(taskId, migrationThread); log.info("create [ReplicationThread] mission : "+taskId); - }else if(migrationThread.isAlive()==false){ + } else if(!migrationThread.isAlive()) { migrationThread = MigrationThread.build(dboDto, orderedDbObjects, dataSource,recordDao, taskDao); migrationThread.init(dmtConfig); _threadTable.put(taskId, migrationThread); log.info("rebuild [ReplicationThread] mission : "+taskId); } - if(migrationThread.getMigrationTaskStatus().equals(MigrationTaskStatus.RUNNING) - && migrationThread.isAlive()==true - && migrationThread.isStop()==false) { + if (migrationThread.getMigrationTaskStatus().equals(MigrationTaskStatus.RUNNING) + && migrationThread.isAlive() + && !migrationThread.isStop()) { log.info("mission is already running ... "+taskId); - }else{ + } else { migrationThread.start(); } } @@ -95,22 +95,22 @@ public class MigrationExecutionService { DataSourceDto dataSourceDto = dataSourceDao.get(tar); DataSource dataSource = DataSourceFactory.getDataSource(dataSourceDto); Map> orderedDbObjects = migrationObjectService.getMigrationObjects(dboDto); - if(migrationThread ==null){ + if (migrationThread ==null) { migrationThread = MigrationThread.build(dboDto, orderedDbObjects, dataSource,recordDao, taskDao); migrationThread.init(dmtConfig); _threadTable.put(taskId, migrationThread); log.info("create [ReplicationThread] mission : "+taskId); - }else if(migrationThread.isAlive()==false){ + } else if(!migrationThread.isAlive()) { migrationThread = MigrationThread.build(dboDto, orderedDbObjects, dataSource,recordDao, taskDao); migrationThread.init(dmtConfig); _threadTable.put(taskId, migrationThread); log.info("rebuild [ReplicationThread] mission : "+taskId); } - if(migrationThread.getMigrationTaskStatus().equals(MigrationTaskStatus.RUNNING) - && migrationThread.isAlive()==true - && migrationThread.isStop()==false) { + if (migrationThread.getMigrationTaskStatus().equals(MigrationTaskStatus.RUNNING) + && migrationThread.isAlive() + && !migrationThread.isStop()) { log.info("mission is already running ... "+taskId); - }else{ + } else { migrationThread.start(); } } @@ -146,14 +146,12 @@ public class MigrationExecutionService { List records = new ArrayList<>(); if (MapUtils.isEmpty(recordDto.getSchemaRecordsMap())){ migrationTaskStatus = MigrationTaskStatus.UNSTART; - }else { - recordDto.getSchemaRecordsMap().values().forEach(e ->{ - records.addAll(e); - }); + } else { + recordDto.getSchemaRecordsMap().values().forEach(records::addAll); int sum = records.stream().mapToInt(ExecuteRecord::getFailed).sum(); if (sum > 0){ migrationTaskStatus = MigrationTaskStatus.FAILED; - }else { + } else { migrationTaskStatus = MigrationTaskStatus.SUCCESS; } } @@ -168,7 +166,7 @@ public class MigrationExecutionService { if (!CollectionUtils.isEmpty(list)){ recordDto = list.stream().max(Comparator.comparing(MigrationRecordDto::getEndTime)).get(); } - }else { + } else { recordDto = migrationThread.getRunningRecord(); } return recordDto; diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java index 7731a52..42db52f 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationObjectService.java @@ -3,7 +3,6 @@ package com.gbase8c.dmt.migration; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.gbase8c.dmt.common.util.JsonMapper; -import com.gbase8c.dmt.common.util.MoUtils; import com.gbase8c.dmt.config.DmtConfig; import com.gbase8c.dmt.dao.DataSourceDao; import com.gbase8c.dmt.dao.SnapshotDao; @@ -26,10 +25,10 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.dbutils.handlers.ColumnListHandler; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import java.io.File; import java.sql.Types; import java.util.HashMap; import java.util.List; @@ -128,12 +127,17 @@ public class MigrationObjectService { .status(Status.UnStart.name()) .build()) .collect(Collectors.toList()); - MoUtils.persistent(dmtConfig, migrationObjectType, mos); + persistent(dmtConfig, migrationObjectType, mos); } return orderedDbObjects; } + public static void persistent(DmtConfig dmtConfig, MigrationObjectType mot, List mos) { + String dbObjectFile = StringUtils.joinWith(File.separator, dmtConfig.getWorkDir(), dmtConfig.getTaskId(), "status", mot.getName() + ".json"); + JsonMapper.nonEmptyMapper().toFile(new File(dbObjectFile), mos); + } + public static long toLong(String str) { long h = 0; if (str != null) { @@ -283,11 +287,7 @@ public class MigrationObjectService { String srcName; boolean preserveCase = true; - if (src.getDbVersion().equals(DbType.DbVersion.MySQL_8)) { - srcName = src.getDbType().name() + "8"; - } else { - srcName = src.getDbType().name(); - } + srcName = src.getDbType().name(); String readerName = srcName.toLowerCase() + "reader"; reader.put("name", readerName); @@ -310,7 +310,7 @@ public class MigrationObjectService { .collect(Collectors.toList()); paraMap.put("column", columnNames); -// // todo just for test +// // todo // paraMap.put("scn", 121118198); // paraMap.put("samplePercentage", 50); paraMap.put("pageSize", 100000); @@ -375,12 +375,7 @@ public class MigrationObjectService { Metadata metadata = MetadataFactory.getMetaData(tar); Map writer = new JSONObject(); String srcName; - //todo -// if (tar.getDbVersion().equals(DbType.DbVersion.Oracle_8i)) { -// srcName = tar.getDbType().name() + "8i"; -// } else { -// srcName = tar.getDbType().name(); -// } + srcName = tar.getDbType().name(); String writerName = srcName.toLowerCase() + "writer"; writer.put("name", writerName); diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java index a0b48c5..8fca4ea 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java @@ -1,6 +1,5 @@ package com.gbase8c.dmt.migration; -import com.gbase8c.dmt.common.util.MoUtils; import com.gbase8c.dmt.config.DmtConfig; import com.gbase8c.dmt.dao.RecordDao; import com.gbase8c.dmt.dao.TaskDao; @@ -17,7 +16,6 @@ import com.gbase8c.dmt.model.migration.record.JobThreadRecord; import com.gbase8c.dmt.model.migration.record.MigrationRecordDto; import com.gbase8c.dmt.model.enums.MigrationTaskStatus; import com.gbase8c.dmt.model.enums.Status; -import com.gbase8c.dmt.model.migration.record.Mo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import lombok.Getter; @@ -29,7 +27,6 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import javax.sql.DataSource; import java.io.Closeable; -import java.io.IOException; import java.util.*; import java.util.concurrent.*; import java.util.stream.Collectors; @@ -51,7 +48,7 @@ public class MigrationThread extends Thread implements Closeable { private DataSource tarDataSource; protected volatile boolean suspend = false; protected volatile boolean stop = false; - private final BlockingQueue> queue = new LinkedBlockingQueue>(); + private final BlockingQueue> queue = new LinkedBlockingQueue<>(); private CompletionService completionService = null; private Map> migrationObjectListMap; private Map> jobListMap = new LinkedHashMap<>(); @@ -109,7 +106,7 @@ public class MigrationThread extends Thread implements Closeable { MoLogger moLogger = new MoLogger(dmtConfig.getWorkDir(), dmtConfig.getTaskId(), type.getName()); moLoggerMap.put(type, moLogger); for (MigrationObject obj : entry.getValue()) { - AbstractJobThread jobThread = null; + AbstractJobThread jobThread; if (type == MigrationObjectType.DATA) { jobThread = new DataXJobThread(type, obj, tarDataSource, moLogger); } else { @@ -216,7 +213,7 @@ public class MigrationThread extends Thread implements Closeable { JobThreadRecord jobThreadRecord = future.get(); jobThreadRecords.add(jobThreadRecord); } catch (InterruptedException | ExecutionException e) { - e.printStackTrace(); + log.error("futureTake exception", e); } } List executeRecords = new ArrayList<>(); @@ -226,45 +223,32 @@ public class MigrationThread extends Thread implements Closeable { .filter(m -> MigrationObjectType.TABLE.equals(m.getMigrationObjectType())) .map(MigrationObject::getTableMapper) .collect(Collectors.toList()); -// if (!type.equals(MigrationObjectType.DATA)) { -// List mos = jobThreadRecords.stream() -// .map(record -> Mo.builder() -// .type(type.getName()) -// .name((MigrationObjectType.TABLESPACE.equals(type) -// || MigrationObjectType.SCHEMA.equals(type)) -// ? record.getDbObject().getName() : record.getDbObject().getSchema() + "." + record.getDbObject().getName()) -// .status(record.getFlag() ? Status.Finished.name() : Status.Failure.name()) -// .msg(record.getFlag() ? null : record.getErrorLog()) -// .build()) -// .collect(Collectors.toList()); -// MoUtils.persistent(dmtConfig, type, mos); -// } this.tableMappers.addAll(tableMappers); Map> jobListMap = jobThreadRecords .stream() .collect(Collectors.groupingBy(JobThreadRecord::getSchemaName)); //任务结果转换 - jobListMap.entrySet().stream().forEach(entry -> { - int success = entry.getValue().stream().mapToInt(jobRecord -> jobRecord.getFlag() ? 1 : 0).sum(); - int failed = entry.getValue().stream().mapToInt(jobRecord -> jobRecord.getFlag() ? 0 : 1).sum(); + jobListMap.forEach((key, value) -> { + int success = value.stream().mapToInt(jobRecord -> jobRecord.getFlag() ? 1 : 0).sum(); + int failed = value.stream().mapToInt(jobRecord -> jobRecord.getFlag() ? 0 : 1).sum(); List errorObjs = Lists.newArrayList(); List successObjs = Lists.newArrayList(); List errorLogList = Lists.newArrayList(); - successObjs = entry.getValue().stream() + successObjs = value.stream() .map(JobThreadRecord::getDbObject) .filter(errorObj -> Status.Finished == errorObj.getStatus()) .collect(Collectors.toList()); if (failed != 0) { - errorObjs = entry.getValue().stream() + errorObjs = value.stream() .map(JobThreadRecord::getDbObject) .filter(errorObj -> Status.Failure == errorObj.getStatus()) .collect(Collectors.toList()); - errorLogList = entry.getValue().stream().map(JobThreadRecord::getErrorLog).filter(obj -> null != obj).collect(Collectors.toList()); + errorLogList = value.stream().map(JobThreadRecord::getErrorLog).filter(Objects::nonNull).collect(Collectors.toList()); } - ExecuteRecord record = ExecuteRecord.builder() - .schemaName(entry.getKey()) + ExecuteRecord record = ExecuteRecord.builder() + .schemaName(key) .type(type) - .total(entry.getValue().size()) + .total(value.size()) .succeed(success) .errorObjs(errorObjs) .successObjs(successObjs) @@ -277,7 +261,7 @@ public class MigrationThread extends Thread implements Closeable { } @Override - public void close() throws IOException { + public void close() { this.stop = true; this.getRecordDao().save(recordDto); } diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/job/AbstractJobThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/job/AbstractJobThread.java index 1f6920e..c71e095 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/job/AbstractJobThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/job/AbstractJobThread.java @@ -13,20 +13,12 @@ import java.util.concurrent.Callable; @AllArgsConstructor @Slf4j public abstract class AbstractJobThread implements Callable { - protected MigrationObjectType type; + protected MigrationObjectType type; protected MigrationObject dbObject; - protected DataSource tarDataSource; - protected MoLogger moLogger; - /** - * Computes a result, or throws an exception if unable to do so. - * - * @return computed result - * @throws Exception if unable to compute a result - */ @Override public JobThreadRecord call() throws Exception { log.info(this.type.getType()); diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/job/ConstraintJobThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/job/ConstraintJobThread.java index 50aea00..3d6c312 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/job/ConstraintJobThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/job/ConstraintJobThread.java @@ -14,12 +14,6 @@ public class ConstraintJobThread extends AbstractJobThread implements Callable { +public class ExecuteRecord { // 任务类型 private MigrationObjectType type; diff --git a/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java b/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java index bd6c8b6..a4aa142 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java +++ b/dmt/src/main/java/com/gbase8c/dmt/service/TaskService.java @@ -546,7 +546,6 @@ public class TaskService { dboDto.setId(task.getId()); } - private List tableNames(Task task, SchemaConfig.SchemaMapper schemaMapper) { List tableNames = null; TableConfig tableConfig = schemaMapper.getTableConfig(); -- Gitee From 9e3ee3510fe61b2fab559942bc34f3a351a3b90b Mon Sep 17 00:00:00 2001 From: zephyr Date: Fri, 15 Sep 2023 14:10:15 +0800 Subject: [PATCH 11/15] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dmt/src/main/java/com/gbase8c/dmt/db/meta/AbstractMeta.java | 1 - .../main/java/com/gbase8c/dmt/model/migration/job/DataxJob.java | 2 -- .../main/java/com/gbase8c/dmt/model/migration/record/Mo.java | 1 + .../com/gbase8c/dmt/model/migration/report/SchemaStats.java | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/dmt/src/main/java/com/gbase8c/dmt/db/meta/AbstractMeta.java b/dmt/src/main/java/com/gbase8c/dmt/db/meta/AbstractMeta.java index 2ece39a..58eafdd 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/db/meta/AbstractMeta.java +++ b/dmt/src/main/java/com/gbase8c/dmt/db/meta/AbstractMeta.java @@ -75,7 +75,6 @@ public abstract class AbstractMeta implements Meta { sql = String.format(sql, tuple2.get0()); return qr.query(sql, rsh, tuple2.get1()); } catch (Exception e) { - e.printStackTrace(); log.error(String.format("query sql[%s] error!", sql), e); throw MetadataException.asMetadataException(String.format("query sql[%s] error!", sql), e); } diff --git a/dmt/src/main/java/com/gbase8c/dmt/model/migration/job/DataxJob.java b/dmt/src/main/java/com/gbase8c/dmt/model/migration/job/DataxJob.java index 3a7e029..ac6bb4e 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/model/migration/job/DataxJob.java +++ b/dmt/src/main/java/com/gbase8c/dmt/model/migration/job/DataxJob.java @@ -2,7 +2,6 @@ package com.gbase8c.dmt.model.migration.job; import com.gbase8c.dmt.model.migration.dto.DataSourceDto; import com.gbase8c.dmt.model.migration.dto.MigrationObject; -import com.gbase8c.dmt.model.migration.dto.TableDto; import com.google.common.collect.Lists; import lombok.*; import lombok.experimental.SuperBuilder; @@ -20,7 +19,6 @@ public class DataxJob extends MigrationObject { private Long jobId; private DataSourceDto src; private DataSourceDto tar; -// private TableDto tableDto; private String json; @Override diff --git a/dmt/src/main/java/com/gbase8c/dmt/model/migration/record/Mo.java b/dmt/src/main/java/com/gbase8c/dmt/model/migration/record/Mo.java index 1d3321a..6cc1719 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/model/migration/record/Mo.java +++ b/dmt/src/main/java/com/gbase8c/dmt/model/migration/record/Mo.java @@ -8,6 +8,7 @@ import lombok.*; @NoArgsConstructor @AllArgsConstructor public class Mo { + private String type; private String name; private String status; diff --git a/dmt/src/main/java/com/gbase8c/dmt/model/migration/report/SchemaStats.java b/dmt/src/main/java/com/gbase8c/dmt/model/migration/report/SchemaStats.java index 36083e7..081f215 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/model/migration/report/SchemaStats.java +++ b/dmt/src/main/java/com/gbase8c/dmt/model/migration/report/SchemaStats.java @@ -1,6 +1,5 @@ package com.gbase8c.dmt.model.migration.report; - import com.gbase8c.dmt.model.migration.dto.TransObjDto; import lombok.*; -- Gitee From 6f4ae7fac6f63ab23326c12c8063c1959784f713 Mon Sep 17 00:00:00 2001 From: zephyr Date: Fri, 15 Sep 2023 16:03:32 +0800 Subject: [PATCH 12/15] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gbase8c/dmt/migration/MigrationThread.java | 15 ++++++++------- .../java/com/gbase8c/dmt/migration/MoLogger.java | 5 ++++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java index 8fca4ea..e94fa83 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MigrationThread.java @@ -157,13 +157,14 @@ public class MigrationThread extends Thread implements Closeable { //阻塞获取值 futureTake(jobList.size(), type); long end = System.currentTimeMillis(); - log.info("\n********************************************************************************************************" + - "\n* type : {}" + - "\n* job size : {} " + - "\n* start time : {} " + - "\n* end time : {} " + - "\n* spend total time : {}ms " + - "\n********************************************************************************************************" + log.info( + "\n********************************************************************************************************" + + "\n* type : {}" + + "\n* job size : {} " + + "\n* start time : {} " + + "\n* end time : {} " + + "\n* spend total time : {}ms " + + "\n********************************************************************************************************" , type , jobList.size() , DateFormatUtils.format(begin, "yyyy-MM-dd HH:mm:ss.SSS") diff --git a/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java b/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java index 3757e49..8a257b9 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java +++ b/dmt/src/main/java/com/gbase8c/dmt/migration/MoLogger.java @@ -13,6 +13,8 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; @Slf4j public class MoLogger { @@ -78,6 +80,7 @@ public class MoLogger { @Override public void run() { log.info("开始数据库对象状态更新: " + file.getPath()); + CountDownLatch latch = new CountDownLatch(1); initMap(); String lastJson = JSON.toJSONString(map.values()); while (!stop) { @@ -86,7 +89,7 @@ public class MoLogger { lastJson = newJson; try { FileUtils.writeStringToFile(file, newJson, StandardCharsets.UTF_8); - Thread.sleep(wait); + latch.await(wait, TimeUnit.MILLISECONDS); } catch (Exception e) { throw new RuntimeException(e); } -- Gitee From 538104eaf210cb0887d2eae2a347181698ff120f Mon Sep 17 00:00:00 2001 From: zephyr Date: Fri, 13 Oct 2023 13:55:11 +0800 Subject: [PATCH 13/15] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- userGuid.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/userGuid.md b/userGuid.md index bafb4e3..50895e6 100644 --- a/userGuid.md +++ b/userGuid.md @@ -71,10 +71,10 @@ ``` shell windows： - java -server -Xms1g -Xmx1g -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${ora-migration-tool.home}/log -Dloglevel=info -Dfile.encoding=UTF-8 -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener -Djava.security.egd=file:///dev/urandom -Ddatax.home=${ora-migration-tool.home} -Ddatax.persistence=true -Dlogback.configurationFile="${ora-migration-tool.home}/conf/logback.xml" -classpath "${ora-migration-tool.home}/lib/*” com.gbase8c.dmt.Dmt -taskid ${tasked} -workdir ${workdir} [-action rbt] [-tables schema.tn1,schema.tn2...] + java -server -Xms1g -Xmx1g -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${ora-migration-tool.home}/log -Dloglevel=info -Dfile.encoding=UTF-8 -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener -Djava.security.egd=file:///dev/urandom -Ddatax.home=${ora-migration-tool.home} -Ddatax.persistence=true -Dlogback.configurationFile="${ora-migration-tool.home}/conf/logback.xml" -classpath "${ora-migration-tool.home}/lib/*" com.gbase8c.dmt.Dmt -taskid ${tasked} -workdir ${workdir} [-action rbt] [-tables schema.tn1,schema.tn2...] 其它（mac/linux）： - java -server -Xms1g -Xmx1g -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${ora-migration-tool.home}/log -Dloglevel=info -Dfile.encoding=UTF-8 -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener -Djava.security.egd=file:///dev/urandom -Ddatax.home=${ora-migration-tool.home} -Ddatax.persistence=true -Dlogback.configurationFile="${ora-migration-tool.home}/conf/logback.xml" -classpath "${ora-migration-tool.home}/lib/*:.” com.gbase8c.dmt.Dmt -taskid ${tasked} -workdir ${workdir} [-action rbt] [-tables schema.tn1,schema.tn2...] + java -server -Xms1g -Xmx1g -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${ora-migration-tool.home}/log -Dloglevel=info -Dfile.encoding=UTF-8 -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener -Djava.security.egd=file:///dev/urandom -Ddatax.home=${ora-migration-tool.home} -Ddatax.persistence=true -Dlogback.configurationFile="${ora-migration-tool.home}/conf/logback.xml" -classpath "${ora-migration-tool.home}/lib/*:." com.gbase8c.dmt.Dmt -taskid ${tasked} -workdir ${workdir} [-action rbt] [-tables schema.tn1,schema.tn2...] ``` * 配置文件说明 -- Gitee From 8bf488dc36acf4f052568c4fb3f50d0b6bc39a99 Mon Sep 17 00:00:00 2001 From: gailibing Date: Fri, 1 Dec 2023 10:00:03 +0800 Subject: [PATCH 14/15] =?UTF-8?q?=E8=B0=83=E6=95=B4=E4=B8=80=E4=BA=9B?= =?UTF-8?q?=E6=98=A0=E5=B0=84=E8=A7=84=E5=88=99=EF=BC=8C=E9=99=84=E8=BF=81?= =?UTF-8?q?=E7=A7=BB=E8=AF=B4=E6=98=8E.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gbase8c/dmt/db/opengauss/MetaImpl.java | 5 +- .../dmt/db/opengauss/TableObjectImpl.java | 93 ++++++++++++++----- .../com/gbase8c/dmt/db/oracle/MetaImpl.java | 12 +-- .../dmt/db/oracle/TableObjectImpl.java | 20 ++-- ...01\347\247\273\350\257\264\346\230\216.md" | 83 +++++++++++++++++ 5 files changed, 173 insertions(+), 40 deletions(-) create mode 100644 "\350\277\201\347\247\273\350\257\264\346\230\216.md" diff --git a/dmt/src/main/java/com/gbase8c/dmt/db/opengauss/MetaImpl.java b/dmt/src/main/java/com/gbase8c/dmt/db/opengauss/MetaImpl.java index 4dd9438..ac5f5a4 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/db/opengauss/MetaImpl.java +++ b/dmt/src/main/java/com/gbase8c/dmt/db/opengauss/MetaImpl.java @@ -174,9 +174,12 @@ public class MetaImpl extends AbstractMeta { // 需要指定长度的字段集合 public static final Set needSetLengthSet = Sets.newHashSet( - "bit", "char", "varchar", "numeric", + "bit", "char", "varchar", "numeric","nvarchar2","nchar", "time", "timestamp", "varbit", "timestamptz", "timetz","interval"); + // 需要指定精度的时间类型集合 + public static final Set timeTypeSet = Sets.newHashSet( + Types.TIME,Types.TIMESTAMP,Types.TIMESTAMP_WITH_TIMEZONE,Types.TIME_WITH_TIMEZONE); private static final Set KEYWORDS = Sets.newHashSet( "all", "analyse", "analyze", "and", "any", "array", "as", "asc", "asymmetric", "authid", "authorization", diff --git a/dmt/src/main/java/com/gbase8c/dmt/db/opengauss/TableObjectImpl.java b/dmt/src/main/java/com/gbase8c/dmt/db/opengauss/TableObjectImpl.java index fc91fc9..84b151b 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/db/opengauss/TableObjectImpl.java +++ b/dmt/src/main/java/com/gbase8c/dmt/db/opengauss/TableObjectImpl.java @@ -553,6 +553,7 @@ public class TableObjectImpl extends MetaImpl implements TableObject { String tarColumnName = column.getTarName(); sb.append(wrap(tarColumnName, preserveCase)).append(" "); String dataType = column.getTarDataType(); + Integer sqlType = column.getSqlType(); if (StringUtils.isNotBlank(dataType) && dataType.equals("interval")) { dataType = column.getDataType(); } @@ -567,38 +568,80 @@ public class TableObjectImpl extends MetaImpl implements TableObject { pkList.add(column.getName()); } - sb.append(dataType); - - //需要指定长度的字段 - if (needSetLengthSet.contains(dataType)) { - - //如果说是numeric类型，是这个样子的 "a23" numeric(255,6) - if (dataType.equals("numeric")) { - Integer dataPrecision = column.getTarDataPrecision(); - Integer dataScale = column.getTarDataScale(); - if (dataPrecision != null) { - if (dataScale != null && dataScale < 0){ - sb.append("(").append(dataPrecision - dataScale); + if(timeTypeSet.contains(sqlType)){ + Integer dataScale = column .getTarDataScale(); + if (dataScale != null) + { + List strings = Arrays.stream(dataType.split(" ")).collect(Collectors.toList()); + if(CollectionUtils.isNotEmpty(strings)){ + String timestampScale = strings.get(0) + "("+ dataScale + ")"; + strings.set(0,timestampScale); + } + sb.append(String.join(" ",strings)); + } + } else { + sb.append(dataType); + if (needSetLengthSet.contains(dataType)) { + + //如果说是numeric类型，是这个样子的 "a23" numeric(255,6) + if (dataType.equals("numeric")) { + Integer dataPrecision = column.getTarDataPrecision(); + Integer dataScale = column.getTarDataScale(); + if (dataPrecision != null) { + if (dataScale != null && dataScale < 0){ + sb.append("(").append(dataPrecision - dataScale); // if (dataScale != null) { sb.append(", ").append(0); // } - sb.append(")"); - scaleLessThan0Col.add(column); - } else { - sb.append("(").append(dataPrecision); - if (dataScale != null) { - sb.append(", ").append(dataScale); + sb.append(")"); + scaleLessThan0Col.add(column); + } else { + sb.append("(").append(dataPrecision); + if (dataScale != null) { + sb.append(", ").append(dataScale); + } + sb.append(")"); } - sb.append(")"); } - } - } else { - //不是numeric字段的 是这个样子的 "a12" varbit(255) + } else { + //不是numeric字段的 是这个样子的 "a12" varbit(255) // Integer dataLength = column.getDataLength(); - if (column.getTarDataLength() != null) { - sb.append("(").append(column.getTarDataLength()).append(")").append(" "); + if (column.getTarDataLength() != null) { + sb.append("(").append(column.getTarDataLength()).append(")").append(" "); + } } - } + } + + //需要指定长度的字段 +// if (needSetLengthSet.contains(dataType)) { +// +// //如果说是numeric类型，是这个样子的 "a23" numeric(255,6) +// if (dataType.equals("numeric")) { +// Integer dataPrecision = column.getTarDataPrecision(); +// Integer dataScale = column.getTarDataScale(); +// if (dataPrecision != null) { +// if (dataScale != null && dataScale < 0){ +// sb.append("(").append(dataPrecision - dataScale); +//// if (dataScale != null) { +// sb.append(", ").append(0); +//// } +// sb.append(")"); +// scaleLessThan0Col.add(column); +// } else { +// sb.append("(").append(dataPrecision); +// if (dataScale != null) { +// sb.append(", ").append(dataScale); +// } +// sb.append(")"); +// } +// } +// } else { +// //不是numeric字段的 是这个样子的 "a12" varbit(255) +//// Integer dataLength = column.getDataLength(); +// if (column.getTarDataLength() != null) { +// sb.append("(").append(column.getTarDataLength()).append(")").append(" "); +// } +// } } //不为空的字段 diff --git a/dmt/src/main/java/com/gbase8c/dmt/db/oracle/MetaImpl.java b/dmt/src/main/java/com/gbase8c/dmt/db/oracle/MetaImpl.java index c19ba9a..e32a646 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/db/oracle/MetaImpl.java +++ b/dmt/src/main/java/com/gbase8c/dmt/db/oracle/MetaImpl.java @@ -43,13 +43,13 @@ public class MetaImpl extends AbstractMeta { dataType2SqlType.put("NUMBER", Types.NUMERIC); dataType2SqlType.put("INTEGER", Types.INTEGER); - dataType2SqlType.put("FLOAT", Types.FLOAT); + dataType2SqlType.put("FLOAT", Types.DOUBLE); dataType2SqlType.put("BINARY_FLOAT", Types.FLOAT); dataType2SqlType.put("BINARY_DOUBLE", Types.DOUBLE); - dataType2SqlType.put("LONG", Types.BLOB); + dataType2SqlType.put("LONG", Types.LONGVARCHAR); dataType2SqlType.put("LONG RAW", Types.BLOB); - dataType2SqlType.put("RAW", Types.CHAR); + dataType2SqlType.put("RAW", Types.BLOB); dataType2SqlType.put("DATE", Types.DATE); dataType2SqlType.put("TIMESTAMP", Types.TIMESTAMP); @@ -118,12 +118,12 @@ public class MetaImpl extends AbstractMeta { @Override public Integer toSqlType(String dataType) { - if (dataType.contains("TIMESTAMP")) { - return dataType2SqlType.getOrDefault("TIMESTAMP", null); - } if (dataType.contains("TIME ZONE")) { return dataType2SqlType.getOrDefault("TIMESTAMP WITH TIME ZONE", null); } + if (dataType.contains("TIMESTAMP")) { + return dataType2SqlType.getOrDefault("TIMESTAMP", null); + } if (dataType.contains("INTERVAL")) { return dataType2SqlType.getOrDefault("INTERVAL", null); } diff --git a/dmt/src/main/java/com/gbase8c/dmt/db/oracle/TableObjectImpl.java b/dmt/src/main/java/com/gbase8c/dmt/db/oracle/TableObjectImpl.java index 3b6de83..37e7620 100644 --- a/dmt/src/main/java/com/gbase8c/dmt/db/oracle/TableObjectImpl.java +++ b/dmt/src/main/java/com/gbase8c/dmt/db/oracle/TableObjectImpl.java @@ -18,6 +18,7 @@ import org.slf4j.LoggerFactory; import java.math.BigDecimal; import java.sql.Timestamp; +import java.sql.Types; import java.text.SimpleDateFormat; import java.util.*; import java.util.regex.Matcher; @@ -129,15 +130,18 @@ public class TableObjectImpl extends MetaImpl implements TableObject { // for (TableDto.ColumnDto columnDto : columnDtos) { columnDto.setSqlType(toSqlType(columnDto.getDataType())); - if (columnDto.getDataType().contains("TIMESTAMP")) { - columnDto.setSqlType(toSqlType("TIMESTAMP")); - } - if (columnDto.getDataType().contains("TIME ZONE")) { - columnDto.setSqlType(toSqlType("TIMESTAMP WITH TIME ZONE")); - } - if (columnDto.getDataType().contains("INTERVAL")) { - columnDto.setSqlType(toSqlType("INTERVAL")); + if (columnDto.getSqlType().equals(Types.NUMERIC) && columnDto.getDataScale() == 0){ + columnDto.setSqlType(Types.INTEGER); } +// if (columnDto.getDataType().contains("TIMESTAMP")) { +// columnDto.setSqlType(toSqlType("TIMESTAMP")); +// } +// if (columnDto.getDataType().contains("TIME ZONE")) { +// columnDto.setSqlType(toSqlType("TIMESTAMP WITH TIME ZONE")); +// } +// if (columnDto.getDataType().contains("INTERVAL")) { +// columnDto.setSqlType(toSqlType("INTERVAL")); +// } // // TODO pk // 主键 diff --git "a/\350\277\201\347\247\273\350\257\264\346\230\216.md" "b/\350\277\201\347\247\273\350\257\264\346\230\216.md" new file mode 100644 index 0000000..3e3386d --- /dev/null +++ "b/\350\277\201\347\247\273\350\257\264\346\230\216.md" @@ -0,0 +1,83 @@ +##默认的类型转换规则 +1.oracle列为查询oracle的all_tab_columns系统表data_type列。 opengauss列为对应后的数据类型，查询information_schema.columns系统表data_type列。 + +2.如task目录下任务配置文件中 “dataTypeConfig”下的“dataTypeMappers”中未配置自定义转换规则，则按照默认转换规则。 +取oracle端all_tab_columns系统表的data_type,data_length,data_precision,data_scale列数据一一对应。 + +| oracle数据类型 | opengauss数据类型 | 备注| +|--------------------------------|------------------|----------------------------------------| +| CHAR | character | 需要拼接长度 | +| NCHAR | character | 需要拼接长度 | +| VARCHAR2 | character varying | 需要拼接长度 | +| NVARCHAR2 | nvarchar2 | 需要拼接长度 | +| NUMBER | numeric | 需要拼接 precision和scale | +| INTEGER | integer | data_type列为NUMBER，且data_scale为0 | +| FLOAT | double_precision | 避免精度丢失，都对应成双精度浮点 | +| BINARY_FLOAT | datax不支持读 | 暂不支持全量迁移 | +| BINARY_DOUBLE | datax不支持读 | 暂不支持全量迁移 | +| LONG | text | | +| LONG RAW | blob | | +| RAW | blob | | +| DATE | date | | +| TIMESTAMP | timestamp | 需要拼接precision | +| TIMESTAMP WITH TIME ZONE | datax不支持读 | 暂不支持全量迁移 | +| TIMESTAMP WITH LOCAL TIME ZONE | datax不支持读 | 暂不支持全量迁移 | +| CLOB | clob | | +| NCLOB | text | | +| BLOB | blob | | +| BFILE | datax不支持读 | 暂不支持全量迁移 | +| ROWID | datax不支持读 | 暂不支持全量迁移 | +| UROWID | datax不支持读 | 暂不支持全量迁移 | +| XMLTYPE | datax不支持读 | 暂不支持全量迁移 | + +如配置自定义转换规则，则按如下格式进行定义： +```json +"dataTypeConfig" : { + "dataTypeMappers" : [ + { + "srcDataType" : "VARCHAR2", + "srcDataLength" : 256, + "tarDataType" : "varchar", + "tarDataLength" : 500 + }, + { + "srcDataType" : "NUMBER", + "srcDataLength" : 22, + "srcDataPrecision" : 5, + "srcDataScale" : 2, + "tarDataType" : "numeric", + "tarDataLength" : 22, + "tarDataPrecision" : 6, + "tarDataScale" : 1 + } +] +}, +``` + +##可用用例，oracle端建表语句 +```oracle +CREATE TABLE test.t_table1( +c_char char, +c_nchar nchar, +c_varchar2 varchar2(255), +c_nvarchar2 nvarchar2(64), +c_number number(5,2), +c_integer integer, +c_float float(4), +c_long LONG, +c_raw raw(2), +c_date DATE, +c_timestamp timestamp, +c_clob clob, +c_nclob nclob, +c_blob blob +); + +INSERT INTO test.T_TABLE1 values('a','b','height','mediumn',1.9,33,3.3,'longinsert','rawi', +TO_DATE('2022-09-30', 'YYYY-MM-DD'), +TO_TIMESTAMP('2022-01-01 10:30:00', 'YYYY-MM-DD HH24:MI:SS'),'clobinsert', +'nclobinsert','48656C6C6F20576F726C64'); + +create table test.t_table2(c_long_raw long raw); +insert into test.t_table2 values('48656C6C6F20576F726C64'); +``` \ No newline at end of file -- Gitee From 2c70bbe22ca494fb78308aa176a0e3ee83579ae7 Mon Sep 17 00:00:00 2001 From: gailibing Date: Fri, 1 Dec 2023 10:16:48 +0800 Subject: [PATCH 15/15] =?UTF-8?q?=E8=BF=81=E7=A7=BB=E8=AF=B4=E6=98=8E.md?= =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- "\350\277\201\347\247\273\350\257\264\346\230\216.md" | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git "a/\350\277\201\347\247\273\350\257\264\346\230\216.md" "b/\350\277\201\347\247\273\350\257\264\346\230\216.md" index 3e3386d..aa73d00 100644 --- "a/\350\277\201\347\247\273\350\257\264\346\230\216.md" +++ "b/\350\277\201\347\247\273\350\257\264\346\230\216.md" @@ -1,4 +1,5 @@ -##默认的类型转换规则 +# 默认的类型转换规则 + 1.oracle列为查询oracle的all_tab_columns系统表data_type列。 opengauss列为对应后的数据类型，查询information_schema.columns系统表data_type列。 2.如task目录下任务配置文件中 “dataTypeConfig”下的“dataTypeMappers”中未配置自定义转换规则，则按照默认转换规则。 @@ -54,7 +55,7 @@ }, ``` -##可用用例，oracle端建表语句 +# 可用用例，oracle端建表语句 ```oracle CREATE TABLE test.t_table1( c_char char, -- Gitee