seg_yinzy
/
DolphinScheduler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix DataX and Sqoop Task execute NullPointerException (#9192)

migrate-dev-docs-to-main
Kerwin 2022-03-26 20:15:41 +08:00 committed by GitHub
parent e8eb50e738
commit d7d756e7b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 41 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
View File

@ -317,6 +317,9 @@ public abstract class BaseTaskProcessor implements ITaskProcessor {
map.forEach((code, parameters) -> {
DataSource datasource = processService.findDataSourceById(code);
if (Objects.isNull(datasource)) {
return;
}
DataSourceParameters dataSourceParameters = new DataSourceParameters();
dataSourceParameters.setType(datasource.getType());
dataSourceParameters.setConnectionParams(datasource.getConnectionParams());

13
dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java
View File

@ -256,17 +256,26 @@ public class DataxParameters extends AbstractParameters {
@Override
public ResourceParametersHelper getResources() {
ResourceParametersHelper resources = super.getResources();
if (customConfig == Flag.YES.ordinal()) {
return resources;
}
resources.put(ResourceType.DATASOURCE, dataSource);
resources.put(ResourceType.DATASOURCE, dataTarget);
return resources;
}
public DataxTaskExecutionContext generateExtendedContext(ResourceParametersHelper parametersHelper) {
DataSourceParameters dbSource = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, dataSource);
DataSourceParameters dbTarget = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, dataTarget);
DataxTaskExecutionContext dataxTaskExecutionContext = new DataxTaskExecutionContext();
if (customConfig == Flag.YES.ordinal()) {
return dataxTaskExecutionContext;
}
DataSourceParameters dbSource = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, dataSource);
DataSourceParameters dbTarget = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, dataTarget);
if (Objects.nonNull(dbSource)) {
dataxTaskExecutionContext.setDataSourceId(dataSource);
dataxTaskExecutionContext.setSourcetype(dbSource.getType());

45
dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java
View File

@ -206,37 +206,46 @@ public class SqoopParameters extends AbstractParameters {
@Override
public ResourceParametersHelper getResources() {
ResourceParametersHelper resources = super.getResources();
if (SqoopJobType.TEMPLATE.getDescp().equals(this.getJobType())) {
SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(this.getSourceParams(), SourceMysqlParameter.class);
TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(this.getTargetParams(), TargetMysqlParameter.class);
if (!SqoopJobType.TEMPLATE.getDescp().equals(this.getJobType())) {
return resources;
}
SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(this.getSourceParams(), SourceMysqlParameter.class);
if (sourceMysqlParameter.getSrcDatasource() != 0) {
resources.put(ResourceType.DATASOURCE, sourceMysqlParameter.getSrcDatasource());
}
TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(this.getTargetParams(), TargetMysqlParameter.class);
if (targetMysqlParameter.getTargetDatasource() != 0) {
resources.put(ResourceType.DATASOURCE, targetMysqlParameter.getTargetDatasource());
}
return resources;
}
public SqoopTaskExecutionContext generateExtendedContext(ResourceParametersHelper parametersHelper) {
SqoopTaskExecutionContext sqoopTaskExecutionContext = new SqoopTaskExecutionContext();
if (!SqoopJobType.TEMPLATE.getDescp().equals(this.getJobType())) {
return sqoopTaskExecutionContext;
}
if (SqoopJobType.TEMPLATE.getDescp().equals(this.getJobType())) {
SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(this.getSourceParams(), SourceMysqlParameter.class);
TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(this.getTargetParams(), TargetMysqlParameter.class);
SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(this.getSourceParams(), SourceMysqlParameter.class);
TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(this.getTargetParams(), TargetMysqlParameter.class);
DataSourceParameters dataSource = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, sourceMysqlParameter.getSrcDatasource());
DataSourceParameters dataTarget = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, targetMysqlParameter.getTargetDatasource());
DataSourceParameters dataSource = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, sourceMysqlParameter.getSrcDatasource());
DataSourceParameters dataTarget = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, targetMysqlParameter.getTargetDatasource());
if (Objects.nonNull(dataSource)) {
sqoopTaskExecutionContext.setDataSourceId(sourceMysqlParameter.getSrcDatasource());
sqoopTaskExecutionContext.setSourcetype(dataSource.getType());
sqoopTaskExecutionContext.setSourceConnectionParams(dataSource.getConnectionParams());
}
if (Objects.nonNull(dataSource)) {
sqoopTaskExecutionContext.setDataSourceId(sourceMysqlParameter.getSrcDatasource());
sqoopTaskExecutionContext.setSourcetype(dataSource.getType());
sqoopTaskExecutionContext.setSourceConnectionParams(dataSource.getConnectionParams());
}
if (Objects.nonNull(dataTarget)) {
sqoopTaskExecutionContext.setDataTargetId(targetMysqlParameter.getTargetDatasource());
sqoopTaskExecutionContext.setTargetType(dataTarget.getType());
sqoopTaskExecutionContext.setTargetConnectionParams(dataTarget.getConnectionParams());
}
if (Objects.nonNull(dataTarget)) {
sqoopTaskExecutionContext.setDataTargetId(targetMysqlParameter.getTargetDatasource());
sqoopTaskExecutionContext.setTargetType(dataTarget.getType());
sqoopTaskExecutionContext.setTargetConnectionParams(dataTarget.getConnectionParams());
}
return sqoopTaskExecutionContext;