[Feature-8579][Task] Task back-end plugin optimization, new plugins only need to modify the plugin's own module (#8584)
* Task plug-in optimization, task-related classes are migrated to the task plugin module * remove tasktype enum class. * fix code style. * fix code style * fix code style * fix ProcessServiceTest.testUpdateResourceInfo test class * fix code style * fix test task plugin manager check parameters null * fix test ut * update aws license * change TaskTimeoutStrategy classslim
parent
8bcbe2a1f4
commit
a1bef43981
|
|
@ -45,6 +45,11 @@
|
|||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-task-all</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-ui-next</artifactId>
|
||||
|
|
|
|||
|
|
@ -17,10 +17,10 @@
|
|||
|
||||
package org.apache.dolphinscheduler.api.controller;
|
||||
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ACCESSTOKEN_BY_USER_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.CREATE_ACCESS_TOKEN_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.DELETE_ACCESS_TOKEN_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.GENERATE_TOKEN_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ACCESSTOKEN_BY_USER_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ACCESSTOKEN_LIST_PAGING_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_ACCESS_TOKEN_ERROR;
|
||||
|
||||
|
|
|
|||
|
|
@ -34,11 +34,10 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.ProcessInstanceService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import static org.apache.dolphinscheduler.api.enums.Status.CREATE_PROCESS_TASK_R
|
|||
import static org.apache.dolphinscheduler.api.enums.Status.DATA_IS_NOT_VALID;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.DELETE_EDGE_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.DELETE_TASK_PROCESS_RELATION_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.MOVE_PROCESS_TASK_RELATION_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_PROCESS_RELATION_ERROR;
|
||||
|
||||
import org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation;
|
||||
|
|
|
|||
|
|
@ -17,12 +17,12 @@
|
|||
|
||||
package org.apache.dolphinscheduler.api.controller;
|
||||
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_AUTHORIZED_USER;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.CREATE_PROJECT_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.DELETE_PROJECT_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.LOGIN_USER_QUERY_PROJECT_LIST_PAGING_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_AUTHORIZED_AND_USER_CREATED_PROJECT_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_AUTHORIZED_PROJECT;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_AUTHORIZED_USER;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROJECT_DETAILS_BY_CODE_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_UNAUTHORIZED_PROJECT_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_PROJECT_ERROR;
|
||||
|
|
|
|||
|
|
@ -34,7 +34,6 @@ import org.apache.dolphinscheduler.api.service.TaskDefinitionService;
|
|||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
|
|
@ -53,6 +52,8 @@ import org.springframework.web.bind.annotation.RequestParam;
|
|||
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import io.swagger.annotations.Api;
|
||||
import io.swagger.annotations.ApiImplicitParam;
|
||||
import io.swagger.annotations.ApiImplicitParams;
|
||||
|
|
@ -60,8 +61,6 @@ import io.swagger.annotations.ApiOperation;
|
|||
import io.swagger.annotations.ApiParam;
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
* task definition controller
|
||||
*/
|
||||
|
|
@ -344,7 +343,7 @@ public class TaskDefinitionController extends BaseController {
|
|||
@ApiParam(name = "projectCode", value = "PROJECT_CODE", required = true) @PathVariable long projectCode,
|
||||
@RequestParam(value = "searchWorkflowName", required = false) String searchWorkflowName,
|
||||
@RequestParam(value = "searchTaskName", required = false) String searchTaskName,
|
||||
@RequestParam(value = "taskType", required = false) TaskType taskType,
|
||||
@RequestParam(value = "taskType", required = false) String taskType,
|
||||
@RequestParam("pageNo") Integer pageNo,
|
||||
@RequestParam("pageSize") Integer pageSize) {
|
||||
Result result = checkPageParams(pageNo, pageSize);
|
||||
|
|
|
|||
|
|
@ -17,10 +17,13 @@
|
|||
|
||||
package org.apache.dolphinscheduler.api.controller;
|
||||
|
||||
import io.swagger.annotations.Api;
|
||||
import io.swagger.annotations.ApiImplicitParam;
|
||||
import io.swagger.annotations.ApiImplicitParams;
|
||||
import io.swagger.annotations.ApiOperation;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.CLOSE_TASK_GROUP_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.CREATE_TASK_GROUP_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_GROUP_LIST_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_GROUP_QUEUE_LIST_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.START_TASK_GROUP_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_TASK_GROUP_ERROR;
|
||||
|
||||
import org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation;
|
||||
import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
||||
import org.apache.dolphinscheduler.api.service.TaskGroupQueueService;
|
||||
|
|
@ -29,6 +32,11 @@ import org.apache.dolphinscheduler.api.utils.Result;
|
|||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import io.swagger.annotations.Api;
|
||||
import io.swagger.annotations.ApiImplicitParam;
|
||||
import io.swagger.annotations.ApiImplicitParams;
|
||||
import io.swagger.annotations.ApiOperation;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
@ -42,16 +50,6 @@ import org.springframework.web.bind.annotation.ResponseStatus;
|
|||
import org.springframework.web.bind.annotation.RestController;
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.CLOSE_TASK_GROUP_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.CREATE_TASK_GROUP_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_GROUP_LIST_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_GROUP_QUEUE_LIST_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.START_TASK_GROUP_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_TASK_GROUP_ERROR;
|
||||
|
||||
|
||||
/**
|
||||
* task group controller
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.TaskInstanceService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ import org.springframework.beans.factory.annotation.Autowired;
|
|||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestAttribute;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@
|
|||
|
||||
package org.apache.dolphinscheduler.api.controller;
|
||||
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.REVOKE_PROJECT_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.AUTHORIZED_USER_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.CREATE_USER_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.DELETE_USER_BY_ID_ERROR;
|
||||
|
|
@ -27,6 +26,7 @@ import static org.apache.dolphinscheduler.api.enums.Status.GRANT_PROJECT_ERROR;
|
|||
import static org.apache.dolphinscheduler.api.enums.Status.GRANT_RESOURCE_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.GRANT_UDF_FUNCTION_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_USER_LIST_PAGING_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.REVOKE_PROJECT_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.UNAUTHORIZED_USER_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_USER_ERROR;
|
||||
import static org.apache.dolphinscheduler.api.enums.Status.USER_LIST_ERROR;
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.dto;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.CommandType;
|
||||
|
|
@ -28,6 +29,7 @@ public class CommandStateCount {
|
|||
private CommandType commandState;
|
||||
|
||||
public CommandStateCount(){}
|
||||
|
||||
public CommandStateCount(int errorCount, int normalCount, CommandType commandState) {
|
||||
this.errorCount = errorCount;
|
||||
this.normalCount = normalCount;
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.dto;
|
||||
|
||||
import org.apache.dolphinscheduler.dao.entity.DefinitionGroupByUser;
|
||||
|
|
@ -31,7 +32,7 @@ public class DefineUserDto {
|
|||
|
||||
public DefineUserDto(List<DefinitionGroupByUser> defineGroupByUsers) {
|
||||
|
||||
for(DefinitionGroupByUser define : defineGroupByUsers){
|
||||
for (DefinitionGroupByUser define : defineGroupByUsers) {
|
||||
count += define.getCount();
|
||||
}
|
||||
this.userList = defineGroupByUsers;
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.dto;
|
|||
|
||||
import java.util.Date;
|
||||
|
||||
|
||||
/**
|
||||
* schedule parameters
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -14,10 +14,11 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.dto;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
|
@ -39,7 +40,6 @@ public class TaskCountDto {
|
|||
*/
|
||||
private List<TaskStateCount> taskCountDtos;
|
||||
|
||||
|
||||
public TaskCountDto(List<ExecuteStatusCount> taskInstanceStateCounts) {
|
||||
countTaskDtos(taskInstanceStateCounts);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,9 +14,10 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.dto;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
|
||||
/**
|
||||
* task state count
|
||||
|
|
@ -31,7 +32,6 @@ public class TaskStateCount {
|
|||
this.count = count;
|
||||
}
|
||||
|
||||
|
||||
public int getCount() {
|
||||
return count;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.dolphinscheduler.api.dto.gantt;
|
||||
|
||||
package org.apache.dolphinscheduler.api.dto.gantt;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.LinkedList;
|
||||
|
|
|
|||
|
|
@ -14,12 +14,13 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.dto.resources;
|
||||
|
||||
/**
|
||||
* directory
|
||||
*/
|
||||
public class Directory extends ResourceComponent{
|
||||
public class Directory extends ResourceComponent {
|
||||
|
||||
@Override
|
||||
public boolean isDirctory() {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.dto.resources.visitor;
|
||||
|
||||
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.dto.treeview;
|
|||
|
||||
import java.util.Date;
|
||||
|
||||
|
||||
/**
|
||||
* Instance
|
||||
*/
|
||||
|
|
@ -97,7 +96,6 @@ public class Instance {
|
|||
this(id, name, code, type, state, startTime, endTime, host, duration, 0);
|
||||
}
|
||||
|
||||
|
||||
public int getId() {
|
||||
return id;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.enums;
|
||||
|
||||
/**
|
||||
|
|
@ -21,7 +22,6 @@ package org.apache.dolphinscheduler.api.enums;
|
|||
*/
|
||||
public enum ExecuteType {
|
||||
|
||||
|
||||
/**
|
||||
* operation type
|
||||
* 1 repeat running
|
||||
|
|
@ -32,10 +32,9 @@ public enum ExecuteType {
|
|||
*/
|
||||
NONE, REPEAT_RUNNING, RECOVER_SUSPENDED_PROCESS, START_FAILURE_TASK_PROCESS, STOP, PAUSE;
|
||||
|
||||
|
||||
public static ExecuteType getEnum(int value){
|
||||
public static ExecuteType getEnum(int value) {
|
||||
for (ExecuteType e: ExecuteType.values()) {
|
||||
if(e.ordinal() == value) {
|
||||
if (e.ordinal() == value) {
|
||||
return e;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,16 +14,17 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.exceptions;
|
||||
|
||||
import static java.lang.annotation.ElementType.METHOD;
|
||||
import static java.lang.annotation.RetentionPolicy.RUNTIME;
|
||||
|
||||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.Target;
|
||||
|
||||
import static java.lang.annotation.ElementType.METHOD;
|
||||
import static java.lang.annotation.RetentionPolicy.RUNTIME;
|
||||
|
||||
/**
|
||||
* controller exception annotation
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.exceptions;
|
||||
|
||||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ public interface AlertGroupService {
|
|||
* @return one alert group
|
||||
*/
|
||||
Map<String, Object> queryAlertGroupById(User loginUser, Integer id);
|
||||
|
||||
/**
|
||||
* paging query alarm group list
|
||||
*
|
||||
|
|
|
|||
|
|
@ -19,10 +19,10 @@
|
|||
package org.apache.dolphinscheduler.api.service;
|
||||
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.enums.DependResult;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.service;
|
|||
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import java.util.Map;
|
||||
|
|
@ -178,7 +177,7 @@ public interface TaskDefinitionService {
|
|||
long projectCode,
|
||||
String searchWorkflowName,
|
||||
String searchTaskName,
|
||||
TaskType taskType,
|
||||
String taskType,
|
||||
Integer pageNo,
|
||||
Integer pageSize);
|
||||
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@
|
|||
package org.apache.dolphinscheduler.api.service;
|
||||
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
package org.apache.dolphinscheduler.api.service;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* work flow lineage service
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@
|
|||
|
||||
package org.apache.dolphinscheduler.api.service.impl;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
import org.apache.dolphinscheduler.api.service.AccessTokenService;
|
||||
import org.apache.dolphinscheduler.api.utils.PageInfo;
|
||||
|
|
@ -30,6 +29,8 @@ import org.apache.dolphinscheduler.dao.entity.AccessToken;
|
|||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.api.service.DataAnalysisService;
|
|||
import org.apache.dolphinscheduler.api.service.ProjectService;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.CommandType;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.TriFunction;
|
||||
|
|
@ -40,6 +39,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ import org.apache.dolphinscheduler.dao.mapper.DqRuleExecuteSqlMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.DqRuleInputEntryMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.DqRuleMapper;
|
||||
import org.apache.dolphinscheduler.dao.utils.DqRuleUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.dp.OptionSourceType;
|
||||
import org.apache.dolphinscheduler.spi.enums.DbType;
|
||||
import org.apache.dolphinscheduler.spi.params.base.FormType;
|
||||
import org.apache.dolphinscheduler.spi.params.base.ParamsOptions;
|
||||
|
|
@ -51,7 +52,6 @@ import org.apache.dolphinscheduler.spi.params.group.GroupParamsProps;
|
|||
import org.apache.dolphinscheduler.spi.params.input.InputParam;
|
||||
import org.apache.dolphinscheduler.spi.params.input.InputParamProps;
|
||||
import org.apache.dolphinscheduler.spi.params.select.SelectParam;
|
||||
import org.apache.dolphinscheduler.spi.task.dq.enums.OptionSourceType;
|
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODES
|
|||
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_PARAMS;
|
||||
import static org.apache.dolphinscheduler.common.Constants.MAX_TASK_TIMEOUT;
|
||||
|
||||
import org.apache.commons.beanutils.BeanUtils;
|
||||
import org.apache.dolphinscheduler.api.enums.ExecuteType;
|
||||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
import org.apache.dolphinscheduler.api.service.ExecutorService;
|
||||
|
|
@ -34,7 +33,6 @@ import org.apache.dolphinscheduler.common.Constants;
|
|||
import org.apache.dolphinscheduler.common.enums.CommandType;
|
||||
import org.apache.dolphinscheduler.common.enums.ComplementDependentMode;
|
||||
import org.apache.dolphinscheduler.common.enums.CycleEnum;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.enums.FailureStrategy;
|
||||
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||
import org.apache.dolphinscheduler.common.enums.Priority;
|
||||
|
|
@ -58,11 +56,13 @@ import org.apache.dolphinscheduler.dao.entity.User;
|
|||
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand;
|
||||
import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
|
||||
|
||||
import org.apache.commons.beanutils.BeanUtils;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.collections.MapUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.api.service.impl;
|
|||
|
||||
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE;
|
||||
import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.COMPLEX_TASK_TYPES;
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SQL;
|
||||
|
||||
import org.apache.dolphinscheduler.api.dto.DagDataSchedule;
|
||||
import org.apache.dolphinscheduler.api.dto.ScheduleParam;
|
||||
|
|
@ -41,16 +43,13 @@ import org.apache.dolphinscheduler.common.enums.Flag;
|
|||
import org.apache.dolphinscheduler.common.enums.Priority;
|
||||
import org.apache.dolphinscheduler.common.enums.ProcessExecutionTypeEnum;
|
||||
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
|
||||
import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
|
||||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.common.enums.WarningType;
|
||||
import org.apache.dolphinscheduler.common.graph.DAG;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
||||
import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
|
||||
import org.apache.dolphinscheduler.common.task.sql.SqlType;
|
||||
import org.apache.dolphinscheduler.common.thread.Stopper;
|
||||
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
|
||||
|
|
@ -82,11 +81,14 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.SqlType;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.SqlParameters;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hadoop.fs.Stat;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.BufferedReader;
|
||||
|
|
@ -185,6 +187,9 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
|
|||
@Autowired
|
||||
private DataSourceMapper dataSourceMapper;
|
||||
|
||||
@Autowired
|
||||
private TaskPluginManager taskPluginManager;
|
||||
|
||||
/**
|
||||
* create process definition
|
||||
*
|
||||
|
|
@ -298,7 +303,12 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
|
|||
return result;
|
||||
}
|
||||
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
|
||||
if (!CheckUtils.checkTaskDefinitionParameters(taskDefinitionLog)) {
|
||||
|
||||
if (!taskPluginManager.checkTaskParameters(ParametersNode.builder()
|
||||
.taskType(taskDefinitionLog.getTaskType())
|
||||
.taskParams(taskDefinitionLog.getTaskParams())
|
||||
.dependence(taskDefinitionLog.getDependence())
|
||||
.build())) {
|
||||
logger.error("task definition {} parameter invalid", taskDefinitionLog.getName());
|
||||
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName());
|
||||
return result;
|
||||
|
|
@ -1085,7 +1095,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
|
|||
sqlParameters.setLocalParams(Collections.emptyList());
|
||||
taskDefinition.setTaskParams(JSONUtils.toJsonString(sqlParameters));
|
||||
taskDefinition.setCode(CodeGenerateUtils.getInstance().genCode());
|
||||
taskDefinition.setTaskType(TaskType.SQL.getDesc());
|
||||
taskDefinition.setTaskType(TASK_TYPE_SQL);
|
||||
taskDefinition.setFailRetryTimes(0);
|
||||
taskDefinition.setFailRetryInterval(0);
|
||||
taskDefinition.setTimeoutFlag(TimeoutFlag.CLOSE);
|
||||
|
|
@ -1287,7 +1297,12 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
|
|||
|
||||
// check whether the process definition json is normal
|
||||
for (TaskNode taskNode : taskNodes) {
|
||||
if (!CheckUtils.checkTaskNodeParameters(taskNode)) {
|
||||
if (!taskPluginManager.checkTaskParameters(ParametersNode.builder()
|
||||
.taskType(taskNode.getType())
|
||||
.taskParams(taskNode.getTaskParams())
|
||||
.dependence(taskNode.getDependence())
|
||||
.switchResult(taskNode.getSwitchResult())
|
||||
.build())) {
|
||||
logger.error("task node {} parameter invalid", taskNode.getName());
|
||||
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskNode.getName());
|
||||
return result;
|
||||
|
|
@ -1650,10 +1665,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
|
|||
List<TaskDefinitionLog> taskDefinitionLogs = processService.genTaskDefineList(processTaskRelations);
|
||||
Map<Long, Long> taskCodeMap = new HashMap<>();
|
||||
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
|
||||
if (TaskType.CONDITIONS.getDesc().equals(taskDefinitionLog.getTaskType())
|
||||
|| TaskType.SWITCH.getDesc().equals(taskDefinitionLog.getTaskType())
|
||||
|| TaskType.SUB_PROCESS.getDesc().equals(taskDefinitionLog.getTaskType())
|
||||
|| TaskType.DEPENDENT.getDesc().equals(taskDefinitionLog.getTaskType())) {
|
||||
if (COMPLEX_TASK_TYPES.contains(taskDefinitionLog.getTaskType())) {
|
||||
putMsg(result, Status.NOT_SUPPORT_COPY_TASK_TYPE, taskDefinitionLog.getTaskType());
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import static org.apache.dolphinscheduler.common.Constants.GLOBAL_PARAMS;
|
|||
import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS;
|
||||
import static org.apache.dolphinscheduler.common.Constants.PROCESS_INSTANCE_STATE;
|
||||
import static org.apache.dolphinscheduler.common.Constants.TASK_LIST;
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
|
||||
|
||||
import org.apache.dolphinscheduler.api.dto.gantt.GanttDto;
|
||||
import org.apache.dolphinscheduler.api.dto.gantt.Task;
|
||||
|
|
@ -34,18 +35,13 @@ import org.apache.dolphinscheduler.api.service.ProcessDefinitionService;
|
|||
import org.apache.dolphinscheduler.api.service.ProcessInstanceService;
|
||||
import org.apache.dolphinscheduler.api.service.ProjectService;
|
||||
import org.apache.dolphinscheduler.api.service.UsersService;
|
||||
import org.apache.dolphinscheduler.api.utils.CheckUtils;
|
||||
import org.apache.dolphinscheduler.api.utils.PageInfo;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.DependResult;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||
import org.apache.dolphinscheduler.common.graph.DAG;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
||||
import org.apache.dolphinscheduler.common.process.Property;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
|
|
@ -67,7 +63,12 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
|
@ -145,6 +146,9 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
|
|||
@Autowired
|
||||
TaskDefinitionMapper taskDefinitionMapper;
|
||||
|
||||
@Autowired
|
||||
private TaskPluginManager taskPluginManager;
|
||||
|
||||
/**
|
||||
* return top n SUCCESS process instance order by running time which started between startTime and endTime
|
||||
*/
|
||||
|
|
@ -336,7 +340,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
|
|||
*/
|
||||
private void addDependResultForTaskList(List<TaskInstance> taskInstanceList) throws IOException {
|
||||
for (TaskInstance taskInstance : taskInstanceList) {
|
||||
if (TaskType.DEPENDENT.getDesc().equalsIgnoreCase(taskInstance.getTaskType())) {
|
||||
if (TASK_TYPE_DEPENDENT.equalsIgnoreCase(taskInstance.getTaskType())) {
|
||||
Result<String> logResult = loggerService.queryLog(
|
||||
taskInstance.getId(), Constants.LOG_QUERY_SKIP_LINE_NUMBER, Constants.LOG_QUERY_LIMIT);
|
||||
if (logResult.getCode() == Status.SUCCESS.ordinal()) {
|
||||
|
|
@ -476,7 +480,11 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
|
|||
return result;
|
||||
}
|
||||
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
|
||||
if (!CheckUtils.checkTaskDefinitionParameters(taskDefinitionLog)) {
|
||||
if (!taskPluginManager.checkTaskParameters(ParametersNode.builder()
|
||||
.taskType(taskDefinitionLog.getTaskType())
|
||||
.taskParams(taskDefinitionLog.getTaskParams())
|
||||
.dependence(taskDefinitionLog.getDependence())
|
||||
.build())) {
|
||||
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName());
|
||||
return result;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,13 +17,16 @@
|
|||
|
||||
package org.apache.dolphinscheduler.api.service.impl;
|
||||
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS;
|
||||
|
||||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
|
||||
import org.apache.dolphinscheduler.api.service.ProcessTaskRelationService;
|
||||
import org.apache.dolphinscheduler.api.service.ProjectService;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ConditionType;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
|
||||
|
|
@ -228,9 +231,9 @@ public class ProcessTaskRelationServiceImpl extends BaseServiceImpl implements P
|
|||
}
|
||||
updateProcessDefiniteVersion(loginUser, result, processDefinition);
|
||||
updateRelation(loginUser, result, processDefinition, processTaskRelationList);
|
||||
if (TaskType.CONDITIONS.getDesc().equals(taskDefinition.getTaskType())
|
||||
|| TaskType.DEPENDENT.getDesc().equals(taskDefinition.getTaskType())
|
||||
|| TaskType.SUB_PROCESS.getDesc().equals(taskDefinition.getTaskType())) {
|
||||
if (TASK_TYPE_CONDITIONS.equals(taskDefinition.getTaskType())
|
||||
|| TASK_TYPE_DEPENDENT.equals(taskDefinition.getTaskType())
|
||||
|| TASK_TYPE_SUB_PROCESS.equals(taskDefinition.getTaskType())) {
|
||||
int deleteTaskDefinition = taskDefinitionMapper.deleteByCode(taskCode);
|
||||
if (0 == deleteTaskDefinition) {
|
||||
putMsg(result, Status.DELETE_TASK_DEFINE_BY_CODE_ERROR);
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.api.enums.Status;
|
|||
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
|
||||
import org.apache.dolphinscheduler.api.service.ProjectService;
|
||||
import org.apache.dolphinscheduler.api.service.TaskDefinitionService;
|
||||
import org.apache.dolphinscheduler.api.utils.CheckUtils;
|
||||
import org.apache.dolphinscheduler.api.utils.PageInfo;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
|
|
@ -29,7 +28,6 @@ import org.apache.dolphinscheduler.common.enums.AuthorizationType;
|
|||
import org.apache.dolphinscheduler.common.enums.ConditionType;
|
||||
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
|
|
@ -46,8 +44,10 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
|
||||
import org.apache.dolphinscheduler.service.permission.PermissionCheck;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
|
@ -103,6 +103,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
|
|||
@Autowired
|
||||
private ProcessService processService;
|
||||
|
||||
@Autowired
|
||||
private TaskPluginManager taskPluginManager;
|
||||
|
||||
/**
|
||||
* create task definition
|
||||
*
|
||||
|
|
@ -129,7 +132,11 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
|
|||
return result;
|
||||
}
|
||||
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
|
||||
if (!CheckUtils.checkTaskDefinitionParameters(taskDefinitionLog)) {
|
||||
if (!taskPluginManager.checkTaskParameters(ParametersNode.builder()
|
||||
.taskType(taskDefinitionLog.getTaskType())
|
||||
.taskParams(taskDefinitionLog.getTaskParams())
|
||||
.dependence(taskDefinitionLog.getDependence())
|
||||
.build())) {
|
||||
logger.error("task definition {} parameter invalid", taskDefinitionLog.getName());
|
||||
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName());
|
||||
return result;
|
||||
|
|
@ -186,7 +193,11 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
|
|||
putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj);
|
||||
return result;
|
||||
}
|
||||
if (!CheckUtils.checkTaskDefinitionParameters(taskDefinition)) {
|
||||
if (!taskPluginManager.checkTaskParameters(ParametersNode.builder()
|
||||
.taskType(taskDefinition.getTaskType())
|
||||
.taskParams(taskDefinition.getTaskParams())
|
||||
.dependence(taskDefinition.getDependence())
|
||||
.build())) {
|
||||
logger.error("task definition {} parameter invalid", taskDefinition.getName());
|
||||
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinition.getName());
|
||||
return result;
|
||||
|
|
@ -411,7 +422,11 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
|
|||
putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj);
|
||||
return null;
|
||||
}
|
||||
if (!CheckUtils.checkTaskDefinitionParameters(taskDefinitionToUpdate)) {
|
||||
if (!taskPluginManager.checkTaskParameters(ParametersNode.builder()
|
||||
.taskType(taskDefinitionToUpdate.getTaskType())
|
||||
.taskParams(taskDefinitionToUpdate.getTaskParams())
|
||||
.dependence(taskDefinitionToUpdate.getDependence())
|
||||
.build())) {
|
||||
logger.error("task definition {} parameter invalid", taskDefinitionToUpdate.getName());
|
||||
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionToUpdate.getName());
|
||||
return null;
|
||||
|
|
@ -643,7 +658,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
|
|||
long projectCode,
|
||||
String searchWorkflowName,
|
||||
String searchTaskName,
|
||||
TaskType taskType,
|
||||
String taskType,
|
||||
Integer pageNo,
|
||||
Integer pageSize) {
|
||||
Result result = new Result();
|
||||
|
|
@ -657,7 +672,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
|
|||
}
|
||||
Page<TaskMainInfo> page = new Page<>(pageNo, pageSize);
|
||||
IPage<TaskMainInfo> taskMainInfoIPage = taskDefinitionMapper.queryDefineListPaging(page, projectCode, searchWorkflowName,
|
||||
searchTaskName, taskType == null ? StringUtils.EMPTY : taskType.getDesc());
|
||||
searchTaskName, taskType == null ? StringUtils.EMPTY : taskType);
|
||||
List<TaskMainInfo> records = taskMainInfoIPage.getRecords();
|
||||
if (!records.isEmpty()) {
|
||||
Map<Long, TaskMainInfo> taskMainInfoMap = new HashMap<>();
|
||||
|
|
|
|||
|
|
@ -17,8 +17,6 @@
|
|||
|
||||
package org.apache.dolphinscheduler.api.service.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
import org.apache.dolphinscheduler.api.service.ProjectService;
|
||||
import org.apache.dolphinscheduler.api.service.TaskGroupQueueService;
|
||||
|
|
@ -29,14 +27,18 @@ import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue;
|
|||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
|
||||
/**
|
||||
* task group queue service
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.api.service.UsersService;
|
|||
import org.apache.dolphinscheduler.api.utils.PageInfo;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.Project;
|
||||
|
|
@ -35,6 +34,7 @@ import org.apache.dolphinscheduler.dao.entity.User;
|
|||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
|
||||
import java.util.Date;
|
||||
|
|
|
|||
|
|
@ -17,13 +17,11 @@
|
|||
|
||||
package org.apache.dolphinscheduler.api.service.impl;
|
||||
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
|
||||
|
||||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
import org.apache.dolphinscheduler.api.service.WorkFlowLineageService;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||
import org.apache.dolphinscheduler.common.model.DependentItem;
|
||||
import org.apache.dolphinscheduler.common.model.DependentTaskModel;
|
||||
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.DependentProcessDefinition;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessLineage;
|
||||
|
|
@ -35,6 +33,9 @@ import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
|
|||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
|
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
|
@ -201,7 +202,7 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF
|
|||
List<TaskDefinitionLog> taskDefinitionLogs = taskDefinitionLogMapper.queryByTaskDefinitions(taskDefinitionList);
|
||||
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
|
||||
if (taskDefinitionLog.getProjectCode() == projectCode) {
|
||||
if (taskDefinitionLog.getTaskType().equals(TaskType.DEPENDENT.getDesc())) {
|
||||
if (taskDefinitionLog.getTaskType().equals(TASK_TYPE_DEPENDENT)) {
|
||||
DependentParameters dependentParameters = JSONUtils.parseObject(taskDefinitionLog.getDependence(), DependentParameters.class);
|
||||
if (dependentParameters != null) {
|
||||
List<DependentTaskModel> dependTaskList = dependentParameters.getDependTaskList();
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
|
|||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.NodeType;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
|
||||
|
|
|
|||
|
|
@ -14,19 +14,14 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.hibernate.validator.internal.constraintvalidators.bv.EmailValidator;
|
||||
|
||||
import java.text.MessageFormat;
|
||||
import java.time.ZoneId;
|
||||
|
|
@ -34,6 +29,7 @@ import java.util.HashMap;
|
|||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.hibernate.validator.internal.constraintvalidators.bv.EmailValidator;
|
||||
|
||||
/**
|
||||
* check utils
|
||||
|
|
@ -136,49 +132,6 @@ public class CheckUtils {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* check task node parameter
|
||||
*
|
||||
* @param taskNode TaskNode
|
||||
* @return true if task node parameters are valid, otherwise return false
|
||||
*/
|
||||
public static boolean checkTaskNodeParameters(TaskNode taskNode) {
|
||||
AbstractParameters abstractParameters;
|
||||
String taskType = taskNode.getType();
|
||||
if (taskType == null) {
|
||||
return false;
|
||||
}
|
||||
if (TaskType.DEPENDENT.getDesc().equalsIgnoreCase(taskType)) {
|
||||
abstractParameters = TaskParametersUtils.getParameters(taskType.toUpperCase(), taskNode.getDependence());
|
||||
} else if (TaskType.SWITCH.getDesc().equalsIgnoreCase(taskType)) {
|
||||
abstractParameters = TaskParametersUtils.getParameters(taskType.toUpperCase(), taskNode.getSwitchResult());
|
||||
} else {
|
||||
abstractParameters = TaskParametersUtils.getParameters(taskType.toUpperCase(), taskNode.getTaskParams());
|
||||
}
|
||||
|
||||
if (abstractParameters != null) {
|
||||
return abstractParameters.checkParameters();
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public static boolean checkTaskDefinitionParameters(TaskDefinition taskDefinition) {
|
||||
AbstractParameters abstractParameters;
|
||||
String taskType = taskDefinition.getTaskType();
|
||||
if (TaskType.DEPENDENT.getDesc().equalsIgnoreCase(taskType)) {
|
||||
abstractParameters = TaskParametersUtils.getParameters(taskType.toUpperCase(), taskDefinition.getDependence());
|
||||
} else {
|
||||
abstractParameters = TaskParametersUtils.getParameters(taskType.toUpperCase(), taskDefinition.getTaskParams());
|
||||
}
|
||||
|
||||
if (abstractParameters != null) {
|
||||
return abstractParameters.checkParameters();
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* regex check
|
||||
*
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.api.enums.Status;
|
|||
import org.apache.dolphinscheduler.api.service.SessionService;
|
||||
import org.apache.dolphinscheduler.api.service.UsersService;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ProfileType;
|
||||
import org.apache.dolphinscheduler.dao.DaoConfiguration;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
|
|
@ -44,9 +43,7 @@ import org.springframework.beans.factory.annotation.Autowired;
|
|||
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Profile;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
import org.springframework.test.context.ActiveProfiles;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ import java.util.Date;
|
|||
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
|
|||
|
|
@ -17,13 +17,24 @@
|
|||
|
||||
package org.apache.dolphinscheduler.api.controller;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
|
||||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
import org.apache.dolphinscheduler.api.service.AlertPluginInstanceService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.http.MediaType;
|
||||
|
|
@ -31,13 +42,7 @@ import org.springframework.test.web.servlet.MvcResult;
|
|||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
|
||||
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
/**
|
||||
* alert plugin instance controller test
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import org.apache.dolphinscheduler.common.Constants;
|
|||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.dao.entity.DqRule;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.spi.task.dq.enums.RuleType;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.dp.RuleType;
|
||||
|
||||
import java.text.MessageFormat;
|
||||
import java.util.ArrayList;
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ import org.apache.dolphinscheduler.api.enums.Status;
|
|||
import org.apache.dolphinscheduler.api.service.ProcessInstanceService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
|
|
|||
|
|
@ -31,10 +31,10 @@ import org.apache.dolphinscheduler.api.service.TaskInstanceService;
|
|||
import org.apache.dolphinscheduler.api.utils.PageInfo;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
|
|
|||
|
|
@ -14,10 +14,12 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.api.enums;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
public class ExecuteTypeTest {
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.api.enums.Status;
|
|||
import org.apache.dolphinscheduler.api.service.impl.DataAnalysisServiceImpl;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.CommandType;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.CommandCount;
|
||||
|
|
@ -41,6 +40,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
|
||||
import java.text.MessageFormat;
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@
|
|||
|
||||
package org.apache.dolphinscheduler.api.service;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
import org.apache.dolphinscheduler.api.service.impl.DataSourceServiceImpl;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
|
|
@ -41,6 +40,8 @@ import org.apache.dolphinscheduler.spi.enums.DbConnectType;
|
|||
import org.apache.dolphinscheduler.spi.enums.DbType;
|
||||
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
|
|
@ -370,7 +371,10 @@ public class DataSourceServiceTest {
|
|||
hiveDataSourceParamDTO.setLoginUserKeytabUsername("test2/hdfs-mycluster@ESZ.COM");
|
||||
connectionParam = DataSourceUtils.buildConnectionParams(hiveDataSourceParamDTO);
|
||||
|
||||
expected = "{\"user\":\"test\",\"password\":\"test\",\"address\":\"jdbc:hive2://192.168.9.1:10000\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im\",\"driverClassName\":\"org.apache.hive.jdbc.HiveDriver\",\"validationQuery\":\"select 1\",\"principal\":\"hive/hdfs-mycluster@ESZ.COM\",\"javaSecurityKrb5Conf\":\"/opt/krb5.conf\",\"loginUserKeytabUsername\":\"test2/hdfs-mycluster@ESZ.COM\",\"loginUserKeytabPath\":\"/opt/hdfs.headless.keytab\"}";
|
||||
expected = "{\"user\":\"test\",\"password\":\"test\",\"address\":\"jdbc:hive2://192.168.9.1:10000\",\"database\":\"im\","
|
||||
+ "\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im\",\"driverClassName\":\"org.apache.hive.jdbc.HiveDriver\",\"validationQuery\":\"select 1\","
|
||||
+ "\"principal\":\"hive/hdfs-mycluster@ESZ.COM\",\"javaSecurityKrb5Conf\":\"/opt/krb5.conf\",\"loginUserKeytabUsername\":\"test2/hdfs-mycluster@ESZ.COM\","
|
||||
+ "\"loginUserKeytabPath\":\"/opt/hdfs.headless.keytab\"}";
|
||||
Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam));
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ import org.apache.dolphinscheduler.common.utils.DateUtils;
|
|||
import org.apache.dolphinscheduler.dao.entity.DqExecuteResult;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.dao.mapper.DqExecuteResultMapper;
|
||||
import org.apache.dolphinscheduler.spi.task.dq.enums.DqTaskState;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.dp.DqTaskState;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
|
|
|
|||
|
|
@ -37,13 +37,13 @@ import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.DqRuleExecuteSqlMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.DqRuleInputEntryMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.DqRuleMapper;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ExecuteSqlType;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.dp.InputType;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.dp.OptionSourceType;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.dp.RuleType;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ValueType;
|
||||
import org.apache.dolphinscheduler.spi.enums.DbType;
|
||||
import org.apache.dolphinscheduler.spi.params.base.FormType;
|
||||
import org.apache.dolphinscheduler.spi.task.dq.enums.ExecuteSqlType;
|
||||
import org.apache.dolphinscheduler.spi.task.dq.enums.InputType;
|
||||
import org.apache.dolphinscheduler.spi.task.dq.enums.OptionSourceType;
|
||||
import org.apache.dolphinscheduler.spi.task.dq.enums.RuleType;
|
||||
import org.apache.dolphinscheduler.spi.task.dq.enums.ValueType;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
|
|
|
|||
|
|
@ -17,8 +17,6 @@
|
|||
|
||||
package org.apache.dolphinscheduler.api.service;
|
||||
|
||||
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
|
||||
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
|
@ -30,13 +28,10 @@ import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl;
|
|||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.CommandType;
|
||||
import org.apache.dolphinscheduler.common.enums.ComplementDependentMode;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.enums.Priority;
|
||||
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
||||
import org.apache.dolphinscheduler.common.enums.RunMode;
|
||||
import org.apache.dolphinscheduler.common.model.Server;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.Command;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
|
|
@ -46,6 +41,7 @@ import org.apache.dolphinscheduler.dao.entity.Tenant;
|
|||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
|
|
|||
|
|
@ -187,7 +187,6 @@ public class LoggerServiceTest {
|
|||
loggerService.getLogBytes(loginUser, projectCode, 1);
|
||||
}
|
||||
|
||||
|
||||
@After
|
||||
public void close() {
|
||||
this.loggerService.close();
|
||||
|
|
|
|||
|
|
@ -27,10 +27,7 @@ import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl;
|
|||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.CommandType;
|
||||
import org.apache.dolphinscheduler.common.enums.DependResult;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.common.graph.DAG;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
|
|
@ -54,7 +51,10 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.text.MessageFormat;
|
||||
|
|
@ -118,6 +118,9 @@ public class ProcessInstanceServiceTest {
|
|||
@Mock
|
||||
TaskDefinitionMapper taskDefinitionMapper;
|
||||
|
||||
@Mock
|
||||
TaskPluginManager taskPluginManager;
|
||||
|
||||
private String shellJson = "[{\"name\":\"\",\"preTaskCode\":0,\"preTaskVersion\":0,\"postTaskCode\":123456789,"
|
||||
+ "\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":\"{}\"},{\"name\":\"\",\"preTaskCode\":123456789,"
|
||||
+ "\"preTaskVersion\":1,\"postTaskCode\":123451234,\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":\"{}\"}]";
|
||||
|
|
@ -130,16 +133,16 @@ public class ProcessInstanceServiceTest {
|
|||
+ ":[\"\"],\"failedNode\":[\"\"]},\"dependence\":{}},\"flag\":\"NORMAL\",\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\","
|
||||
+ "\"failRetryTimes\":\"0\",\"failRetryInterval\":\"1\",\"timeoutFlag\":\"CLOSE\",\"timeoutNotifyStrategy\":\"\",\"timeout\":null,\"delayTime\":\"0\"}]";
|
||||
|
||||
private String taskRelationJson = "[{\"name\":\"\",\"preTaskCode\":4254865123776,\"preTaskVersion\":1,\"postTaskCode\":4254862762304,\"postTaskVersion\":1,\"conditionType\":0," +
|
||||
"\"conditionParams\":{}},{\"name\":\"\",\"preTaskCode\":0,\"preTaskVersion\":0,\"postTaskCode\":4254865123776,\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":{}}]";
|
||||
private String taskRelationJson = "[{\"name\":\"\",\"preTaskCode\":4254865123776,\"preTaskVersion\":1,\"postTaskCode\":4254862762304,\"postTaskVersion\":1,\"conditionType\":0,"
|
||||
+ "\"conditionParams\":{}},{\"name\":\"\",\"preTaskCode\":0,\"preTaskVersion\":0,\"postTaskCode\":4254865123776,\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":{}}]";
|
||||
|
||||
private String taskDefinitionJson = "[{\"code\":4254862762304,\"name\":\"test1\",\"version\":1,\"description\":\"\",\"delayTime\":0,\"taskType\":\"SHELL\",\"taskParams\":{\"resourceList\":[]," +
|
||||
"\"localParams\":[],\"rawScript\":\"echo 1\",\"dependence\":{},\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{},\"switchResult\":{}},\"flag\":\"YES\"," +
|
||||
"\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":1,\"timeoutFlag\":\"CLOSE\",\"timeoutNotifyStrategy\":null,\"timeout\":0," +
|
||||
"\"environmentCode\":-1},{\"code\":4254865123776,\"name\":\"test2\",\"version\":1,\"description\":\"\",\"delayTime\":0,\"taskType\":\"SHELL\",\"taskParams\":{\"resourceList\":[]," +
|
||||
"\"localParams\":[],\"rawScript\":\"echo 2\",\"dependence\":{},\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{},\"switchResult\":{}},\"flag\":\"YES\"," +
|
||||
"\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":1,\"timeoutFlag\":\"CLOSE\",\"timeoutNotifyStrategy\":\"WARN\",\"timeout\":0," +
|
||||
"\"environmentCode\":-1}]";
|
||||
private String taskDefinitionJson = "[{\"code\":4254862762304,\"name\":\"test1\",\"version\":1,\"description\":\"\",\"delayTime\":0,\"taskType\":\"SHELL\",\"taskParams\":{\"resourceList\":[],"
|
||||
+ "\"localParams\":[],\"rawScript\":\"echo 1\",\"dependence\":{},\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{},\"switchResult\":{}},\"flag\":\"YES\","
|
||||
+ "\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":1,\"timeoutFlag\":\"CLOSE\",\"timeoutNotifyStrategy\":null,\"timeout\":0,"
|
||||
+ "\"environmentCode\":-1},{\"code\":4254865123776,\"name\":\"test2\",\"version\":1,\"description\":\"\",\"delayTime\":0,\"taskType\":\"SHELL\",\"taskParams\":{\"resourceList\":[],"
|
||||
+ "\"localParams\":[],\"rawScript\":\"echo 2\",\"dependence\":{},\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{},\"switchResult\":{}},\"flag\":\"YES\","
|
||||
+ "\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":1,\"timeoutFlag\":\"CLOSE\",\"timeoutNotifyStrategy\":\"WARN\",\"timeout\":0,"
|
||||
+ "\"environmentCode\":-1}]";
|
||||
|
||||
@Test
|
||||
public void testQueryProcessInstanceList() {
|
||||
|
|
@ -310,7 +313,7 @@ public class ProcessInstanceServiceTest {
|
|||
ProcessInstance processInstance = getProcessInstance();
|
||||
processInstance.setState(ExecutionStatus.SUCCESS);
|
||||
TaskInstance taskInstance = new TaskInstance();
|
||||
taskInstance.setTaskType(TaskType.SHELL.getDesc());
|
||||
taskInstance.setTaskType("SHELL");
|
||||
List<TaskInstance> taskInstanceList = new ArrayList<>();
|
||||
taskInstanceList.add(taskInstance);
|
||||
Result res = new Result();
|
||||
|
|
@ -361,7 +364,7 @@ public class ProcessInstanceServiceTest {
|
|||
|
||||
//task not sub process
|
||||
TaskInstance taskInstance = getTaskInstance();
|
||||
taskInstance.setTaskType(TaskType.HTTP.getDesc());
|
||||
taskInstance.setTaskType("HTTP");
|
||||
taskInstance.setProcessInstanceId(1);
|
||||
putMsg(result, Status.SUCCESS, projectCode);
|
||||
when(processService.findTaskInstanceById(1)).thenReturn(taskInstance);
|
||||
|
|
@ -373,7 +376,7 @@ public class ProcessInstanceServiceTest {
|
|||
|
||||
//sub process not exist
|
||||
TaskInstance subTask = getTaskInstance();
|
||||
subTask.setTaskType(TaskType.SUB_PROCESS.getDesc());
|
||||
subTask.setTaskType("SUB_PROCESS");
|
||||
subTask.setProcessInstanceId(1);
|
||||
putMsg(result, Status.SUCCESS, projectCode);
|
||||
when(processService.findTaskInstanceById(subTask.getId())).thenReturn(subTask);
|
||||
|
|
@ -442,6 +445,7 @@ public class ProcessInstanceServiceTest {
|
|||
List<TaskDefinitionLog> taskDefinitionLogs = JSONUtils.toList(taskDefinitionJson, TaskDefinitionLog.class);
|
||||
when(processDefinitionService.checkProcessNodeList(taskRelationJson, taskDefinitionLogs)).thenReturn(result);
|
||||
putMsg(result, Status.SUCCESS, projectCode);
|
||||
when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true);
|
||||
Map<String, Object> processInstanceFinishRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1,
|
||||
taskRelationJson, taskDefinitionJson,"2020-02-21 00:00:00", true, "", "", 0, "root");
|
||||
Assert.assertEquals(Status.SUCCESS, processInstanceFinishRes.get(Constants.STATUS));
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.api.enums.Status;
|
|||
import org.apache.dolphinscheduler.api.service.impl.ProcessTaskRelationServiceImpl;
|
||||
import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
|
||||
|
|
@ -251,7 +250,7 @@ public class ProcessTaskRelationServiceTest {
|
|||
taskDefinition.setProjectCode(1L);
|
||||
taskDefinition.setCode(1L);
|
||||
taskDefinition.setVersion(1);
|
||||
taskDefinition.setTaskType(TaskType.SHELL.getDesc());
|
||||
taskDefinition.setTaskType("SHELL");
|
||||
return taskDefinition;
|
||||
}
|
||||
|
||||
|
|
@ -476,7 +475,7 @@ public class ProcessTaskRelationServiceTest {
|
|||
Mockito.when(processDefinitionMapper.queryByCode(processDefinitionCode)).thenReturn(getProcessDefinition());
|
||||
Mockito.when(taskDefinitionMapper.queryByCode(taskCode)).thenReturn(getTaskDefinition());
|
||||
TaskDefinition taskDefinition = new TaskDefinition();
|
||||
taskDefinition.setTaskType(TaskType.CONDITIONS.getDesc());
|
||||
taskDefinition.setTaskType("CONDITIONS");
|
||||
Mockito.when(taskDefinitionMapper.queryByCode(taskCode)).thenReturn(taskDefinition);
|
||||
List<ProcessTaskRelation> processTaskRelationList = Lists.newArrayList();
|
||||
ProcessTaskRelation processTaskRelation = new ProcessTaskRelation();
|
||||
|
|
|
|||
|
|
@ -22,9 +22,7 @@ import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl;
|
|||
import org.apache.dolphinscheduler.api.service.impl.TaskDefinitionServiceImpl;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.common.task.shell.ShellParameters;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.Project;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
|
||||
|
|
@ -34,7 +32,9 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
|
||||
import org.apache.dolphinscheduler.plugin.task.shell.ShellParameters;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager;
|
||||
|
||||
import java.text.MessageFormat;
|
||||
import java.util.ArrayList;
|
||||
|
|
@ -74,6 +74,9 @@ public class TaskDefinitionServiceImplTest {
|
|||
@Mock
|
||||
private ProcessTaskRelationMapper processTaskRelationMapper;
|
||||
|
||||
@Mock
|
||||
private TaskPluginManager taskPluginManager;
|
||||
|
||||
@Test
|
||||
public void createTaskDefinition() {
|
||||
long projectCode = 1L;
|
||||
|
|
@ -98,6 +101,7 @@ public class TaskDefinitionServiceImplTest {
|
|||
+ "\"timeoutNotifyStrategy\":0,\"timeout\":0,\"delayTime\":0,\"resourceIds\":\"\"}]";
|
||||
List<TaskDefinitionLog> taskDefinitions = JSONUtils.toList(createTaskDefinitionJson, TaskDefinitionLog.class);
|
||||
Mockito.when(processService.saveTaskDefine(loginUser, projectCode, taskDefinitions, Boolean.TRUE)).thenReturn(1);
|
||||
Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true);
|
||||
Map<String, Object> relation = taskDefinitionService
|
||||
.createTaskDefinition(loginUser, projectCode, createTaskDefinitionJson);
|
||||
Assert.assertEquals(Status.SUCCESS, relation.get(Constants.STATUS));
|
||||
|
|
@ -132,6 +136,7 @@ public class TaskDefinitionServiceImplTest {
|
|||
Mockito.when(taskDefinitionMapper.updateById(Mockito.any(TaskDefinitionLog.class))).thenReturn(1);
|
||||
Mockito.when(taskDefinitionLogMapper.insert(Mockito.any(TaskDefinitionLog.class))).thenReturn(1);
|
||||
Mockito.when(taskDefinitionLogMapper.queryMaxVersionForDefinition(taskCode)).thenReturn(1);
|
||||
Mockito.when(taskPluginManager.checkTaskParameters(Mockito.any())).thenReturn(true);
|
||||
result = taskDefinitionService.updateTaskDefinition(loginUser, projectCode, taskCode, taskDefinitionJson);
|
||||
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
|
||||
}
|
||||
|
|
@ -247,7 +252,7 @@ public class TaskDefinitionServiceImplTest {
|
|||
taskDefinition.setProjectCode(1L);
|
||||
taskDefinition.setCode(1L);
|
||||
taskDefinition.setVersion(1);
|
||||
taskDefinition.setTaskType(TaskType.SHELL.getDesc());
|
||||
taskDefinition.setTaskType("SHELL");
|
||||
return taskDefinition;
|
||||
}
|
||||
|
||||
|
|
@ -313,7 +318,7 @@ public class TaskDefinitionServiceImplTest {
|
|||
taskDefinition.setCode(taskCode);
|
||||
String params = "{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo 1\",\"conditionResult\":{\"successNode\":[\"\"],\"failedNode\":[\"\"]},\"dependence\":{}}";
|
||||
taskDefinition.setTaskParams(params);
|
||||
taskDefinition.setTaskType(TaskType.SHELL.getDesc());
|
||||
taskDefinition.setTaskType("SHELL");
|
||||
Mockito.when(taskDefinitionMapper.queryByCode(taskCode)).thenReturn(taskDefinition);
|
||||
TaskDefinitionLog taskDefinitionLog = new TaskDefinitionLog(taskDefinition);
|
||||
Mockito.when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(taskCode, taskDefinition.getVersion())).thenReturn(taskDefinitionLog);
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl;
|
|||
import org.apache.dolphinscheduler.api.service.impl.TaskInstanceServiceImpl;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
|
|
@ -38,6 +37,7 @@ import org.apache.dolphinscheduler.dao.entity.User;
|
|||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
|
||||
import java.text.MessageFormat;
|
||||
|
|
|
|||
|
|
@ -23,22 +23,6 @@ import static org.junit.Assert.assertTrue;
|
|||
|
||||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ProgramType;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.datax.DataxParameters;
|
||||
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
|
||||
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
|
||||
import org.apache.dolphinscheduler.common.task.http.HttpParameters;
|
||||
import org.apache.dolphinscheduler.common.task.mr.MapReduceParameters;
|
||||
import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters;
|
||||
import org.apache.dolphinscheduler.common.task.python.PythonParameters;
|
||||
import org.apache.dolphinscheduler.common.task.shell.ShellParameters;
|
||||
import org.apache.dolphinscheduler.common.task.spark.SparkParameters;
|
||||
import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
|
||||
import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
|
|
@ -108,142 +92,4 @@ public class CheckUtilsTest {
|
|||
assertTrue(CheckUtils.checkPhone("17362537263"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCheckTaskNodeParameters() {
|
||||
TaskNode taskNode = new TaskNode();
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
taskNode.setType("unKnown");
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
taskNode.setParams("unKnown");
|
||||
taskNode.setType("unKnown");
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
taskNode.setParams("unKnown");
|
||||
taskNode.setType(null);
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
// sub SubProcessParameters
|
||||
SubProcessParameters subProcessParameters = new SubProcessParameters();
|
||||
taskNode.setParams(JSONUtils.toJsonString(subProcessParameters));
|
||||
taskNode.setType(TaskType.SUB_PROCESS.getDesc());
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
subProcessParameters.setProcessDefinitionCode(1234L);
|
||||
taskNode.setParams(JSONUtils.toJsonString(subProcessParameters));
|
||||
taskNode.setType(TaskType.SUB_PROCESS.getDesc());
|
||||
assertTrue(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
// ShellParameters
|
||||
ShellParameters shellParameters = new ShellParameters();
|
||||
taskNode.setParams(JSONUtils.toJsonString(shellParameters));
|
||||
taskNode.setType(TaskType.SHELL.getDesc());
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
shellParameters.setRawScript("");
|
||||
taskNode.setParams(JSONUtils.toJsonString(shellParameters));
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
shellParameters.setRawScript("sss");
|
||||
taskNode.setParams(JSONUtils.toJsonString(shellParameters));
|
||||
assertTrue(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
// ProcedureParameters
|
||||
ProcedureParameters procedureParameters = new ProcedureParameters();
|
||||
taskNode.setParams(JSONUtils.toJsonString(procedureParameters));
|
||||
taskNode.setType(TaskType.PROCEDURE.getDesc());
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
procedureParameters.setDatasource(1);
|
||||
procedureParameters.setType("xx");
|
||||
procedureParameters.setMethod("yy");
|
||||
taskNode.setParams(JSONUtils.toJsonString(procedureParameters));
|
||||
assertTrue(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
// SqlParameters
|
||||
SqlParameters sqlParameters = new SqlParameters();
|
||||
taskNode.setParams(JSONUtils.toJsonString(sqlParameters));
|
||||
taskNode.setType(TaskType.SQL.getDesc());
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
sqlParameters.setDatasource(1);
|
||||
sqlParameters.setType("xx");
|
||||
sqlParameters.setSql("yy");
|
||||
taskNode.setParams(JSONUtils.toJsonString(sqlParameters));
|
||||
assertTrue(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
// MapReduceParameters
|
||||
MapReduceParameters mapreduceParameters = new MapReduceParameters();
|
||||
taskNode.setParams(JSONUtils.toJsonString(mapreduceParameters));
|
||||
taskNode.setType(TaskType.MR.getDesc());
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
ResourceInfo resourceInfoMapreduce = new ResourceInfo();
|
||||
resourceInfoMapreduce.setId(1);
|
||||
resourceInfoMapreduce.setRes("");
|
||||
mapreduceParameters.setMainJar(resourceInfoMapreduce);
|
||||
mapreduceParameters.setProgramType(ProgramType.JAVA);
|
||||
taskNode.setParams(JSONUtils.toJsonString(mapreduceParameters));
|
||||
taskNode.setType(TaskType.MR.getDesc());
|
||||
assertTrue(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
// SparkParameters
|
||||
SparkParameters sparkParameters = new SparkParameters();
|
||||
taskNode.setParams(JSONUtils.toJsonString(sparkParameters));
|
||||
taskNode.setType(TaskType.SPARK.getDesc());
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
sparkParameters.setMainJar(new ResourceInfo());
|
||||
sparkParameters.setProgramType(ProgramType.SCALA);
|
||||
sparkParameters.setSparkVersion("1.1.1");
|
||||
taskNode.setParams(JSONUtils.toJsonString(sparkParameters));
|
||||
assertTrue(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
// PythonParameters
|
||||
PythonParameters pythonParameters = new PythonParameters();
|
||||
taskNode.setParams(JSONUtils.toJsonString(pythonParameters));
|
||||
taskNode.setType(TaskType.PYTHON.getDesc());
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
pythonParameters.setRawScript("ss");
|
||||
taskNode.setParams(JSONUtils.toJsonString(pythonParameters));
|
||||
assertTrue(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
// DependentParameters
|
||||
DependentParameters dependentParameters = new DependentParameters();
|
||||
taskNode.setDependence(JSONUtils.toJsonString(dependentParameters));
|
||||
taskNode.setType(TaskType.DEPENDENT.getDesc());
|
||||
assertTrue(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
// FlinkParameters
|
||||
FlinkParameters flinkParameters = new FlinkParameters();
|
||||
taskNode.setParams(JSONUtils.toJsonString(flinkParameters));
|
||||
taskNode.setType(TaskType.FLINK.getDesc());
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
flinkParameters.setMainJar(new ResourceInfo());
|
||||
flinkParameters.setProgramType(ProgramType.JAVA);
|
||||
taskNode.setParams(JSONUtils.toJsonString(flinkParameters));
|
||||
taskNode.setType(TaskType.FLINK.getDesc());
|
||||
assertTrue(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
// HTTP
|
||||
HttpParameters httpParameters = new HttpParameters();
|
||||
taskNode.setParams(JSONUtils.toJsonString(httpParameters));
|
||||
taskNode.setType(TaskType.HTTP.getDesc());
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
httpParameters.setUrl("httpUrl");
|
||||
taskNode.setParams(JSONUtils.toJsonString(httpParameters));
|
||||
taskNode.setType(TaskType.HTTP.getDesc());
|
||||
assertTrue(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
|
||||
// DataxParameters
|
||||
DataxParameters dataxParameters = new DataxParameters();
|
||||
taskNode.setParams(JSONUtils.toJsonString(dataxParameters));
|
||||
taskNode.setType(TaskType.DATAX.getDesc());
|
||||
assertFalse(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
dataxParameters.setCustomConfig(0);
|
||||
dataxParameters.setDataSource(111);
|
||||
dataxParameters.setDataTarget(333);
|
||||
dataxParameters.setSql(TaskType.SQL.getDesc());
|
||||
dataxParameters.setTargetTable("tar");
|
||||
taskNode.setParams(JSONUtils.toJsonString(dataxParameters));
|
||||
taskNode.setType(TaskType.DATAX.getDesc());
|
||||
assertTrue(CheckUtils.checkTaskNodeParameters(taskNode));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,11 @@
|
|||
<artifactId>dolphinscheduler-spi</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-task-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
|
||||
package org.apache.dolphinscheduler.common;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.lang.SystemUtils;
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@
|
|||
|
||||
package org.apache.dolphinscheduler.common.enums;
|
||||
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
|
||||
import io.netty.channel.Channel;
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@
|
|||
|
||||
package org.apache.dolphinscheduler.common.enums;
|
||||
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
|
||||
/**
|
||||
* type of task state
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,84 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.enums;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.EnumValue;
|
||||
|
||||
/**
|
||||
* task node type
|
||||
*/
|
||||
public enum TaskType {
|
||||
/**
|
||||
* 0 SHELL
|
||||
* 1 SQL
|
||||
* 2 SUB_PROCESS
|
||||
* 3 PROCEDURE
|
||||
* 4 MR
|
||||
* 5 SPARK
|
||||
* 6 PYTHON
|
||||
* 7 DEPENDENT
|
||||
* 8 FLINK
|
||||
* 9 HTTP
|
||||
* 10 DATAX
|
||||
* 11 CONDITIONS
|
||||
* 12 SQOOP
|
||||
* 13 SEATUNNEL
|
||||
* 14 SWITCH
|
||||
* 15 PIGEON
|
||||
* 16 DATA_QUALITY
|
||||
* 17 EMR
|
||||
* 18 BLOCKING
|
||||
*/
|
||||
SHELL(0, "SHELL"),
|
||||
SQL(1, "SQL"),
|
||||
SUB_PROCESS(2, "SUB_PROCESS"),
|
||||
PROCEDURE(3, "PROCEDURE"),
|
||||
MR(4, "MR"),
|
||||
SPARK(5, "SPARK"),
|
||||
PYTHON(6, "PYTHON"),
|
||||
DEPENDENT(7, "DEPENDENT"),
|
||||
FLINK(8, "FLINK"),
|
||||
HTTP(9, "HTTP"),
|
||||
DATAX(10, "DATAX"),
|
||||
CONDITIONS(11, "CONDITIONS"),
|
||||
SQOOP(12, "SQOOP"),
|
||||
SEATUNNEL(13, "SEATUNNEL"),
|
||||
SWITCH(14, "SWITCH"),
|
||||
PIGEON(15, "PIGEON"),
|
||||
DATA_QUALITY(16, "DATA_QUALITY"),
|
||||
EMR(17, "EMR"),
|
||||
BLOCKING(18, "BLOCKING");
|
||||
;
|
||||
|
||||
TaskType(int code, String desc) {
|
||||
this.code = code;
|
||||
this.desc = desc;
|
||||
}
|
||||
|
||||
@EnumValue
|
||||
private final int code;
|
||||
private final String desc;
|
||||
|
||||
public int getCode() {
|
||||
return code;
|
||||
}
|
||||
|
||||
public String getDesc() {
|
||||
return desc;
|
||||
}
|
||||
}
|
||||
|
|
@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.common.model;
|
|||
|
||||
import java.util.Date;
|
||||
|
||||
|
||||
/**
|
||||
* server
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -17,12 +17,15 @@
|
|||
|
||||
package org.apache.dolphinscheduler.common.model;
|
||||
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SWITCH;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.Priority;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter;
|
||||
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
|
||||
|
|
@ -387,11 +390,11 @@ public class TaskNode {
|
|||
}
|
||||
|
||||
public boolean isConditionsTask() {
|
||||
return TaskType.CONDITIONS.getDesc().equalsIgnoreCase(this.getType());
|
||||
return TASK_TYPE_CONDITIONS.equalsIgnoreCase(this.getType());
|
||||
}
|
||||
|
||||
public boolean isSwitchTask() {
|
||||
return TaskType.SWITCH.toString().equalsIgnoreCase(this.getType());
|
||||
return TASK_TYPE_SWITCH.equalsIgnoreCase(this.getType());
|
||||
}
|
||||
|
||||
public List<PreviousTaskNode> getPreTaskNodeList() {
|
||||
|
|
@ -399,7 +402,7 @@ public class TaskNode {
|
|||
}
|
||||
|
||||
public boolean isBlockingTask() {
|
||||
return TaskType.BLOCKING.getDesc().equalsIgnoreCase(this.getType());
|
||||
return TASK_TYPE_BLOCKING.equalsIgnoreCase(this.getType());
|
||||
}
|
||||
|
||||
public void setPreTaskNodeList(List<PreviousTaskNode> preTaskNodeList) {
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.common.model;
|
|||
import java.util.Date;
|
||||
import java.util.Set;
|
||||
|
||||
|
||||
/**
|
||||
* server
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,140 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.process;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.DataType;
|
||||
import org.apache.dolphinscheduler.common.enums.Direct;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Objects;
|
||||
|
||||
public class Property implements Serializable {
|
||||
/**
|
||||
* key
|
||||
*/
|
||||
private String prop;
|
||||
|
||||
/**
|
||||
* input/output
|
||||
*/
|
||||
private Direct direct;
|
||||
|
||||
/**
|
||||
* data type
|
||||
*/
|
||||
private DataType type;
|
||||
|
||||
/**
|
||||
* value
|
||||
*/
|
||||
private String value;
|
||||
|
||||
public Property() {
|
||||
}
|
||||
|
||||
public Property(String prop,Direct direct,DataType type,String value) {
|
||||
this.prop = prop;
|
||||
this.direct = direct;
|
||||
this.type = type;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* getter method
|
||||
*
|
||||
* @return the prop
|
||||
* @see Property#prop
|
||||
*/
|
||||
public String getProp() {
|
||||
return prop;
|
||||
}
|
||||
|
||||
/**
|
||||
* setter method
|
||||
*
|
||||
* @param prop the prop to set
|
||||
* @see Property#prop
|
||||
*/
|
||||
public void setProp(String prop) {
|
||||
this.prop = prop;
|
||||
}
|
||||
|
||||
/**
|
||||
* getter method
|
||||
*
|
||||
* @return the value
|
||||
* @see Property#value
|
||||
*/
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* setter method
|
||||
*
|
||||
* @param value the value to set
|
||||
* @see Property#value
|
||||
*/
|
||||
public void setValue(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public Direct getDirect() {
|
||||
return direct;
|
||||
}
|
||||
|
||||
public void setDirect(Direct direct) {
|
||||
this.direct = direct;
|
||||
}
|
||||
|
||||
public DataType getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public void setType(DataType type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
Property property = (Property) o;
|
||||
return Objects.equals(prop, property.prop)
|
||||
&& Objects.equals(value, property.value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(prop, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Property{"
|
||||
+ "prop='" + prop + '\''
|
||||
+ ", direct=" + direct
|
||||
+ ", type=" + type
|
||||
+ ", value='" + value + '\''
|
||||
+ '}';
|
||||
}
|
||||
}
|
||||
|
|
@ -17,9 +17,6 @@
|
|||
|
||||
package org.apache.dolphinscheduler.common.shell;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
|
@ -31,6 +28,9 @@ import java.util.TimerTask;
|
|||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* A base class for running a Unix command.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -1,181 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.Direct;
|
||||
import org.apache.dolphinscheduler.common.process.Property;
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
|
||||
/**
|
||||
* job params related class
|
||||
*/
|
||||
public abstract class AbstractParameters implements IParameters {
|
||||
|
||||
@Override
|
||||
public abstract boolean checkParameters();
|
||||
|
||||
@Override
|
||||
public abstract List<ResourceInfo> getResourceFilesList();
|
||||
|
||||
/**
|
||||
* local parameters
|
||||
*/
|
||||
public List<Property> localParams;
|
||||
|
||||
/**
|
||||
* var pool
|
||||
*/
|
||||
public List<Property> varPool;
|
||||
|
||||
/**
|
||||
* get local parameters list
|
||||
*
|
||||
* @return Property list
|
||||
*/
|
||||
public List<Property> getLocalParams() {
|
||||
return localParams;
|
||||
}
|
||||
|
||||
public void setLocalParams(List<Property> localParams) {
|
||||
this.localParams = localParams;
|
||||
}
|
||||
|
||||
/**
|
||||
* get local parameters map
|
||||
*
|
||||
* @return parameters map
|
||||
*/
|
||||
public Map<String, Property> getLocalParametersMap() {
|
||||
if (localParams != null) {
|
||||
Map<String, Property> localParametersMaps = new LinkedHashMap<>();
|
||||
|
||||
for (Property property : localParams) {
|
||||
localParametersMaps.put(property.getProp(), property);
|
||||
}
|
||||
return localParametersMaps;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* get varPool map
|
||||
*
|
||||
* @return parameters map
|
||||
*/
|
||||
public Map<String, Property> getVarPoolMap() {
|
||||
if (varPool != null) {
|
||||
Map<String, Property> varPoolMap = new LinkedHashMap<>();
|
||||
for (Property property : varPool) {
|
||||
varPoolMap.put(property.getProp(), property);
|
||||
}
|
||||
return varPoolMap;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public List<Property> getVarPool() {
|
||||
return varPool;
|
||||
}
|
||||
|
||||
public void setVarPool(String varPool) {
|
||||
if (StringUtils.isEmpty(varPool)) {
|
||||
this.varPool = new ArrayList<>();
|
||||
} else {
|
||||
this.varPool = JSONUtils.toList(varPool, Property.class);
|
||||
}
|
||||
}
|
||||
|
||||
public void dealOutParam(String result) {
|
||||
if (CollectionUtils.isEmpty(localParams)) {
|
||||
return;
|
||||
}
|
||||
List<Property> outProperty = getOutProperty(localParams);
|
||||
if (CollectionUtils.isEmpty(outProperty)) {
|
||||
return;
|
||||
}
|
||||
if (StringUtils.isEmpty(result)) {
|
||||
varPool.addAll(outProperty);
|
||||
return;
|
||||
}
|
||||
Map<String, String> taskResult = getMapByString(result);
|
||||
if (taskResult == null || taskResult.size() == 0) {
|
||||
return;
|
||||
}
|
||||
for (Property info : outProperty) {
|
||||
info.setValue(taskResult.get(info.getProp()));
|
||||
varPool.add(info);
|
||||
}
|
||||
}
|
||||
|
||||
public List<Property> getOutProperty(List<Property> params) {
|
||||
if (CollectionUtils.isEmpty(params)) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
List<Property> result = new ArrayList<>();
|
||||
for (Property info : params) {
|
||||
if (info.getDirect() == Direct.OUT) {
|
||||
result.add(info);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public List<Map<String, String>> getListMapByString(String json) {
|
||||
List<Map<String, String>> allParams = new ArrayList<>();
|
||||
ArrayNode paramsByJson = JSONUtils.parseArray(json);
|
||||
Iterator<JsonNode> listIterator = paramsByJson.iterator();
|
||||
while (listIterator.hasNext()) {
|
||||
Map<String, String> param = JSONUtils.parseObject(listIterator.next().toString(), new TypeReference<Map<String, String>>() {});
|
||||
allParams.add(param);
|
||||
}
|
||||
return allParams;
|
||||
}
|
||||
|
||||
/**
|
||||
* shell's result format is key=value$VarPool$key=value$VarPool$
|
||||
* @param result
|
||||
* @return
|
||||
*/
|
||||
public static Map<String, String> getMapByString(String result) {
|
||||
String[] formatResult = result.split("\\$VarPool\\$");
|
||||
Map<String, String> format = new HashMap<>();
|
||||
for (String info : formatResult) {
|
||||
if (!StringUtils.isEmpty(info) && info.contains("=")) {
|
||||
String[] keyValue = info.split("=");
|
||||
format.put(keyValue[0], keyValue[1]);
|
||||
}
|
||||
}
|
||||
return format;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,252 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.datax;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* DataX parameter
|
||||
*/
|
||||
public class DataxParameters extends AbstractParameters {
|
||||
|
||||
/**
|
||||
* if custom json config,eg 0, 1
|
||||
*/
|
||||
private int customConfig;
|
||||
|
||||
/**
|
||||
* if customConfig eq 1 ,then json is usable
|
||||
*/
|
||||
private String json;
|
||||
|
||||
/**
|
||||
* data source type,eg MYSQL, POSTGRES ...
|
||||
*/
|
||||
private String dsType;
|
||||
|
||||
/**
|
||||
* datasource id
|
||||
*/
|
||||
private int dataSource;
|
||||
|
||||
/**
|
||||
* data target type,eg MYSQL, POSTGRES ...
|
||||
*/
|
||||
private String dtType;
|
||||
|
||||
/**
|
||||
* datatarget id
|
||||
*/
|
||||
private int dataTarget;
|
||||
|
||||
/**
|
||||
* sql
|
||||
*/
|
||||
private String sql;
|
||||
|
||||
/**
|
||||
* target table
|
||||
*/
|
||||
private String targetTable;
|
||||
|
||||
/**
|
||||
* Pre Statements
|
||||
*/
|
||||
private List<String> preStatements;
|
||||
|
||||
/**
|
||||
* Post Statements
|
||||
*/
|
||||
private List<String> postStatements;
|
||||
|
||||
/**
|
||||
* speed byte num
|
||||
*/
|
||||
private int jobSpeedByte;
|
||||
|
||||
/**
|
||||
* speed record count
|
||||
*/
|
||||
private int jobSpeedRecord;
|
||||
|
||||
/**
|
||||
* Xms memory
|
||||
*/
|
||||
private int xms;
|
||||
|
||||
/**
|
||||
* Xmx memory
|
||||
*/
|
||||
private int xmx;
|
||||
|
||||
public int getCustomConfig() {
|
||||
return customConfig;
|
||||
}
|
||||
|
||||
public void setCustomConfig(int customConfig) {
|
||||
this.customConfig = customConfig;
|
||||
}
|
||||
|
||||
public String getJson() {
|
||||
return json;
|
||||
}
|
||||
|
||||
public void setJson(String json) {
|
||||
this.json = json;
|
||||
}
|
||||
|
||||
public String getDsType() {
|
||||
return dsType;
|
||||
}
|
||||
|
||||
public void setDsType(String dsType) {
|
||||
this.dsType = dsType;
|
||||
}
|
||||
|
||||
public int getDataSource() {
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
public void setDataSource(int dataSource) {
|
||||
this.dataSource = dataSource;
|
||||
}
|
||||
|
||||
public String getDtType() {
|
||||
return dtType;
|
||||
}
|
||||
|
||||
public void setDtType(String dtType) {
|
||||
this.dtType = dtType;
|
||||
}
|
||||
|
||||
public int getDataTarget() {
|
||||
return dataTarget;
|
||||
}
|
||||
|
||||
public void setDataTarget(int dataTarget) {
|
||||
this.dataTarget = dataTarget;
|
||||
}
|
||||
|
||||
public String getSql() {
|
||||
return sql;
|
||||
}
|
||||
|
||||
public void setSql(String sql) {
|
||||
this.sql = sql;
|
||||
}
|
||||
|
||||
public String getTargetTable() {
|
||||
return targetTable;
|
||||
}
|
||||
|
||||
public void setTargetTable(String targetTable) {
|
||||
this.targetTable = targetTable;
|
||||
}
|
||||
|
||||
public List<String> getPreStatements() {
|
||||
return preStatements;
|
||||
}
|
||||
|
||||
public void setPreStatements(List<String> preStatements) {
|
||||
this.preStatements = preStatements;
|
||||
}
|
||||
|
||||
public List<String> getPostStatements() {
|
||||
return postStatements;
|
||||
}
|
||||
|
||||
public void setPostStatements(List<String> postStatements) {
|
||||
this.postStatements = postStatements;
|
||||
}
|
||||
|
||||
public int getJobSpeedByte() {
|
||||
return jobSpeedByte;
|
||||
}
|
||||
|
||||
public void setJobSpeedByte(int jobSpeedByte) {
|
||||
this.jobSpeedByte = jobSpeedByte;
|
||||
}
|
||||
|
||||
public int getJobSpeedRecord() {
|
||||
return jobSpeedRecord;
|
||||
}
|
||||
|
||||
public void setJobSpeedRecord(int jobSpeedRecord) {
|
||||
this.jobSpeedRecord = jobSpeedRecord;
|
||||
}
|
||||
|
||||
public int getXms() {
|
||||
return xms;
|
||||
}
|
||||
|
||||
public void setXms(int xms) {
|
||||
this.xms = xms;
|
||||
}
|
||||
|
||||
public int getXmx() {
|
||||
return xmx;
|
||||
}
|
||||
|
||||
public void setXmx(int xmx) {
|
||||
this.xmx = xmx;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkParameters() {
|
||||
if (customConfig == Flag.NO.ordinal()) {
|
||||
return dataSource != 0
|
||||
&& dataTarget != 0
|
||||
&& !StringUtils.isEmpty(sql)
|
||||
&& !StringUtils.isEmpty(targetTable);
|
||||
} else {
|
||||
return !StringUtils.isEmpty(json);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceInfo> getResourceFilesList() {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "DataxParameters{"
|
||||
+ "customConfig=" + customConfig
|
||||
+ ", json='" + json + '\''
|
||||
+ ", dsType='" + dsType + '\''
|
||||
+ ", dataSource=" + dataSource
|
||||
+ ", dtType='" + dtType + '\''
|
||||
+ ", dataTarget=" + dataTarget
|
||||
+ ", sql='" + sql + '\''
|
||||
+ ", targetTable='" + targetTable + '\''
|
||||
+ ", preStatements=" + preStatements
|
||||
+ ", postStatements=" + postStatements
|
||||
+ ", jobSpeedByte=" + jobSpeedByte
|
||||
+ ", jobSpeedRecord=" + jobSpeedRecord
|
||||
+ ", xms=" + xms
|
||||
+ ", xmx=" + xmx
|
||||
+ '}';
|
||||
}
|
||||
}
|
||||
|
|
@ -1,103 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.dq;
|
||||
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
import org.apache.dolphinscheduler.common.task.spark.SparkParameters;
|
||||
|
||||
import org.apache.commons.collections.MapUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* DataQualityParameters
|
||||
*/
|
||||
public class DataQualityParameters extends AbstractParameters {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DataQualityParameters.class);
|
||||
|
||||
/**
|
||||
* rule id
|
||||
*/
|
||||
private int ruleId;
|
||||
/**
|
||||
* rule input entry value map
|
||||
*/
|
||||
private Map<String,String> ruleInputParameter;
|
||||
/**
|
||||
* spark parameters
|
||||
*/
|
||||
private SparkParameters sparkParameters;
|
||||
|
||||
public int getRuleId() {
|
||||
return ruleId;
|
||||
}
|
||||
|
||||
public void setRuleId(int ruleId) {
|
||||
this.ruleId = ruleId;
|
||||
}
|
||||
|
||||
public Map<String, String> getRuleInputParameter() {
|
||||
return ruleInputParameter;
|
||||
}
|
||||
|
||||
public void setRuleInputParameter(Map<String, String> ruleInputParameter) {
|
||||
this.ruleInputParameter = ruleInputParameter;
|
||||
}
|
||||
|
||||
/**
|
||||
* In this function ,we need more detailed check every parameter,
|
||||
* if the parameter is non-conformant will return false
|
||||
* @return boolean result
|
||||
*/
|
||||
@Override
|
||||
public boolean checkParameters() {
|
||||
|
||||
if (ruleId == 0) {
|
||||
logger.error("rule id is null");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (MapUtils.isEmpty(ruleInputParameter)) {
|
||||
logger.error("rule input parameter is empty");
|
||||
return false;
|
||||
}
|
||||
|
||||
return sparkParameters != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceInfo> getResourceFilesList() {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
public SparkParameters getSparkParameters() {
|
||||
return sparkParameters;
|
||||
}
|
||||
|
||||
public void setSparkParameters(SparkParameters sparkParameters) {
|
||||
this.sparkParameters = sparkParameters;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.emr;
|
||||
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
public class EmrParameters extends AbstractParameters {
|
||||
/**
|
||||
* job flow define in json format
|
||||
*/
|
||||
private String jobFlowDefineJson;
|
||||
|
||||
@Override
|
||||
public boolean checkParameters() {
|
||||
return StringUtils.isNotEmpty(jobFlowDefineJson);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<ResourceInfo> getResourceFilesList() {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
|
||||
public String getJobFlowDefineJson() {
|
||||
return jobFlowDefineJson;
|
||||
}
|
||||
|
||||
public void setJobFlowDefineJson(String jobFlowDefineJson) {
|
||||
this.jobFlowDefineJson = jobFlowDefineJson;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "EmrParameters{" +
|
||||
"jobFlowDefineJson='" + jobFlowDefineJson + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
@ -1,241 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.flink;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.ProgramType;
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* flink parameters
|
||||
*/
|
||||
public class FlinkParameters extends AbstractParameters {
|
||||
|
||||
/**
|
||||
* major jar
|
||||
*/
|
||||
private ResourceInfo mainJar;
|
||||
|
||||
/**
|
||||
* major class
|
||||
*/
|
||||
private String mainClass;
|
||||
|
||||
/**
|
||||
* deploy mode yarn-cluster yarn-local
|
||||
*/
|
||||
private String deployMode;
|
||||
|
||||
/**
|
||||
* arguments
|
||||
*/
|
||||
private String mainArgs;
|
||||
|
||||
/**
|
||||
* slot count
|
||||
*/
|
||||
private int slot;
|
||||
|
||||
/**
|
||||
* parallelism
|
||||
*/
|
||||
private int parallelism;
|
||||
|
||||
/**
|
||||
* yarn application name
|
||||
*/
|
||||
private String appName;
|
||||
|
||||
/**
|
||||
* taskManager count
|
||||
*/
|
||||
private int taskManager;
|
||||
|
||||
/**
|
||||
* job manager memory
|
||||
*/
|
||||
private String jobManagerMemory;
|
||||
|
||||
/**
|
||||
* task manager memory
|
||||
*/
|
||||
private String taskManagerMemory;
|
||||
|
||||
/**
|
||||
* resource list
|
||||
*/
|
||||
private List<ResourceInfo> resourceList = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* The YARN queue to submit to
|
||||
*/
|
||||
private String queue;
|
||||
|
||||
/**
|
||||
* other arguments
|
||||
*/
|
||||
private String others;
|
||||
|
||||
/**
|
||||
* flink version
|
||||
*/
|
||||
private String flinkVersion;
|
||||
|
||||
/**
|
||||
* program type
|
||||
* 0 JAVA,1 SCALA,2 PYTHON
|
||||
*/
|
||||
private ProgramType programType;
|
||||
|
||||
public ResourceInfo getMainJar() {
|
||||
return mainJar;
|
||||
}
|
||||
|
||||
public void setMainJar(ResourceInfo mainJar) {
|
||||
this.mainJar = mainJar;
|
||||
}
|
||||
|
||||
public String getMainClass() {
|
||||
return mainClass;
|
||||
}
|
||||
|
||||
public void setMainClass(String mainClass) {
|
||||
this.mainClass = mainClass;
|
||||
}
|
||||
|
||||
public String getDeployMode() {
|
||||
return deployMode;
|
||||
}
|
||||
|
||||
public void setDeployMode(String deployMode) {
|
||||
this.deployMode = deployMode;
|
||||
}
|
||||
|
||||
public String getMainArgs() {
|
||||
return mainArgs;
|
||||
}
|
||||
|
||||
public void setMainArgs(String mainArgs) {
|
||||
this.mainArgs = mainArgs;
|
||||
}
|
||||
|
||||
public int getSlot() {
|
||||
return slot;
|
||||
}
|
||||
|
||||
public void setSlot(int slot) {
|
||||
this.slot = slot;
|
||||
}
|
||||
|
||||
public int getParallelism() {
|
||||
return parallelism;
|
||||
}
|
||||
|
||||
public void setParallelism(int parallelism) {
|
||||
this.parallelism = parallelism;
|
||||
}
|
||||
|
||||
public String getAppName() {
|
||||
return appName;
|
||||
}
|
||||
|
||||
public void setAppName(String appName) {
|
||||
this.appName = appName;
|
||||
}
|
||||
|
||||
public int getTaskManager() {
|
||||
return taskManager;
|
||||
}
|
||||
|
||||
public void setTaskManager(int taskManager) {
|
||||
this.taskManager = taskManager;
|
||||
}
|
||||
|
||||
public String getJobManagerMemory() {
|
||||
return jobManagerMemory;
|
||||
}
|
||||
|
||||
public void setJobManagerMemory(String jobManagerMemory) {
|
||||
this.jobManagerMemory = jobManagerMemory;
|
||||
}
|
||||
|
||||
public String getTaskManagerMemory() {
|
||||
return taskManagerMemory;
|
||||
}
|
||||
|
||||
public void setTaskManagerMemory(String taskManagerMemory) {
|
||||
this.taskManagerMemory = taskManagerMemory;
|
||||
}
|
||||
|
||||
public String getQueue() {
|
||||
return queue;
|
||||
}
|
||||
|
||||
public void setQueue(String queue) {
|
||||
this.queue = queue;
|
||||
}
|
||||
|
||||
public List<ResourceInfo> getResourceList() {
|
||||
return resourceList;
|
||||
}
|
||||
|
||||
public void setResourceList(List<ResourceInfo> resourceList) {
|
||||
this.resourceList = resourceList;
|
||||
}
|
||||
|
||||
public String getOthers() {
|
||||
return others;
|
||||
}
|
||||
|
||||
public void setOthers(String others) {
|
||||
this.others = others;
|
||||
}
|
||||
|
||||
public ProgramType getProgramType() {
|
||||
return programType;
|
||||
}
|
||||
|
||||
public void setProgramType(ProgramType programType) {
|
||||
this.programType = programType;
|
||||
}
|
||||
|
||||
public String getFlinkVersion() {
|
||||
return flinkVersion;
|
||||
}
|
||||
|
||||
public void setFlinkVersion(String flinkVersion) {
|
||||
this.flinkVersion = flinkVersion;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkParameters() {
|
||||
return mainJar != null && programType != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceInfo> getResourceFilesList() {
|
||||
if (mainJar != null && !resourceList.contains(mainJar)) {
|
||||
resourceList.add(mainJar);
|
||||
}
|
||||
return resourceList;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,138 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.http;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.HttpCheckCondition;
|
||||
import org.apache.dolphinscheduler.common.enums.HttpMethod;
|
||||
import org.apache.dolphinscheduler.common.process.HttpProperty;
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* http parameter
|
||||
*/
|
||||
public class HttpParameters extends AbstractParameters {
|
||||
/**
|
||||
* url
|
||||
*/
|
||||
private String url;
|
||||
|
||||
/**
|
||||
* httpMethod
|
||||
*/
|
||||
private HttpMethod httpMethod;
|
||||
|
||||
/**
|
||||
* http params
|
||||
*/
|
||||
private List<HttpProperty> httpParams;
|
||||
|
||||
/**
|
||||
* httpCheckCondition
|
||||
*/
|
||||
private HttpCheckCondition httpCheckCondition = HttpCheckCondition.STATUS_CODE_DEFAULT;
|
||||
|
||||
/**
|
||||
* condition
|
||||
*/
|
||||
private String condition;
|
||||
|
||||
|
||||
/**
|
||||
* Connect Timeout
|
||||
* Unit: ms
|
||||
*/
|
||||
private int connectTimeout;
|
||||
|
||||
/**
|
||||
* Socket Timeout
|
||||
* Unit: ms
|
||||
*/
|
||||
private int socketTimeout;
|
||||
|
||||
@Override
|
||||
public boolean checkParameters() {
|
||||
return !StringUtils.isEmpty(url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceInfo> getResourceFilesList() {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
return url;
|
||||
}
|
||||
|
||||
public void setUrl(String url) {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
public HttpMethod getHttpMethod() {
|
||||
return httpMethod;
|
||||
}
|
||||
|
||||
public void setHttpMethod(HttpMethod httpMethod) {
|
||||
this.httpMethod = httpMethod;
|
||||
}
|
||||
|
||||
public List<HttpProperty> getHttpParams() {
|
||||
return httpParams;
|
||||
}
|
||||
|
||||
public void setHttpParams(List<HttpProperty> httpParams) {
|
||||
this.httpParams = httpParams;
|
||||
}
|
||||
|
||||
public HttpCheckCondition getHttpCheckCondition() {
|
||||
return httpCheckCondition;
|
||||
}
|
||||
|
||||
public void setHttpCheckCondition(HttpCheckCondition httpCheckCondition) {
|
||||
this.httpCheckCondition = httpCheckCondition;
|
||||
}
|
||||
|
||||
public String getCondition() {
|
||||
return condition;
|
||||
}
|
||||
|
||||
public void setCondition(String condition) {
|
||||
this.condition = condition;
|
||||
}
|
||||
|
||||
public int getConnectTimeout() {
|
||||
return connectTimeout;
|
||||
}
|
||||
|
||||
public void setConnectTimeout(int connectTimeout) {
|
||||
this.connectTimeout = connectTimeout;
|
||||
}
|
||||
|
||||
public int getSocketTimeout() {
|
||||
return socketTimeout;
|
||||
}
|
||||
|
||||
public void setSocketTimeout(int socketTimeout) {
|
||||
this.socketTimeout = socketTimeout;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,160 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.mr;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.ProgramType;
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* mapreduce parameters
|
||||
*/
|
||||
public class MapReduceParameters extends AbstractParameters {
|
||||
|
||||
/**
|
||||
* major jar
|
||||
*/
|
||||
private ResourceInfo mainJar;
|
||||
|
||||
/**
|
||||
* major class
|
||||
*/
|
||||
private String mainClass;
|
||||
|
||||
/**
|
||||
* arguments
|
||||
*/
|
||||
private String mainArgs;
|
||||
|
||||
/**
|
||||
* other arguments
|
||||
*/
|
||||
private String others;
|
||||
|
||||
/**
|
||||
* app name
|
||||
*/
|
||||
private String appName;
|
||||
|
||||
/**
|
||||
* queue
|
||||
*/
|
||||
private String queue;
|
||||
|
||||
/**
|
||||
* resource list
|
||||
*/
|
||||
private List<ResourceInfo> resourceList = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* program type
|
||||
* 0 JAVA,1 SCALA,2 PYTHON
|
||||
*/
|
||||
private ProgramType programType;
|
||||
|
||||
public String getMainClass() {
|
||||
return mainClass;
|
||||
}
|
||||
|
||||
public void setMainClass(String mainClass) {
|
||||
this.mainClass = mainClass;
|
||||
}
|
||||
|
||||
public String getMainArgs() {
|
||||
return mainArgs;
|
||||
}
|
||||
|
||||
public void setMainArgs(String mainArgs) {
|
||||
this.mainArgs = mainArgs;
|
||||
}
|
||||
|
||||
public String getOthers() {
|
||||
return others;
|
||||
}
|
||||
|
||||
public void setOthers(String others) {
|
||||
this.others = others;
|
||||
}
|
||||
|
||||
public String getAppName() {
|
||||
return appName;
|
||||
}
|
||||
|
||||
public void setAppName(String appName) {
|
||||
this.appName = appName;
|
||||
}
|
||||
|
||||
public String getQueue() {
|
||||
return queue;
|
||||
}
|
||||
|
||||
public void setQueue(String queue) {
|
||||
this.queue = queue;
|
||||
}
|
||||
|
||||
public List<ResourceInfo> getResourceList() {
|
||||
return this.resourceList;
|
||||
}
|
||||
|
||||
public void setResourceList(List<ResourceInfo> resourceList) {
|
||||
this.resourceList = resourceList;
|
||||
}
|
||||
|
||||
public void setMainJar(ResourceInfo mainJar) {
|
||||
this.mainJar = mainJar;
|
||||
}
|
||||
|
||||
public ResourceInfo getMainJar() {
|
||||
return mainJar;
|
||||
}
|
||||
|
||||
public ProgramType getProgramType() {
|
||||
return programType;
|
||||
}
|
||||
|
||||
public void setProgramType(ProgramType programType) {
|
||||
this.programType = programType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkParameters() {
|
||||
return this.mainJar != null && this.programType != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceInfo> getResourceFilesList() {
|
||||
if (mainJar != null && !resourceList.contains(mainJar)) {
|
||||
resourceList.add(mainJar);
|
||||
}
|
||||
|
||||
return resourceList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "mainJar= " + mainJar
|
||||
+ "mainClass=" + mainClass
|
||||
+ "mainArgs=" + mainArgs
|
||||
+ "queue=" + queue
|
||||
+ "other mainArgs=" + others
|
||||
;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.procedure;
|
||||
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* procedure parameter
|
||||
*/
|
||||
public class ProcedureParameters extends AbstractParameters {
|
||||
|
||||
/**
|
||||
* data source type,eg MYSQL, POSTGRES, HIVE ...
|
||||
*/
|
||||
private String type;
|
||||
|
||||
/**
|
||||
* data source id
|
||||
*/
|
||||
private int datasource;
|
||||
|
||||
/**
|
||||
* procedure name
|
||||
*/
|
||||
private String method;
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public void setType(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public int getDatasource() {
|
||||
return datasource;
|
||||
}
|
||||
|
||||
public void setDatasource(int datasource) {
|
||||
this.datasource = datasource;
|
||||
}
|
||||
|
||||
public String getMethod() {
|
||||
return method;
|
||||
}
|
||||
|
||||
public void setMethod(String method) {
|
||||
this.method = method;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkParameters() {
|
||||
return datasource != 0 && !StringUtils.isEmpty(type) && !StringUtils.isEmpty(method);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceInfo> getResourceFilesList() {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "ProcessdureParam{"
|
||||
+ "type='" + type + '\''
|
||||
+ ", datasource=" + datasource
|
||||
+ ", method='" + method + '\''
|
||||
+ '}';
|
||||
}
|
||||
}
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.python;
|
||||
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class PythonParameters extends AbstractParameters {
|
||||
/**
|
||||
* origin python script
|
||||
*/
|
||||
private String rawScript;
|
||||
|
||||
/**
|
||||
* resource list
|
||||
*/
|
||||
private List<ResourceInfo> resourceList;
|
||||
|
||||
public String getRawScript() {
|
||||
return rawScript;
|
||||
}
|
||||
|
||||
public void setRawScript(String rawScript) {
|
||||
this.rawScript = rawScript;
|
||||
}
|
||||
|
||||
public List<ResourceInfo> getResourceList() {
|
||||
return resourceList;
|
||||
}
|
||||
|
||||
public void setResourceList(List<ResourceInfo> resourceList) {
|
||||
this.resourceList = resourceList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkParameters() {
|
||||
return rawScript != null && !rawScript.isEmpty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceInfo> getResourceFilesList() {
|
||||
return this.resourceList;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.shell;
|
||||
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* shell parameters
|
||||
*/
|
||||
public class ShellParameters extends AbstractParameters {
|
||||
/**
|
||||
* shell script
|
||||
*/
|
||||
private String rawScript;
|
||||
|
||||
/**
|
||||
* resource list
|
||||
*/
|
||||
private List<ResourceInfo> resourceList;
|
||||
|
||||
public String getRawScript() {
|
||||
return rawScript;
|
||||
}
|
||||
|
||||
public void setRawScript(String rawScript) {
|
||||
this.rawScript = rawScript;
|
||||
}
|
||||
|
||||
public List<ResourceInfo> getResourceList() {
|
||||
return resourceList;
|
||||
}
|
||||
|
||||
public void setResourceList(List<ResourceInfo> resourceList) {
|
||||
this.resourceList = resourceList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkParameters() {
|
||||
return rawScript != null && !rawScript.isEmpty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceInfo> getResourceFilesList() {
|
||||
return resourceList;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,242 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.spark;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.ProgramType;
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* spark parameters
|
||||
*/
|
||||
public class SparkParameters extends AbstractParameters {
|
||||
|
||||
/**
|
||||
* main jar
|
||||
*/
|
||||
private ResourceInfo mainJar;
|
||||
|
||||
/**
|
||||
* main class
|
||||
*/
|
||||
private String mainClass;
|
||||
|
||||
/**
|
||||
* deploy mode
|
||||
*/
|
||||
private String deployMode;
|
||||
|
||||
/**
|
||||
* arguments
|
||||
*/
|
||||
private String mainArgs;
|
||||
|
||||
/**
|
||||
* driver-cores Number of cores used by the driver, only in cluster mode
|
||||
*/
|
||||
private int driverCores;
|
||||
|
||||
/**
|
||||
* driver-memory Memory for driver
|
||||
*/
|
||||
|
||||
private String driverMemory;
|
||||
|
||||
/**
|
||||
* num-executors Number of executors to launch
|
||||
*/
|
||||
private int numExecutors;
|
||||
|
||||
/**
|
||||
* executor-cores Number of cores per executor
|
||||
*/
|
||||
private int executorCores;
|
||||
|
||||
/**
|
||||
* Memory per executor
|
||||
*/
|
||||
private String executorMemory;
|
||||
|
||||
/**
|
||||
* app name
|
||||
*/
|
||||
private String appName;
|
||||
|
||||
/**
|
||||
* The YARN queue to submit to
|
||||
*/
|
||||
private String queue;
|
||||
|
||||
/**
|
||||
* other arguments
|
||||
*/
|
||||
private String others;
|
||||
|
||||
/**
|
||||
* program type
|
||||
* 0 JAVA,1 SCALA,2 PYTHON
|
||||
*/
|
||||
private ProgramType programType;
|
||||
|
||||
/**
|
||||
* spark version
|
||||
*/
|
||||
private String sparkVersion;
|
||||
|
||||
/**
|
||||
* resource list
|
||||
*/
|
||||
private List<ResourceInfo> resourceList = new ArrayList<>();
|
||||
|
||||
public ResourceInfo getMainJar() {
|
||||
return mainJar;
|
||||
}
|
||||
|
||||
public void setMainJar(ResourceInfo mainJar) {
|
||||
this.mainJar = mainJar;
|
||||
}
|
||||
|
||||
public String getMainClass() {
|
||||
return mainClass;
|
||||
}
|
||||
|
||||
public void setMainClass(String mainClass) {
|
||||
this.mainClass = mainClass;
|
||||
}
|
||||
|
||||
public String getDeployMode() {
|
||||
return deployMode;
|
||||
}
|
||||
|
||||
public void setDeployMode(String deployMode) {
|
||||
this.deployMode = deployMode;
|
||||
}
|
||||
|
||||
public String getMainArgs() {
|
||||
return mainArgs;
|
||||
}
|
||||
|
||||
public void setMainArgs(String mainArgs) {
|
||||
this.mainArgs = mainArgs;
|
||||
}
|
||||
|
||||
public int getDriverCores() {
|
||||
return driverCores;
|
||||
}
|
||||
|
||||
public void setDriverCores(int driverCores) {
|
||||
this.driverCores = driverCores;
|
||||
}
|
||||
|
||||
public String getDriverMemory() {
|
||||
return driverMemory;
|
||||
}
|
||||
|
||||
public void setDriverMemory(String driverMemory) {
|
||||
this.driverMemory = driverMemory;
|
||||
}
|
||||
|
||||
public int getNumExecutors() {
|
||||
return numExecutors;
|
||||
}
|
||||
|
||||
public void setNumExecutors(int numExecutors) {
|
||||
this.numExecutors = numExecutors;
|
||||
}
|
||||
|
||||
public int getExecutorCores() {
|
||||
return executorCores;
|
||||
}
|
||||
|
||||
public void setExecutorCores(int executorCores) {
|
||||
this.executorCores = executorCores;
|
||||
}
|
||||
|
||||
public String getExecutorMemory() {
|
||||
return executorMemory;
|
||||
}
|
||||
|
||||
public void setExecutorMemory(String executorMemory) {
|
||||
this.executorMemory = executorMemory;
|
||||
}
|
||||
|
||||
public String getAppName() {
|
||||
return appName;
|
||||
}
|
||||
|
||||
public void setAppName(String appName) {
|
||||
this.appName = appName;
|
||||
}
|
||||
|
||||
public String getQueue() {
|
||||
return queue;
|
||||
}
|
||||
|
||||
public void setQueue(String queue) {
|
||||
this.queue = queue;
|
||||
}
|
||||
|
||||
public String getOthers() {
|
||||
return others;
|
||||
}
|
||||
|
||||
public void setOthers(String others) {
|
||||
this.others = others;
|
||||
}
|
||||
|
||||
public List<ResourceInfo> getResourceList() {
|
||||
return resourceList;
|
||||
}
|
||||
|
||||
public void setResourceList(List<ResourceInfo> resourceList) {
|
||||
this.resourceList = resourceList;
|
||||
}
|
||||
|
||||
public ProgramType getProgramType() {
|
||||
return programType;
|
||||
}
|
||||
|
||||
public void setProgramType(ProgramType programType) {
|
||||
this.programType = programType;
|
||||
}
|
||||
|
||||
public String getSparkVersion() {
|
||||
return sparkVersion;
|
||||
}
|
||||
|
||||
public void setSparkVersion(String sparkVersion) {
|
||||
this.sparkVersion = sparkVersion;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkParameters() {
|
||||
return mainJar != null && programType != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceInfo> getResourceFilesList() {
|
||||
if (mainJar != null && !resourceList.contains(mainJar)) {
|
||||
resourceList.add(mainJar);
|
||||
}
|
||||
return resourceList;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,295 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.sql;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.DataType;
|
||||
import org.apache.dolphinscheduler.common.process.Property;
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Sql/Hql parameter
|
||||
*/
|
||||
public class SqlParameters extends AbstractParameters {
|
||||
/**
|
||||
* data source type,eg MYSQL, POSTGRES, HIVE ...
|
||||
*/
|
||||
private String type;
|
||||
|
||||
/**
|
||||
* datasource id
|
||||
*/
|
||||
private int datasource;
|
||||
|
||||
/**
|
||||
* sql
|
||||
*/
|
||||
private String sql;
|
||||
|
||||
/**
|
||||
* sql type
|
||||
* 0 query
|
||||
* 1 NON_QUERY
|
||||
*/
|
||||
private int sqlType;
|
||||
|
||||
/**
|
||||
* send email
|
||||
*/
|
||||
private Boolean sendEmail;
|
||||
|
||||
/**
|
||||
* display rows
|
||||
*/
|
||||
private int displayRows;
|
||||
|
||||
/**
|
||||
* udf list
|
||||
*/
|
||||
private String udfs;
|
||||
/**
|
||||
* show type
|
||||
* 0 TABLE
|
||||
* 1 TEXT
|
||||
* 2 attachment
|
||||
* 3 TABLE+attachment
|
||||
*/
|
||||
private String showType;
|
||||
/**
|
||||
* SQL connection parameters
|
||||
*/
|
||||
private String connParams;
|
||||
/**
|
||||
* Pre Statements
|
||||
*/
|
||||
private List<String> preStatements;
|
||||
/**
|
||||
* Post Statements
|
||||
*/
|
||||
private List<String> postStatements;
|
||||
|
||||
/**
|
||||
* groupId
|
||||
*/
|
||||
private int groupId;
|
||||
/**
|
||||
* title
|
||||
*/
|
||||
private String title;
|
||||
|
||||
private int limit;
|
||||
|
||||
public int getLimit() {
|
||||
return limit;
|
||||
}
|
||||
|
||||
public void setLimit(int limit) {
|
||||
this.limit = limit;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public void setType(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public int getDatasource() {
|
||||
return datasource;
|
||||
}
|
||||
|
||||
public void setDatasource(int datasource) {
|
||||
this.datasource = datasource;
|
||||
}
|
||||
|
||||
public String getSql() {
|
||||
return sql;
|
||||
}
|
||||
|
||||
public void setSql(String sql) {
|
||||
this.sql = sql;
|
||||
}
|
||||
|
||||
public String getUdfs() {
|
||||
return udfs;
|
||||
}
|
||||
|
||||
public void setUdfs(String udfs) {
|
||||
this.udfs = udfs;
|
||||
}
|
||||
|
||||
public int getSqlType() {
|
||||
return sqlType;
|
||||
}
|
||||
|
||||
public void setSqlType(int sqlType) {
|
||||
this.sqlType = sqlType;
|
||||
}
|
||||
|
||||
public Boolean getSendEmail() {
|
||||
return sendEmail;
|
||||
}
|
||||
|
||||
public void setSendEmail(Boolean sendEmail) {
|
||||
this.sendEmail = sendEmail;
|
||||
}
|
||||
|
||||
public int getDisplayRows() {
|
||||
return displayRows;
|
||||
}
|
||||
|
||||
public void setDisplayRows(int displayRows) {
|
||||
this.displayRows = displayRows;
|
||||
}
|
||||
|
||||
public String getShowType() {
|
||||
return showType;
|
||||
}
|
||||
|
||||
public void setShowType(String showType) {
|
||||
this.showType = showType;
|
||||
}
|
||||
|
||||
public String getConnParams() {
|
||||
return connParams;
|
||||
}
|
||||
|
||||
public void setConnParams(String connParams) {
|
||||
this.connParams = connParams;
|
||||
}
|
||||
|
||||
public String getTitle() {
|
||||
return title;
|
||||
}
|
||||
|
||||
public void setTitle(String title) {
|
||||
this.title = title;
|
||||
}
|
||||
|
||||
public List<String> getPreStatements() {
|
||||
return preStatements;
|
||||
}
|
||||
|
||||
public void setPreStatements(List<String> preStatements) {
|
||||
this.preStatements = preStatements;
|
||||
}
|
||||
|
||||
public List<String> getPostStatements() {
|
||||
return postStatements;
|
||||
}
|
||||
|
||||
public void setPostStatements(List<String> postStatements) {
|
||||
this.postStatements = postStatements;
|
||||
}
|
||||
|
||||
public int getGroupId() {
|
||||
return groupId;
|
||||
}
|
||||
|
||||
public void setGroupId(int groupId) {
|
||||
this.groupId = groupId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkParameters() {
|
||||
return datasource != 0 && !StringUtils.isEmpty(type) && !StringUtils.isEmpty(sql);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceInfo> getResourceFilesList() {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dealOutParam(String result) {
|
||||
if (CollectionUtils.isEmpty(localParams)) {
|
||||
return;
|
||||
}
|
||||
List<Property> outProperty = getOutProperty(localParams);
|
||||
if (CollectionUtils.isEmpty(outProperty)) {
|
||||
return;
|
||||
}
|
||||
if (StringUtils.isEmpty(result)) {
|
||||
varPool.addAll(outProperty);
|
||||
return;
|
||||
}
|
||||
List<Map<String, String>> sqlResult = getListMapByString(result);
|
||||
if (CollectionUtils.isEmpty(sqlResult)) {
|
||||
return;
|
||||
}
|
||||
//if sql return more than one line
|
||||
if (sqlResult.size() > 1) {
|
||||
Map<String, List<String>> sqlResultFormat = new HashMap<>();
|
||||
//init sqlResultFormat
|
||||
Set<String> keySet = sqlResult.get(0).keySet();
|
||||
for (String key : keySet) {
|
||||
sqlResultFormat.put(key, new ArrayList<>());
|
||||
}
|
||||
for (Map<String, String> info : sqlResult) {
|
||||
info.forEach((key, value) -> {
|
||||
sqlResultFormat.get(key).add(value);
|
||||
});
|
||||
}
|
||||
for (Property info : outProperty) {
|
||||
if (info.getType() == DataType.LIST) {
|
||||
info.setValue(JSONUtils.toJsonString(sqlResultFormat.get(info.getProp())));
|
||||
varPool.add(info);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//result only one line
|
||||
Map<String, String> firstRow = sqlResult.get(0);
|
||||
for (Property info : outProperty) {
|
||||
info.setValue(String.valueOf(firstRow.get(info.getProp())));
|
||||
varPool.add(info);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "SqlParameters{"
|
||||
+ "type='" + type + '\''
|
||||
+ ", datasource=" + datasource
|
||||
+ ", sql='" + sql + '\''
|
||||
+ ", sqlType=" + sqlType
|
||||
+ ", sendEmail=" + sendEmail
|
||||
+ ", displayRows=" + displayRows
|
||||
+ ", limit=" + limit
|
||||
+ ", udfs='" + udfs + '\''
|
||||
+ ", showType='" + showType + '\''
|
||||
+ ", connParams='" + connParams + '\''
|
||||
+ ", groupId='" + groupId + '\''
|
||||
+ ", title='" + title + '\''
|
||||
+ ", preStatements=" + preStatements
|
||||
+ ", postStatements=" + postStatements
|
||||
+ '}';
|
||||
}
|
||||
}
|
||||
|
|
@ -1,205 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.SqoopJobType;
|
||||
import org.apache.dolphinscheduler.common.process.Property;
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* sqoop parameters
|
||||
*/
|
||||
public class SqoopParameters extends AbstractParameters {
|
||||
|
||||
/**
|
||||
* sqoop job type:
|
||||
* CUSTOM - custom sqoop job
|
||||
* TEMPLATE - sqoop template job
|
||||
*/
|
||||
private String jobType;
|
||||
|
||||
/**
|
||||
* customJob eq 1, use customShell
|
||||
*/
|
||||
private String customShell;
|
||||
|
||||
/**
|
||||
* sqoop job name - map-reduce job name
|
||||
*/
|
||||
private String jobName;
|
||||
|
||||
/**
|
||||
* model type
|
||||
*/
|
||||
private String modelType;
|
||||
/**
|
||||
* concurrency
|
||||
*/
|
||||
private int concurrency;
|
||||
/**
|
||||
* source type
|
||||
*/
|
||||
private String sourceType;
|
||||
/**
|
||||
* target type
|
||||
*/
|
||||
private String targetType;
|
||||
/**
|
||||
* source params
|
||||
*/
|
||||
private String sourceParams;
|
||||
/**
|
||||
* target params
|
||||
*/
|
||||
private String targetParams;
|
||||
|
||||
/**
|
||||
* hadoop custom param for sqoop job
|
||||
*/
|
||||
private List<Property> hadoopCustomParams;
|
||||
|
||||
/**
|
||||
* sqoop advanced param
|
||||
*/
|
||||
private List<Property> sqoopAdvancedParams;
|
||||
|
||||
public String getModelType() {
|
||||
return modelType;
|
||||
}
|
||||
|
||||
public void setModelType(String modelType) {
|
||||
this.modelType = modelType;
|
||||
}
|
||||
|
||||
public int getConcurrency() {
|
||||
return concurrency;
|
||||
}
|
||||
|
||||
public void setConcurrency(int concurrency) {
|
||||
this.concurrency = concurrency;
|
||||
}
|
||||
|
||||
public String getSourceType() {
|
||||
return sourceType;
|
||||
}
|
||||
|
||||
public void setSourceType(String sourceType) {
|
||||
this.sourceType = sourceType;
|
||||
}
|
||||
|
||||
public String getTargetType() {
|
||||
return targetType;
|
||||
}
|
||||
|
||||
public void setTargetType(String targetType) {
|
||||
this.targetType = targetType;
|
||||
}
|
||||
|
||||
public String getSourceParams() {
|
||||
return sourceParams;
|
||||
}
|
||||
|
||||
public void setSourceParams(String sourceParams) {
|
||||
this.sourceParams = sourceParams;
|
||||
}
|
||||
|
||||
public String getTargetParams() {
|
||||
return targetParams;
|
||||
}
|
||||
|
||||
public void setTargetParams(String targetParams) {
|
||||
this.targetParams = targetParams;
|
||||
}
|
||||
|
||||
public String getJobType() {
|
||||
return jobType;
|
||||
}
|
||||
|
||||
public void setJobType(String jobType) {
|
||||
this.jobType = jobType;
|
||||
}
|
||||
|
||||
public String getJobName() {
|
||||
return jobName;
|
||||
}
|
||||
|
||||
public void setJobName(String jobName) {
|
||||
this.jobName = jobName;
|
||||
}
|
||||
|
||||
public String getCustomShell() {
|
||||
return customShell;
|
||||
}
|
||||
|
||||
public void setCustomShell(String customShell) {
|
||||
this.customShell = customShell;
|
||||
}
|
||||
|
||||
public List<Property> getHadoopCustomParams() {
|
||||
return hadoopCustomParams;
|
||||
}
|
||||
|
||||
public void setHadoopCustomParams(List<Property> hadoopCustomParams) {
|
||||
this.hadoopCustomParams = hadoopCustomParams;
|
||||
}
|
||||
|
||||
public List<Property> getSqoopAdvancedParams() {
|
||||
return sqoopAdvancedParams;
|
||||
}
|
||||
|
||||
public void setSqoopAdvancedParams(List<Property> sqoopAdvancedParams) {
|
||||
this.sqoopAdvancedParams = sqoopAdvancedParams;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkParameters() {
|
||||
|
||||
boolean sqoopParamsCheck = false;
|
||||
|
||||
if (StringUtils.isEmpty(jobType)) {
|
||||
return sqoopParamsCheck;
|
||||
}
|
||||
|
||||
if (SqoopJobType.TEMPLATE.getDescp().equals(jobType)) {
|
||||
sqoopParamsCheck = StringUtils.isEmpty(customShell)
|
||||
&& !StringUtils.isEmpty(modelType)
|
||||
&& !StringUtils.isEmpty(jobName)
|
||||
&& concurrency != 0
|
||||
&& !StringUtils.isEmpty(sourceType)
|
||||
&& !StringUtils.isEmpty(targetType)
|
||||
&& !StringUtils.isEmpty(sourceParams)
|
||||
&& !StringUtils.isEmpty(targetParams);
|
||||
} else if (SqoopJobType.CUSTOM.getDescp().equals(jobType)) {
|
||||
sqoopParamsCheck = !StringUtils.isEmpty(customShell)
|
||||
&& StringUtils.isEmpty(jobName);
|
||||
}
|
||||
|
||||
return sqoopParamsCheck;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceInfo> getResourceFilesList() {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop.sources;
|
||||
|
||||
/**
|
||||
* source hdfs parameter
|
||||
*/
|
||||
public class SourceHdfsParameter {
|
||||
|
||||
/**
|
||||
* export dir
|
||||
*/
|
||||
private String exportDir;
|
||||
|
||||
public String getExportDir() {
|
||||
return exportDir;
|
||||
}
|
||||
|
||||
public void setExportDir(String exportDir) {
|
||||
this.exportDir = exportDir;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop.sources;
|
||||
|
||||
/**
|
||||
* source hive parameter
|
||||
*/
|
||||
public class SourceHiveParameter {
|
||||
|
||||
/**
|
||||
* hive database
|
||||
*/
|
||||
private String hiveDatabase;
|
||||
/**
|
||||
* hive table
|
||||
*/
|
||||
private String hiveTable;
|
||||
/**
|
||||
* hive partition key
|
||||
*/
|
||||
private String hivePartitionKey;
|
||||
/**
|
||||
* hive partition value
|
||||
*/
|
||||
private String hivePartitionValue;
|
||||
|
||||
public String getHiveDatabase() {
|
||||
return hiveDatabase;
|
||||
}
|
||||
|
||||
public void setHiveDatabase(String hiveDatabase) {
|
||||
this.hiveDatabase = hiveDatabase;
|
||||
}
|
||||
|
||||
public String getHiveTable() {
|
||||
return hiveTable;
|
||||
}
|
||||
|
||||
public void setHiveTable(String hiveTable) {
|
||||
this.hiveTable = hiveTable;
|
||||
}
|
||||
|
||||
public String getHivePartitionKey() {
|
||||
return hivePartitionKey;
|
||||
}
|
||||
|
||||
public void setHivePartitionKey(String hivePartitionKey) {
|
||||
this.hivePartitionKey = hivePartitionKey;
|
||||
}
|
||||
|
||||
public String getHivePartitionValue() {
|
||||
return hivePartitionValue;
|
||||
}
|
||||
|
||||
public void setHivePartitionValue(String hivePartitionValue) {
|
||||
this.hivePartitionValue = hivePartitionValue;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,137 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop.sources;
|
||||
|
||||
import org.apache.dolphinscheduler.common.process.Property;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* source mysql parameter
|
||||
*/
|
||||
public class SourceMysqlParameter {
|
||||
|
||||
/**
|
||||
* src datasource
|
||||
*/
|
||||
private int srcDatasource;
|
||||
/**
|
||||
* src table
|
||||
*/
|
||||
private String srcTable;
|
||||
/**
|
||||
* src query type
|
||||
*/
|
||||
private int srcQueryType;
|
||||
/**
|
||||
* src query sql
|
||||
*/
|
||||
private String srcQuerySql;
|
||||
/**
|
||||
* src column type
|
||||
*/
|
||||
private int srcColumnType;
|
||||
/**
|
||||
* src columns
|
||||
*/
|
||||
private String srcColumns;
|
||||
/**
|
||||
* src condition list
|
||||
*/
|
||||
private List<Property> srcConditionList;
|
||||
/**
|
||||
* map column hive
|
||||
*/
|
||||
private List<Property> mapColumnHive;
|
||||
/**
|
||||
* map column java
|
||||
*/
|
||||
private List<Property> mapColumnJava;
|
||||
|
||||
public int getSrcDatasource() {
|
||||
return srcDatasource;
|
||||
}
|
||||
|
||||
public void setSrcDatasource(int srcDatasource) {
|
||||
this.srcDatasource = srcDatasource;
|
||||
}
|
||||
|
||||
public String getSrcTable() {
|
||||
return srcTable;
|
||||
}
|
||||
|
||||
public void setSrcTable(String srcTable) {
|
||||
this.srcTable = srcTable;
|
||||
}
|
||||
|
||||
public int getSrcQueryType() {
|
||||
return srcQueryType;
|
||||
}
|
||||
|
||||
public void setSrcQueryType(int srcQueryType) {
|
||||
this.srcQueryType = srcQueryType;
|
||||
}
|
||||
|
||||
public String getSrcQuerySql() {
|
||||
return srcQuerySql;
|
||||
}
|
||||
|
||||
public void setSrcQuerySql(String srcQuerySql) {
|
||||
this.srcQuerySql = srcQuerySql;
|
||||
}
|
||||
|
||||
public int getSrcColumnType() {
|
||||
return srcColumnType;
|
||||
}
|
||||
|
||||
public void setSrcColumnType(int srcColumnType) {
|
||||
this.srcColumnType = srcColumnType;
|
||||
}
|
||||
|
||||
public String getSrcColumns() {
|
||||
return srcColumns;
|
||||
}
|
||||
|
||||
public void setSrcColumns(String srcColumns) {
|
||||
this.srcColumns = srcColumns;
|
||||
}
|
||||
|
||||
public List<Property> getSrcConditionList() {
|
||||
return srcConditionList;
|
||||
}
|
||||
|
||||
public void setSrcConditionList(List<Property> srcConditionList) {
|
||||
this.srcConditionList = srcConditionList;
|
||||
}
|
||||
|
||||
public List<Property> getMapColumnHive() {
|
||||
return mapColumnHive;
|
||||
}
|
||||
|
||||
public void setMapColumnHive(List<Property> mapColumnHive) {
|
||||
this.mapColumnHive = mapColumnHive;
|
||||
}
|
||||
|
||||
public List<Property> getMapColumnJava() {
|
||||
return mapColumnJava;
|
||||
}
|
||||
|
||||
public void setMapColumnJava(List<Property> mapColumnJava) {
|
||||
this.mapColumnJava = mapColumnJava;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop.targets;
|
||||
|
||||
/**
|
||||
* target hdfs parameter
|
||||
*/
|
||||
public class TargetHdfsParameter {
|
||||
|
||||
/**
|
||||
* target dir
|
||||
*/
|
||||
private String targetPath;
|
||||
/**
|
||||
* delete target dir
|
||||
*/
|
||||
private boolean deleteTargetDir;
|
||||
/**
|
||||
* file type
|
||||
*/
|
||||
private String fileType;
|
||||
/**
|
||||
* compression codec
|
||||
*/
|
||||
private String compressionCodec;
|
||||
/**
|
||||
* fields terminated
|
||||
*/
|
||||
private String fieldsTerminated;
|
||||
/**
|
||||
* lines terminated
|
||||
*/
|
||||
private String linesTerminated;
|
||||
|
||||
public String getTargetPath() {
|
||||
return targetPath;
|
||||
}
|
||||
|
||||
public void setTargetPath(String targetPath) {
|
||||
this.targetPath = targetPath;
|
||||
}
|
||||
|
||||
public boolean isDeleteTargetDir() {
|
||||
return deleteTargetDir;
|
||||
}
|
||||
|
||||
public void setDeleteTargetDir(boolean deleteTargetDir) {
|
||||
this.deleteTargetDir = deleteTargetDir;
|
||||
}
|
||||
|
||||
public String getFileType() {
|
||||
return fileType;
|
||||
}
|
||||
|
||||
public void setFileType(String fileType) {
|
||||
this.fileType = fileType;
|
||||
}
|
||||
|
||||
public String getCompressionCodec() {
|
||||
return compressionCodec;
|
||||
}
|
||||
|
||||
public void setCompressionCodec(String compressionCodec) {
|
||||
this.compressionCodec = compressionCodec;
|
||||
}
|
||||
|
||||
public String getFieldsTerminated() {
|
||||
return fieldsTerminated;
|
||||
}
|
||||
|
||||
public void setFieldsTerminated(String fieldsTerminated) {
|
||||
this.fieldsTerminated = fieldsTerminated;
|
||||
}
|
||||
|
||||
public String getLinesTerminated() {
|
||||
return linesTerminated;
|
||||
}
|
||||
|
||||
public void setLinesTerminated(String linesTerminated) {
|
||||
this.linesTerminated = linesTerminated;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,121 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop.targets;
|
||||
|
||||
/**
|
||||
* target hive parameter
|
||||
*/
|
||||
public class TargetHiveParameter {
|
||||
|
||||
/**
|
||||
* hive database
|
||||
*/
|
||||
private String hiveDatabase;
|
||||
/**
|
||||
* hive table
|
||||
*/
|
||||
private String hiveTable;
|
||||
/**
|
||||
* create hive table
|
||||
*/
|
||||
private boolean createHiveTable;
|
||||
/**
|
||||
* drop delimiter
|
||||
*/
|
||||
private boolean dropDelimiter;
|
||||
/**
|
||||
* hive overwrite
|
||||
*/
|
||||
private boolean hiveOverWrite;
|
||||
/**
|
||||
* replace delimiter
|
||||
*/
|
||||
private String replaceDelimiter;
|
||||
/**
|
||||
* hive partition key
|
||||
*/
|
||||
private String hivePartitionKey;
|
||||
/**
|
||||
* hive partition value
|
||||
*/
|
||||
private String hivePartitionValue;
|
||||
|
||||
public String getHiveDatabase() {
|
||||
return hiveDatabase;
|
||||
}
|
||||
|
||||
public void setHiveDatabase(String hiveDatabase) {
|
||||
this.hiveDatabase = hiveDatabase;
|
||||
}
|
||||
|
||||
public String getHiveTable() {
|
||||
return hiveTable;
|
||||
}
|
||||
|
||||
public void setHiveTable(String hiveTable) {
|
||||
this.hiveTable = hiveTable;
|
||||
}
|
||||
|
||||
public boolean isCreateHiveTable() {
|
||||
return createHiveTable;
|
||||
}
|
||||
|
||||
public void setCreateHiveTable(boolean createHiveTable) {
|
||||
this.createHiveTable = createHiveTable;
|
||||
}
|
||||
|
||||
public boolean isDropDelimiter() {
|
||||
return dropDelimiter;
|
||||
}
|
||||
|
||||
public void setDropDelimiter(boolean dropDelimiter) {
|
||||
this.dropDelimiter = dropDelimiter;
|
||||
}
|
||||
|
||||
public boolean isHiveOverWrite() {
|
||||
return hiveOverWrite;
|
||||
}
|
||||
|
||||
public void setHiveOverWrite(boolean hiveOverWrite) {
|
||||
this.hiveOverWrite = hiveOverWrite;
|
||||
}
|
||||
|
||||
public String getReplaceDelimiter() {
|
||||
return replaceDelimiter;
|
||||
}
|
||||
|
||||
public void setReplaceDelimiter(String replaceDelimiter) {
|
||||
this.replaceDelimiter = replaceDelimiter;
|
||||
}
|
||||
|
||||
public String getHivePartitionKey() {
|
||||
return hivePartitionKey;
|
||||
}
|
||||
|
||||
public void setHivePartitionKey(String hivePartitionKey) {
|
||||
this.hivePartitionKey = hivePartitionKey;
|
||||
}
|
||||
|
||||
public String getHivePartitionValue() {
|
||||
return hivePartitionValue;
|
||||
}
|
||||
|
||||
public void setHivePartitionValue(String hivePartitionValue) {
|
||||
this.hivePartitionValue = hivePartitionValue;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,133 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop.targets;
|
||||
|
||||
/**
|
||||
* target mysql parameter
|
||||
*/
|
||||
public class TargetMysqlParameter {
|
||||
|
||||
/**
|
||||
* target datasource
|
||||
*/
|
||||
private int targetDatasource;
|
||||
/**
|
||||
* target table
|
||||
*/
|
||||
private String targetTable;
|
||||
/**
|
||||
* target columns
|
||||
*/
|
||||
private String targetColumns;
|
||||
/**
|
||||
* fields terminated
|
||||
*/
|
||||
private String fieldsTerminated;
|
||||
/**
|
||||
* lines terminated
|
||||
*/
|
||||
private String linesTerminated;
|
||||
/**
|
||||
* pre query
|
||||
*/
|
||||
private String preQuery;
|
||||
/**
|
||||
* is update
|
||||
*/
|
||||
private boolean isUpdate;
|
||||
/**
|
||||
* target update key
|
||||
*/
|
||||
private String targetUpdateKey;
|
||||
/**
|
||||
* target update mode
|
||||
*/
|
||||
private String targetUpdateMode;
|
||||
|
||||
public int getTargetDatasource() {
|
||||
return targetDatasource;
|
||||
}
|
||||
|
||||
public void setTargetDatasource(int targetDatasource) {
|
||||
this.targetDatasource = targetDatasource;
|
||||
}
|
||||
|
||||
public String getTargetTable() {
|
||||
return targetTable;
|
||||
}
|
||||
|
||||
public void setTargetTable(String targetTable) {
|
||||
this.targetTable = targetTable;
|
||||
}
|
||||
|
||||
public String getTargetColumns() {
|
||||
return targetColumns;
|
||||
}
|
||||
|
||||
public void setTargetColumns(String targetColumns) {
|
||||
this.targetColumns = targetColumns;
|
||||
}
|
||||
|
||||
public String getFieldsTerminated() {
|
||||
return fieldsTerminated;
|
||||
}
|
||||
|
||||
public void setFieldsTerminated(String fieldsTerminated) {
|
||||
this.fieldsTerminated = fieldsTerminated;
|
||||
}
|
||||
|
||||
public String getLinesTerminated() {
|
||||
return linesTerminated;
|
||||
}
|
||||
|
||||
public void setLinesTerminated(String linesTerminated) {
|
||||
this.linesTerminated = linesTerminated;
|
||||
}
|
||||
|
||||
public String getPreQuery() {
|
||||
return preQuery;
|
||||
}
|
||||
|
||||
public void setPreQuery(String preQuery) {
|
||||
this.preQuery = preQuery;
|
||||
}
|
||||
|
||||
public boolean getIsUpdate() {
|
||||
return isUpdate;
|
||||
}
|
||||
|
||||
public void setUpdate(boolean update) {
|
||||
isUpdate = update;
|
||||
}
|
||||
|
||||
public String getTargetUpdateKey() {
|
||||
return targetUpdateKey;
|
||||
}
|
||||
|
||||
public void setTargetUpdateKey(String targetUpdateKey) {
|
||||
this.targetUpdateKey = targetUpdateKey;
|
||||
}
|
||||
|
||||
public String getTargetUpdateMode() {
|
||||
return targetUpdateMode;
|
||||
}
|
||||
|
||||
public void setTargetUpdateMode(String targetUpdateMode) {
|
||||
this.targetUpdateMode = targetUpdateMode;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task.tis;
|
||||
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* TIS parameter
|
||||
*/
|
||||
public class PigeonCommonParameters extends AbstractParameters {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PigeonCommonParameters.class);
|
||||
/**
|
||||
* TIS target job name
|
||||
*/
|
||||
private String jobName;
|
||||
|
||||
public String getTargetJobName() {
|
||||
return jobName;
|
||||
}
|
||||
|
||||
public void setTargetJobName(String jobName) {
|
||||
this.jobName = jobName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkParameters() {
|
||||
return StringUtils.isNotBlank(this.jobName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceInfo> getResourceFilesList() {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
|
|
@ -23,7 +23,6 @@ import org.apache.commons.beanutils.BeanMap;
|
|||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
|
|||
|
|
@ -20,9 +20,9 @@ package org.apache.dolphinscheduler.common.utils;
|
|||
import static org.apache.dolphinscheduler.common.Constants.RESOURCE_UPLOAD_PATH;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.enums.ResUploadType;
|
||||
import org.apache.dolphinscheduler.common.exception.BaseException;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.common.Constants;
|
|||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.text.DateFormat;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
package org.apache.dolphinscheduler.common.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.spi.task.TaskConstants;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.FileInputStream;
|
||||
|
|
|
|||
|
|
@ -19,10 +19,10 @@ package org.apache.dolphinscheduler.common.utils;
|
|||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.CommandType;
|
||||
import org.apache.dolphinscheduler.common.process.Property;
|
||||
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
|
|
@ -31,6 +31,7 @@ import java.util.HashMap;
|
|||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
|
@ -152,7 +153,7 @@ public class ParameterUtils {
|
|||
*/
|
||||
public static Map<String, String> convert(Map<String, Property> paramsMap) {
|
||||
Map<String, String> map = new HashMap<>();
|
||||
Iterator<Map.Entry<String, Property>> iter = paramsMap.entrySet().iterator();
|
||||
Iterator<Entry<String, Property>> iter = paramsMap.entrySet().iterator();
|
||||
while (iter.hasNext()) {
|
||||
Map.Entry<String, Property> en = iter.next();
|
||||
map.put(en.getKey(), en.getValue().getValue());
|
||||
|
|
|
|||
|
|
@ -1,106 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
import org.apache.dolphinscheduler.common.task.blocking.BlockingParameters;
|
||||
import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters;
|
||||
import org.apache.dolphinscheduler.common.task.datax.DataxParameters;
|
||||
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
|
||||
import org.apache.dolphinscheduler.common.task.dq.DataQualityParameters;
|
||||
import org.apache.dolphinscheduler.common.task.emr.EmrParameters;
|
||||
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
|
||||
import org.apache.dolphinscheduler.common.task.http.HttpParameters;
|
||||
import org.apache.dolphinscheduler.common.task.mr.MapReduceParameters;
|
||||
import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters;
|
||||
import org.apache.dolphinscheduler.common.task.python.PythonParameters;
|
||||
import org.apache.dolphinscheduler.common.task.shell.ShellParameters;
|
||||
import org.apache.dolphinscheduler.common.task.spark.SparkParameters;
|
||||
import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
|
||||
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
|
||||
import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters;
|
||||
import org.apache.dolphinscheduler.common.task.switchtask.SwitchParameters;
|
||||
import org.apache.dolphinscheduler.common.task.tis.PigeonCommonParameters;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* task parameters utils
|
||||
*/
|
||||
public class TaskParametersUtils {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(TaskParametersUtils.class);
|
||||
|
||||
private TaskParametersUtils() {
|
||||
throw new UnsupportedOperationException("Construct TaskParametersUtils");
|
||||
}
|
||||
|
||||
/**
|
||||
* get task parameters
|
||||
*
|
||||
* @param taskType task type
|
||||
* @param parameter parameter
|
||||
* @return task parameters
|
||||
*/
|
||||
public static AbstractParameters getParameters(String taskType, String parameter) {
|
||||
switch (taskType) {
|
||||
case "SUB_PROCESS":
|
||||
return JSONUtils.parseObject(parameter, SubProcessParameters.class);
|
||||
case "SHELL":
|
||||
case "SEATUNNEL":
|
||||
return JSONUtils.parseObject(parameter, ShellParameters.class);
|
||||
case "PROCEDURE":
|
||||
return JSONUtils.parseObject(parameter, ProcedureParameters.class);
|
||||
case "SQL":
|
||||
return JSONUtils.parseObject(parameter, SqlParameters.class);
|
||||
case "MR":
|
||||
return JSONUtils.parseObject(parameter, MapReduceParameters.class);
|
||||
case "SPARK":
|
||||
return JSONUtils.parseObject(parameter, SparkParameters.class);
|
||||
case "PYTHON":
|
||||
return JSONUtils.parseObject(parameter, PythonParameters.class);
|
||||
case "DEPENDENT":
|
||||
return JSONUtils.parseObject(parameter, DependentParameters.class);
|
||||
case "FLINK":
|
||||
return JSONUtils.parseObject(parameter, FlinkParameters.class);
|
||||
case "HTTP":
|
||||
return JSONUtils.parseObject(parameter, HttpParameters.class);
|
||||
case "DATAX":
|
||||
return JSONUtils.parseObject(parameter, DataxParameters.class);
|
||||
case "CONDITIONS":
|
||||
return JSONUtils.parseObject(parameter, ConditionsParameters.class);
|
||||
case "SQOOP":
|
||||
return JSONUtils.parseObject(parameter, SqoopParameters.class);
|
||||
case "DATA_QUALITY":
|
||||
return JSONUtils.parseObject(parameter, DataQualityParameters.class);
|
||||
case "SWITCH":
|
||||
return JSONUtils.parseObject(parameter, SwitchParameters.class);
|
||||
case "BLOCKING":
|
||||
return JSONUtils.parseObject(parameter, BlockingParameters.class);
|
||||
case "PIGEON":
|
||||
return JSONUtils.parseObject(parameter, PigeonCommonParameters.class);
|
||||
case "EMR":
|
||||
return JSONUtils.parseObject(parameter, EmrParameters.class);
|
||||
default:
|
||||
logger.error("not support task type: {}", taskType);
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -17,6 +17,12 @@
|
|||
|
||||
package org.apache.dolphinscheduler.common.utils.placeholder;
|
||||
|
||||
import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_DATE;
|
||||
import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_TIME;
|
||||
import static org.apache.dolphinscheduler.common.utils.DateUtils.format;
|
||||
|
||||
import static org.apache.commons.lang.time.DateUtils.addDays;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.CommandType;
|
||||
|
||||
|
|
@ -24,12 +30,6 @@ import java.util.Date;
|
|||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_DATE;
|
||||
import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_TIME;
|
||||
import static org.apache.dolphinscheduler.common.utils.DateUtils.format;
|
||||
import static org.apache.commons.lang.time.DateUtils.addDays;
|
||||
|
||||
|
||||
/**
|
||||
* business time utils
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -14,13 +14,14 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.utils.placeholder;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* placeholder utils
|
||||
*/
|
||||
|
|
@ -39,7 +40,6 @@ public class PlaceholderUtils {
|
|||
|
||||
public static final String PLACEHOLDER_SUFFIX = "}";
|
||||
|
||||
|
||||
/**
|
||||
* Replaces all placeholders of format {@code ${name}} with the value returned
|
||||
* from the supplied {@link PropertyPlaceholderHelper.PlaceholderResolver}.
|
||||
|
|
@ -94,11 +94,10 @@ public class PlaceholderUtils {
|
|||
try {
|
||||
return paramsMap.get(placeholderName);
|
||||
} catch (Exception ex) {
|
||||
logger.error("resolve placeholder '{}' in [ {} ]" , placeholderName, value, ex);
|
||||
logger.error("resolve placeholder '{}' in [ {} ]", placeholderName, value, ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common;
|
||||
|
||||
import org.apache.commons.lang.SystemUtils;
|
||||
|
|
|
|||
|
|
@ -1,132 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.task;
|
||||
|
||||
import org.apache.dolphinscheduler.common.task.dq.DataQualityParameters;
|
||||
import org.apache.dolphinscheduler.common.task.spark.SparkParameters;
|
||||
import org.apache.dolphinscheduler.spi.params.base.ParamsOptions;
|
||||
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
|
||||
import org.apache.dolphinscheduler.spi.params.base.TriggerType;
|
||||
import org.apache.dolphinscheduler.spi.params.base.Validate;
|
||||
import org.apache.dolphinscheduler.spi.params.input.InputParam;
|
||||
import org.apache.dolphinscheduler.spi.params.input.InputParamProps;
|
||||
import org.apache.dolphinscheduler.spi.params.select.SelectParam;
|
||||
import org.apache.dolphinscheduler.spi.params.select.SelectParamProps;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* DataQualityParameterTest
|
||||
*/
|
||||
public class DataQualityParameterTest {
|
||||
|
||||
private DataQualityParameters dataQualityParameters = null;
|
||||
|
||||
@Before
|
||||
public void before() {
|
||||
dataQualityParameters = new DataQualityParameters();
|
||||
dataQualityParameters.setRuleId(1);
|
||||
dataQualityParameters.setSparkParameters(new SparkParameters());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCheckParameterNormal() {
|
||||
|
||||
Map<String,String> inputParameterValue = new HashMap<>();
|
||||
inputParameterValue.put("src_connector_type","JDBC");
|
||||
inputParameterValue.put("src_datasource_id","1");
|
||||
inputParameterValue.put("src_table","test1");
|
||||
inputParameterValue.put("src_filter","date=2012-10-05");
|
||||
inputParameterValue.put("src_field","id");
|
||||
|
||||
inputParameterValue.put("rule_type","1");
|
||||
inputParameterValue.put("process_definition_id","1");
|
||||
inputParameterValue.put("task_instance_id","1");
|
||||
inputParameterValue.put("check_type","1");
|
||||
inputParameterValue.put("threshold","1000");
|
||||
inputParameterValue.put("create_time","2012-10-05");
|
||||
inputParameterValue.put("update_time","2012-10-05");
|
||||
|
||||
dataQualityParameters.setRuleInputParameter(inputParameterValue);
|
||||
|
||||
Assert.assertTrue(dataQualityParameters.checkParameters());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRuleInputParameter() {
|
||||
String formCreateJson = "[{\"field\":\"src_connector_type\",\"name\":\"源数据类型\","
|
||||
+ "\"props\":{\"disabled\":false,\"multiple\":false,\"size\":\"small\"},"
|
||||
+ "\"type\":\"select\",\"title\":\"源数据类型\",\"value\":\"JDBC\","
|
||||
+ "\"options\":[{\"label\":\"HIVE\",\"value\":\"HIVE\",\"disabled\":false},"
|
||||
+ "{\"label\":\"JDBC\",\"value\":\"JDBC\",\"disabled\":false}]},"
|
||||
+ "{\"props\":{\"disabled\":false,\"rows\":0,\"placeholder\":\"Please enter source table name\","
|
||||
+ "\"size\":\"small\"},\"field\":\"src_table\",\"name\":\"源数据表\","
|
||||
+ "\"type\":\"input\",\"title\":\"源数据表\",\"validate\":[{\"required\":true,\"type\":\"string\","
|
||||
+ "\"trigger\":\"blur\"}]}]";
|
||||
|
||||
List<PluginParams> pluginParamsList = new ArrayList<>();
|
||||
SelectParamProps selectParamProps = new SelectParamProps();
|
||||
selectParamProps.setMultiple(false);
|
||||
selectParamProps.setDisabled(false);
|
||||
selectParamProps.setSize("small");
|
||||
|
||||
SelectParam srcConnectorType = SelectParam.newBuilder("src_connector_type","源数据类型")
|
||||
.setProps(selectParamProps)
|
||||
.addOptions(new ParamsOptions("HIVE","HIVE",false))
|
||||
.addOptions(new ParamsOptions("JDBC","JDBC",false))
|
||||
.setValue("JDBC")
|
||||
.build();
|
||||
|
||||
InputParamProps inputParamProps = new InputParamProps();
|
||||
inputParamProps.setPlaceholder("Please enter source table name");
|
||||
inputParamProps.setDisabled(false);
|
||||
inputParamProps.setSize("small");
|
||||
inputParamProps.setRows(0);
|
||||
|
||||
InputParam srcTable = InputParam.newBuilder("src_table","源数据表")
|
||||
.setProps(inputParamProps)
|
||||
.addValidate(Validate.newBuilder().setType("string").setRequired(true).setTrigger(TriggerType.BLUR.getTriggerType()).build())
|
||||
.build();
|
||||
|
||||
pluginParamsList.add(srcConnectorType);
|
||||
pluginParamsList.add(srcTable);
|
||||
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
|
||||
String result = null;
|
||||
|
||||
try {
|
||||
result = mapper.writeValueAsString(pluginParamsList);
|
||||
} catch (JsonProcessingException e) {
|
||||
Assert.fail();
|
||||
}
|
||||
|
||||
Assert.assertEquals(formCreateJson,result);
|
||||
}
|
||||
}
|
||||
|
|
@ -17,10 +17,10 @@
|
|||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.DataType;
|
||||
import org.apache.dolphinscheduler.common.enums.Direct;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.process.Property;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
|
|
|
|||
|
|
@ -14,9 +14,11 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue