[maven-release-plugin] prepare for next development iteration

fix re-run time error

ambari_plugin/common-services/DOLPHIN/1.3.4/alerts.json

@@ -65,7 +65,7 @@
         "enabled": true,
         "source": {
           "type": "SCRIPT",
-          "path": "DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py",
+          "path": "DOLPHIN/1.3.4/package/alerts/alert_dolphin_scheduler_status.py",
           "parameters": [
 
             {
@@ -98,7 +98,7 @@
         "enabled": true,
         "source": {
           "type": "SCRIPT",
-          "path": "DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py",
+          "path": "DOLPHIN/1.3.4/package/alerts/alert_dolphin_scheduler_status.py",
           "parameters": [
 
             {
@@ -131,7 +131,7 @@
         "enabled": true,
         "source": {
           "type": "SCRIPT",
-          "path": "DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py",
+          "path": "DOLPHIN/1.3.4/package/alerts/alert_dolphin_scheduler_status.py",
           "parameters": [
 
             {

ambari_plugin/common-services/DOLPHIN/1.3.4/metainfo.xml

@@ -22,7 +22,7 @@
             <name>DOLPHIN</name>
             <displayName>Dolphin Scheduler</displayName>
             <comment>分布式易扩展的可视化DAG工作流任务调度系统</comment>
-            <version>1.3.3</version>
+            <version>1.3.4</version>
             <components>
                 <component>
                     <name>DOLPHIN_MASTER</name>

ambari_plugin/statcks/DOLPHIN/metainfo.xml

@@ -20,7 +20,7 @@
   <services>
     <service>
       <name>DOLPHIN</name>
-      <extends>common-services/DOLPHIN/1.3.3</extends>
+      <extends>common-services/DOLPHIN/1.3.4</extends>
     </service>
   </services>
 </metainfo>

docker/build/Dockerfile

@@ -29,8 +29,7 @@ ENV DEBIAN_FRONTEND noninteractive
 RUN apk update && \
     apk --update add --no-cache dos2unix shadow bash openrc python2 python3 sudo vim wget iputils net-tools openssh-server py-pip tini && \
     apk add --update procps && \
-    openrc boot && \
-    pip install kazoo
+    openrc boot
 
 #2. install jdk
 RUN apk add openjdk8

docker/build/README.md

@@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
 #### You can start a dolphinscheduler instance
 ```
 $ docker run -dit --name dolphinscheduler \ 
--e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
+-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
 -p 8888:8888 \
 dolphinscheduler all
 ```
@@ -25,14 +25,14 @@ The default postgres user `root`, postgres password `root` and database `dolphin
 
 The default zookeeper is created in the `startup.sh`.
 
-#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`**
+#### Or via Environment Variables **`DATABASE_HOST`** **`DATABASE_PORT`** **`DATABASE_DATABASE`** **`ZOOKEEPER_QUORUM`**
 
 You can specify **existing postgres service**. Example:
 
 ```
 $ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
 -p 8888:8888 \
 dolphinscheduler all
 ```
@@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example:
 ```
 $ docker run -dit --name dolphinscheduler \
 -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
 -p 8888:8888 \
 dolphinscheduler all
 ```
@@ -56,8 +56,8 @@ You can start a standalone dolphinscheduler server.
 ```
 $ docker run -dit --name dolphinscheduler \
 -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
 dolphinscheduler master-server
 ```
 
@@ -66,8 +66,8 @@ dolphinscheduler master-server
 ```
 $ docker run -dit --name dolphinscheduler \
 -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
 dolphinscheduler worker-server
 ```
 
@@ -75,8 +75,8 @@ dolphinscheduler worker-server
 
 ```
 $ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
 -p 12345:12345 \
 dolphinscheduler api-server
 ```
@@ -85,8 +85,8 @@ dolphinscheduler api-server
 
 ```
 $ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
 dolphinscheduler alert-server
 ```
 
@@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
 dolphinscheduler frontend
 ```
 
-**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
+**Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
 
 ## How to build a docker image
 
@@ -124,33 +124,51 @@ Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script
 
 The Dolphin Scheduler image uses several environment variables which are easy to miss. While none of the variables are required, they may significantly aid you in using the image.
 
-**`POSTGRESQL_HOST`**
+**`DATABASE_TYPE`**
 
-This environment variable sets the host for PostgreSQL. The default value is `127.0.0.1`.
+This environment variable sets the type for database. The default value is `postgresql`.
 
 **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
 
-**`POSTGRESQL_PORT`**
+**`DATABASE_DRIVER`**
 
-This environment variable sets the port for PostgreSQL. The default value is `5432`.
-
-**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. 
-
-**`POSTGRESQL_USERNAME`**
-
-This environment variable sets the username for PostgreSQL. The default value is `root`.
+This environment variable sets the type for database. The default value is `org.postgresql.Driver`.
 
 **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
 
-**`POSTGRESQL_PASSWORD`**
+**`DATABASE_HOST`**
 
-This environment variable sets the password for PostgreSQL. The default value is `root`.
+This environment variable sets the host for database. The default value is `127.0.0.1`.
 
 **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
 
-**`POSTGRESQL_DATABASE`**
+**`DATABASE_PORT`**
 
-This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`.
+This environment variable sets the port for database. The default value is `5432`.
+
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+
+**`DATABASE_USERNAME`**
+
+This environment variable sets the username for database. The default value is `root`.
+
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+
+**`DATABASE_PASSWORD`**
+
+This environment variable sets the password for database. The default value is `root`.
+
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+
+**`DATABASE_DATABASE`**
+
+This environment variable sets the database for database. The default value is `dolphinscheduler`.
+
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+
+**`DATABASE_PARAMS`**
+
+This environment variable sets the database for database. The default value is `characterEncoding=utf8`.
 
 **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
 

docker/build/README_zh_CN.md

@@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
 #### 你可以运行一个dolphinscheduler实例
 ```
 $ docker run -dit --name dolphinscheduler \ 
--e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
+-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
 -p 8888:8888 \
 dolphinscheduler all
 ```
@@ -25,14 +25,14 @@ dolphinscheduler all
 
 同时，默认的`Zookeeper`也会在`startup.sh`脚本中被创建。
 
-#### 或者通过环境变量 **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务
+#### 或者通过环境变量 **`DATABASE_HOST`** **`DATABASE_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务
 
 你可以指定一个已经存在的 **`Postgres`** 服务. 如下:
 
 ```
 $ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
 -p 8888:8888 \
 dolphinscheduler all
 ```
@@ -42,7 +42,7 @@ dolphinscheduler all
 ```
 $ docker run -dit --name dolphinscheduler \
 -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
 -p 8888:8888 \
 dolphinscheduler all
 ```
@@ -56,8 +56,8 @@ dolphinscheduler all
 ```
 $ docker run -dit --name dolphinscheduler \
 -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
 dolphinscheduler master-server
 ```
 
@@ -66,8 +66,8 @@ dolphinscheduler master-server
 ```
 $ docker run -dit --name dolphinscheduler \
 -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
 dolphinscheduler worker-server
 ```
 
@@ -75,8 +75,8 @@ dolphinscheduler worker-server
 
 ```
 $ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
 -p 12345:12345 \
 dolphinscheduler api-server
 ```
@@ -85,8 +85,8 @@ dolphinscheduler api-server
 
 ```
 $ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
 dolphinscheduler alert-server
 ```
 
@@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
 dolphinscheduler frontend
 ```
 
-**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM`。
+**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`。
 
 ## 如何构建一个docker镜像
 
@@ -124,33 +124,51 @@ c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat
 
 Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些变量不是必须的，但是可以帮助你更容易配置镜像并根据你的需求定义相应的服务配置。
 
-**`POSTGRESQL_HOST`**
+**`DATABASE_TYPE`**
 
-配置`PostgreSQL`的`HOST`， 默认值 `127.0.0.1`。
+配置`database`的`TYPE`， 默认值 `postgresql`。
 
 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
 
-**`POSTGRESQL_PORT`**
+**`DATABASE_DRIVER`**
 
-配置`PostgreSQL`的`PORT`， 默认值 `5432`。
+配置`database`的`DRIVER`， 默认值 `org.postgresql.Driver`。
 
 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
 
-**`POSTGRESQL_USERNAME`**
+**`DATABASE_HOST`**
 
-配置`PostgreSQL`的`USERNAME`， 默认值 `root`。
+配置`database`的`HOST`， 默认值 `127.0.0.1`。
 
 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
 
-**`POSTGRESQL_PASSWORD`**
+**`DATABASE_PORT`**
 
-配置`PostgreSQL`的`PASSWORD`， 默认值 `root`。
+配置`database`的`PORT`， 默认值 `5432`。
 
 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
 
-**`POSTGRESQL_DATABASE`**
+**`DATABASE_USERNAME`**
 
-配置`PostgreSQL`的`DATABASE`， 默认值 `dolphinscheduler`。
+配置`database`的`USERNAME`， 默认值 `root`。
+
+**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
+
+**`DATABASE_PASSWORD`**
+
+配置`database`的`PASSWORD`， 默认值 `root`。
+
+**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
+
+**`DATABASE_DATABASE`**
+
+配置`database`的`DATABASE`， 默认值 `dolphinscheduler`。
+
+**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
+
+**`DATABASE_PARAMS`**
+
+配置`database`的`PARAMS`， 默认值 `characterEncoding=utf8`。
 
 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
 

docker/build/conf/dolphinscheduler/common.properties.tpl

@@ -31,7 +31,7 @@ resource.storage.type=${DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE}
 # HDFS
 #============================================================================
 # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
-#resource.upload.path=/dolphinscheduler
+resource.upload.path=${RESOURCE_UPLOAD_PATH}
 
 # whether kerberos starts
 #hadoop.security.authentication.startup.state=false
@@ -61,13 +61,13 @@ kerberos.expire.time=7
 fs.defaultFS=${DOLPHINSCHEDULER_FS_DEFAULTFS}
 
 # if resource.storage.type=S3，s3 endpoint
-#fs.s3a.endpoint=http://192.168.199.91:9010
+fs.s3a.endpoint=${FS_S3A_ENDPOINT}
 
 # if resource.storage.type=S3，s3 access key
-#fs.s3a.access.key=A3DXS30FO22544RE
+fs.s3a.access.key=${FS_S3A_ACCESS_KEY}
 
 # if resource.storage.type=S3，s3 secret key
-#fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
+fs.s3a.secret.key=${FS_S3A_SECRET_KEY}
 
 # if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty  TODO
 yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx

docker/build/conf/dolphinscheduler/datasource.properties.tpl

@@ -15,16 +15,11 @@
 # limitations under the License.
 #
 
-
-# mysql
-#spring.datasource.driver-class-name=com.mysql.jdbc.Driver
-#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
-
-# postgre
-spring.datasource.driver-class-name=org.postgresql.Driver
-spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8
-spring.datasource.username=${POSTGRESQL_USERNAME}
-spring.datasource.password=${POSTGRESQL_PASSWORD}
+# db
+spring.datasource.driver-class-name=${DATABASE_DRIVER}
+spring.datasource.url=jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS}
+spring.datasource.username=${DATABASE_USERNAME}
+spring.datasource.password=${DATABASE_PASSWORD}
 
 ## base spring data source configuration todo need to remove
 #spring.datasource.type=com.alibaba.druid.pool.DruidDataSource

docker/build/conf/dolphinscheduler/zookeeper.properties.tpl

@@ -19,7 +19,7 @@
 zookeeper.quorum=${ZOOKEEPER_QUORUM}
 
 # dolphinscheduler root directory
-#zookeeper.dolphinscheduler.root=/dolphinscheduler
+zookeeper.dolphinscheduler.root=${ZOOKEEPER_ROOT}
 
 # dolphinscheduler failover directory
 #zookeeper.session.timeout=60000

docker/build/startup-init-conf.sh

@@ -24,22 +24,32 @@ echo "init env variables"
 #============================================================================
 # Database Source
 #============================================================================
-export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"}
-export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"}
-export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"}
-export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"}
-export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"}
+export DATABASE_HOST=${DATABASE_HOST:-"127.0.0.1"}
+export DATABASE_PORT=${DATABASE_PORT:-"5432"}
+export DATABASE_USERNAME=${DATABASE_USERNAME:-"root"}
+export DATABASE_PASSWORD=${DATABASE_PASSWORD:-"root"}
+export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"}
+export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"}
+export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"}
+export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
 
 #============================================================================
 # System
 #============================================================================
 export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"}
 export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
+export DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE=${DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE:-"HDFS"}
+export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/ds"}
+export DOLPHINSCHEDULER_FS_DEFAULTFS=${DOLPHINSCHEDULER_FS_DEFAULTFS:-"file:///data/dolphinscheduler"}
+export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"}
+export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"}
+export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"}
 
 #============================================================================
 # Zookeeper
 #============================================================================
 export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"}
+export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"}
 
 #============================================================================
 # Master Server

docker/build/startup.sh

@@ -22,24 +22,32 @@ DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
 DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script
 DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
 
-# start postgresql
-initPostgreSQL() {
-    echo "test postgresql service"
-    while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
+# start database
+initDatabase() {
+    echo "test ${DATABASE_TYPE} service"
+    while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
         counter=$((counter+1))
         if [ $counter == 30 ]; then
-            echo "Error: Couldn't connect to postgresql."
+            echo "Error: Couldn't connect to ${DATABASE_TYPE}."
             exit 1
         fi
-        echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
+        echo "Trying to connect to ${DATABASE_TYPE} at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
         sleep 5
     done
 
-    echo "connect postgresql service"
-    v=$(sudo -u postgres PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h ${POSTGRESQL_HOST} -p ${POSTGRESQL_PORT} -U ${POSTGRESQL_USERNAME} -d dolphinscheduler -tAc "select 1")
-    if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then
-        echo "Error: Can't connect to database...${v}"
-        exit 1
+    echo "connect ${DATABASE_TYPE} service"
+    if [ ${DATABASE_TYPE} = "mysql" ]; then
+        v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "select 1" 2>&1)
+        if [ "$(echo ${v} | grep 'ERROR' | wc -l)" -eq 1 ]; then
+            echo "Error: Can't connect to database...${v}"
+            exit 1
+        fi
+    else
+        v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1")
+        if [ "$(echo ${v} | grep 'FATAL' | wc -l)" -eq 1 ]; then
+            echo "Error: Can't connect to database...${v}"
+            exit 1
+        fi
     fi
 
     echo "import sql data"
@@ -123,7 +131,7 @@ LOGFILE=/var/log/nginx/access.log
 case "$1" in
     (all)
         initZK
-        initPostgreSQL
+        initDatabase
         initMasterServer
         initWorkerServer
         initApiServer
@@ -134,25 +142,25 @@ case "$1" in
     ;;
     (master-server)
         initZK
-        initPostgreSQL
+        initDatabase
         initMasterServer
         LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log
     ;;
     (worker-server)
         initZK
-        initPostgreSQL
+        initDatabase
         initWorkerServer
         initLoggerServer
         LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log
     ;;
     (api-server)
         initZK
-        initPostgreSQL
+        initDatabase
         initApiServer
         LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log
     ;;
     (alert-server)
-        initPostgreSQL
+        initDatabase
         initAlertServer
         LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log
     ;;

docker/docker-swarm/docker-compose.yml

@@ -57,12 +57,13 @@ services:
     - 12345:12345
     environment:
       TZ: Asia/Shanghai
-      POSTGRESQL_HOST: dolphinscheduler-postgresql
-      POSTGRESQL_PORT: 5432
-      POSTGRESQL_USERNAME: root
-      POSTGRESQL_PASSWORD: root
-      POSTGRESQL_DATABASE: dolphinscheduler
+      DATABASE_HOST: dolphinscheduler-postgresql
+      DATABASE_PORT: 5432
+      DATABASE_USERNAME: root
+      DATABASE_PASSWORD: root
+      DATABASE_DATABASE: dolphinscheduler
       ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+      ZOOKEEPER_ROOT: /dolphinscheduler
       DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE: "HDFS"
       DOLPHINSCHEDULER_FS_DEFAULTFS: "file:///data/dolphinscheduler"
     healthcheck:
@@ -123,11 +124,11 @@ services:
       ENTERPRISE_WECHAT_SECRET: ""
       ENTERPRISE_WECHAT_AGENT_ID: ""
       ENTERPRISE_WECHAT_USERS: ""
-      POSTGRESQL_HOST: dolphinscheduler-postgresql
-      POSTGRESQL_PORT: 5432
-      POSTGRESQL_USERNAME: root
-      POSTGRESQL_PASSWORD: root
-      POSTGRESQL_DATABASE: dolphinscheduler
+      DATABASE_HOST: dolphinscheduler-postgresql
+      DATABASE_PORT: 5432
+      DATABASE_USERNAME: root
+      DATABASE_PASSWORD: root
+      DATABASE_DATABASE: dolphinscheduler
     healthcheck:
       test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
       interval: 30s
@@ -156,12 +157,13 @@ services:
       MASTER_TASK_COMMIT_INTERVAL: "1000"
       MASTER_MAX_CPULOAD_AVG: "100"
       MASTER_RESERVED_MEMORY: "0.1"
-      POSTGRESQL_HOST: dolphinscheduler-postgresql
-      POSTGRESQL_PORT: 5432
-      POSTGRESQL_USERNAME: root
-      POSTGRESQL_PASSWORD: root
-      POSTGRESQL_DATABASE: dolphinscheduler
+      DATABASE_HOST: dolphinscheduler-postgresql
+      DATABASE_PORT: 5432
+      DATABASE_USERNAME: root
+      DATABASE_PASSWORD: root
+      DATABASE_DATABASE: dolphinscheduler
       ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+      ZOOKEEPER_ROOT: /dolphinscheduler
     healthcheck:
       test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
       interval: 30s
@@ -192,12 +194,13 @@ services:
       WORKER_RESERVED_MEMORY: "0.1"
       WORKER_GROUP: "default"
       DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
-      POSTGRESQL_HOST: dolphinscheduler-postgresql
-      POSTGRESQL_PORT: 5432
-      POSTGRESQL_USERNAME: root
-      POSTGRESQL_PASSWORD: root
-      POSTGRESQL_DATABASE: dolphinscheduler
+      DATABASE_HOST: dolphinscheduler-postgresql
+      DATABASE_PORT: 5432
+      DATABASE_USERNAME: root
+      DATABASE_PASSWORD: root
+      DATABASE_DATABASE: dolphinscheduler
       ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+      ZOOKEEPER_ROOT: /dolphinscheduler
       DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE: "HDFS"
       DOLPHINSCHEDULER_FS_DEFAULTFS: "file:///data/dolphinscheduler"
     healthcheck:

docker/docker-swarm/docker-stack.yml

@@ -58,11 +58,11 @@ services:
     - 12345:12345
     environment:
       TZ: Asia/Shanghai
-      POSTGRESQL_HOST: dolphinscheduler-postgresql
-      POSTGRESQL_PORT: 5432
-      POSTGRESQL_USERNAME: root
-      POSTGRESQL_PASSWORD: root
-      POSTGRESQL_DATABASE: dolphinscheduler
+      DATABASE_HOST: dolphinscheduler-postgresql
+      DATABASE_PORT: 5432
+      DATABASE_USERNAME: root
+      DATABASE_PASSWORD: root
+      DATABASE_DATABASE: dolphinscheduler
       ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
     healthcheck:
       test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
@@ -120,11 +120,11 @@ services:
       ENTERPRISE_WECHAT_SECRET: ""
       ENTERPRISE_WECHAT_AGENT_ID: ""
       ENTERPRISE_WECHAT_USERS: ""
-      POSTGRESQL_HOST: dolphinscheduler-postgresql
-      POSTGRESQL_PORT: 5432
-      POSTGRESQL_USERNAME: root
-      POSTGRESQL_PASSWORD: root
-      POSTGRESQL_DATABASE: dolphinscheduler
+      DATABASE_HOST: dolphinscheduler-postgresql
+      DATABASE_PORT: 5432
+      DATABASE_USERNAME: root
+      DATABASE_PASSWORD: root
+      DATABASE_DATABASE: dolphinscheduler
     healthcheck:
       test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
       interval: 30
@@ -153,11 +153,11 @@ services:
       MASTER_TASK_COMMIT_INTERVAL: "1000"
       MASTER_MAX_CPULOAD_AVG: "100"
       MASTER_RESERVED_MEMORY: "0.1"
-      POSTGRESQL_HOST: dolphinscheduler-postgresql
-      POSTGRESQL_PORT: 5432
-      POSTGRESQL_USERNAME: root
-      POSTGRESQL_PASSWORD: root
-      POSTGRESQL_DATABASE: dolphinscheduler
+      DATABASE_HOST: dolphinscheduler-postgresql
+      DATABASE_PORT: 5432
+      DATABASE_USERNAME: root
+      DATABASE_PASSWORD: root
+      DATABASE_DATABASE: dolphinscheduler
       ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
     healthcheck:
       test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
@@ -188,11 +188,11 @@ services:
       WORKER_RESERVED_MEMORY: "0.1"
       WORKER_GROUP: "default"
       DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
-      POSTGRESQL_HOST: dolphinscheduler-postgresql
-      POSTGRESQL_PORT: 5432
-      POSTGRESQL_USERNAME: root
-      POSTGRESQL_PASSWORD: root
-      POSTGRESQL_DATABASE: dolphinscheduler
+      DATABASE_HOST: dolphinscheduler-postgresql
+      DATABASE_PORT: 5432
+      DATABASE_USERNAME: root
+      DATABASE_PASSWORD: root
+      DATABASE_DATABASE: dolphinscheduler
       ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
     healthcheck:
       test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]

dolphinscheduler-alert/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>org.apache.dolphinscheduler</groupId>
         <artifactId>dolphinscheduler</artifactId>
-        <version>1.3.4-SNAPSHOT</version>
+        <version>1.3.5-SNAPSHOT</version>
     </parent>
     <artifactId>dolphinscheduler-alert</artifactId>
     <name>${project.artifactId}</name>

dolphinscheduler-api/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.dolphinscheduler</groupId>
     <artifactId>dolphinscheduler</artifactId>
-    <version>1.3.4-SNAPSHOT</version>
+    <version>1.3.5-SNAPSHOT</version>
   </parent>
   <artifactId>dolphinscheduler-api</artifactId>
   <name>${project.artifactId}</name>
@@ -96,16 +96,24 @@
       <artifactId>commons-collections</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>org.quartz-scheduler</groupId>
-      <artifactId>quartz</artifactId>
-      <exclusions>
-        <exclusion>
-          <artifactId>c3p0</artifactId>
-          <groupId>c3p0</groupId>
-        </exclusion>
-      </exclusions>
-    </dependency>
+        <dependency>
+            <groupId>org.quartz-scheduler</groupId>
+            <artifactId>quartz</artifactId>
+            <exclusions>
+              <exclusion>
+                <groupId>com.mchange</groupId>
+                <artifactId>c3p0</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.mchange</groupId>
+                <artifactId>mchange-commons-java</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.zaxxer</groupId>
+                <artifactId>HikariCP-java6</artifactId>
+              </exclusion>
+            </exclusions>
+        </dependency>
 
     <dependency>
       <groupId>org.quartz-scheduler</groupId>

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java

@@ -217,7 +217,7 @@ public enum Status {
     DATA_IS_NOT_VALID(50017,"data {0} not valid", "数据[{0}]无效"),
     DATA_IS_NULL(50018,"data {0} is null", "数据[{0}]不能为空"),
     PROCESS_NODE_HAS_CYCLE(50019,"process node has cycle", "流程节点间存在循环依赖"),
-    PROCESS_NODE_S_PARAMETER_INVALID(50020,"process node %s parameter invalid", "流程节点[%s]参数无效"),
+    PROCESS_NODE_S_PARAMETER_INVALID(50020,"process node {0} parameter invalid", "流程节点[{0}]参数无效"),
     PROCESS_DEFINE_STATE_ONLINE(50021, "process definition {0} is already on line", "工作流定义[{0}]已上线"),
     DELETE_PROCESS_DEFINE_BY_ID_ERROR(50022,"delete process definition by id error", "删除工作流定义错误"),
     SCHEDULE_CRON_STATE_ONLINE(50023,"the status of schedule {0} is already on line", "调度配置[{0}]已上线"),

dolphinscheduler-common/pom.xml

@@ -21,7 +21,7 @@
 	<parent>
 		<groupId>org.apache.dolphinscheduler</groupId>
 		<artifactId>dolphinscheduler</artifactId>
-		<version>1.3.4-SNAPSHOT</version>
+		<version>1.3.5-SNAPSHOT</version>
 	</parent>
 	<artifactId>dolphinscheduler-common</artifactId>
 	<name>dolphinscheduler-common</name>

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@@ -215,6 +215,11 @@ public final class Constants {
      */
     public static final String COLON = ":";
 
+    /**
+     * SPACE " "
+     */
+    public static final String SPACE = " ";
+
     /**
      * SINGLE_SLASH /
      */
@@ -225,6 +230,15 @@ public final class Constants {
      */
     public static final String DOUBLE_SLASH = "//";
 
+    /**
+     * SINGLE_QUOTES "'"
+     */
+    public static final String SINGLE_QUOTES = "'";
+    /**
+     * DOUBLE_QUOTES "\""
+     */
+    public static final String DOUBLE_QUOTES = "\"";
+
     /**
      * SEMICOLON ;
      */
@@ -970,4 +984,14 @@ public final class Constants {
     public static final int ABNORMAL_NODE_STATUS = 1;
 
 
+    /**
+     * exec shell scripts
+     */
+    public static final String SH = "sh";
+
+    /**
+     * pstree, get pud and sub pid
+     */
+    public static final String PSTREE = "pstree";
+
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopJobType.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.common.enums;
+
+import com.baomidou.mybatisplus.annotation.EnumValue;
+
+public enum  SqoopJobType {
+    CUSTOM(0, "CUSTOM"),
+    TEMPLATE(1, "TEMPLATE");
+
+    SqoopJobType(int code, String descp){
+        this.code = code;
+        this.descp = descp;
+    }
+
+    @EnumValue
+    private final int code;
+    private final String descp;
+
+    public int getCode() {
+        return code;
+    }
+
+    public String getDescp() {
+        return descp;
+    }
+}

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopQueryType.java

@@ -14,20 +14,28 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.common.enums;
 
-public enum QueryType {
+public enum SqoopQueryType {
 
-    FORM,
-    SQL;
+    FORM(0, "SQOOP_QUERY_FORM"),
+    SQL(1, "SQOOP_QUERY_SQL");
 
-    public static QueryType getEnum(int value){
-        for (QueryType e:QueryType.values()) {
-            if(e.ordinal() == value) {
-                return e;
-            }
-        }
-        //For values out of enum scope
-        return null;
+    private final Integer code;
+
+    private final String desc;
+
+    SqoopQueryType(Integer code, String desc) {
+        this.code = code;
+        this.desc = desc;
+    }
+
+    public Integer getCode() {
+        return code;
+    }
+
+    public String getDesc() {
+        return desc;
     }
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java

@@ -78,7 +78,7 @@ public abstract class AbstractShell {
   /**
    * If or not script finished executing
    */
-  private volatile AtomicBoolean completed;
+  private AtomicBoolean completed;
   
   public AbstractShell() {
     this(0L);
@@ -128,7 +128,7 @@ public abstract class AbstractShell {
   /**
    * Run a command   actual work
    */
-  private void runCommand() throws IOException { 
+  private void runCommand() throws IOException {
     ProcessBuilder builder = new ProcessBuilder(getExecString());
     Timer timeOutTimer = null;
     ShellTimeoutTimerTask timeoutTimerTask = null;
@@ -153,11 +153,11 @@ public abstract class AbstractShell {
       timeOutTimer.schedule(timeoutTimerTask, timeOutInterval);
     }
     final BufferedReader errReader = 
-            new BufferedReader(new InputStreamReader(process
-                                                     .getErrorStream()));
-    BufferedReader inReader = 
-            new BufferedReader(new InputStreamReader(process
-                                                     .getInputStream()));
+            new BufferedReader(
+                    new InputStreamReader(process.getErrorStream()));
+    BufferedReader inReader =
+            new BufferedReader(
+                    new InputStreamReader(process.getInputStream()));
     final StringBuilder errMsg = new StringBuilder();
     
     // read error and input streams as this would free up the buffers
@@ -177,23 +177,35 @@ public abstract class AbstractShell {
         }
       }
     };
+    Thread inThread = new Thread() {
+      @Override
+      public void run() {
+        try {
+          parseExecResult(inReader);
+        } catch (IOException ioe) {
+          logger.warn("Error reading the in stream", ioe);
+        }
+        super.run();
+      }
+    };
     try {
       errThread.start();
+      inThread.start();
     } catch (IllegalStateException ise) { }
     try {
       // parse the output
-      parseExecResult(inReader);
-      exitCode  = process.waitFor();
+      exitCode = process.waitFor();
       try {
-        // make sure that the error thread exits
+        // make sure that the error and in thread exits
         errThread.join();
+        inThread.join();
       } catch (InterruptedException ie) {
-        logger.warn("Interrupted while reading the error stream", ie);
+        logger.warn("Interrupted while reading the error and in stream", ie);
       }
-      completed.set(true);
+      completed.compareAndSet(false,true);
       //the timeout thread handling
       //taken care in finally block
-      if (exitCode != 0) {
+      if (exitCode != 0 || errMsg.length() > 0) {
         throw new ExitCodeException(exitCode, errMsg.toString());
       }
     } catch (InterruptedException ie) {

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/SqoopParameters.java

@@ -16,6 +16,8 @@
  */
 package org.apache.dolphinscheduler.common.task.sqoop;
 
+import org.apache.dolphinscheduler.common.enums.SqoopJobType;
+import org.apache.dolphinscheduler.common.process.Property;
 import org.apache.dolphinscheduler.common.process.ResourceInfo;
 import org.apache.dolphinscheduler.common.task.AbstractParameters;
 import org.apache.dolphinscheduler.common.utils.StringUtils;
@@ -28,6 +30,23 @@ import java.util.List;
  */
 public class SqoopParameters  extends AbstractParameters {
 
+    /**
+     * sqoop job type:
+     * CUSTOM - custom sqoop job
+     * TEMPLATE - sqoop template job
+     */
+    private String jobType;
+
+    /**
+     * customJob eq 1, use customShell
+     */
+    private String customShell;
+
+    /**
+     * sqoop job name - map-reduce job name
+     */
+    private String jobName;
+
     /**
      * model type
      */
@@ -53,6 +72,16 @@ public class SqoopParameters  extends AbstractParameters {
      */
     private String targetParams;
 
+    /**
+     * hadoop custom param for sqoop job
+     */
+    private List<Property> hadoopCustomParams;
+
+    /**
+     * sqoop advanced param
+     */
+    private List<Property> sqoopAdvancedParams;
+
     public String getModelType() {
         return modelType;
     }
@@ -101,18 +130,74 @@ public class SqoopParameters  extends AbstractParameters {
         this.targetParams = targetParams;
     }
 
+    public String getJobType() {
+        return jobType;
+    }
+
+    public void setJobType(String jobType) {
+        this.jobType = jobType;
+    }
+
+    public String getJobName() {
+        return jobName;
+    }
+
+    public void setJobName(String jobName) {
+        this.jobName = jobName;
+    }
+
+    public String getCustomShell() {
+        return customShell;
+    }
+
+    public void setCustomShell(String customShell) {
+        this.customShell = customShell;
+    }
+
+    public List<Property> getHadoopCustomParams() {
+        return hadoopCustomParams;
+    }
+
+    public void setHadoopCustomParams(List<Property> hadoopCustomParams) {
+        this.hadoopCustomParams = hadoopCustomParams;
+    }
+
+    public List<Property> getSqoopAdvancedParams() {
+        return sqoopAdvancedParams;
+    }
+
+    public void setSqoopAdvancedParams(List<Property> sqoopAdvancedParams) {
+        this.sqoopAdvancedParams = sqoopAdvancedParams;
+    }
+
     @Override
     public boolean checkParameters() {
-        return StringUtils.isNotEmpty(modelType)&&
-                concurrency != 0 &&
-                StringUtils.isNotEmpty(sourceType)&&
-                StringUtils.isNotEmpty(targetType)&&
-                StringUtils.isNotEmpty(sourceParams)&&
-                StringUtils.isNotEmpty(targetParams);
+
+        boolean sqoopParamsCheck = false;
+
+        if (StringUtils.isEmpty(jobType)) {
+            return sqoopParamsCheck;
+        }
+
+        if (SqoopJobType.TEMPLATE.getDescp().equals(jobType)) {
+            sqoopParamsCheck = StringUtils.isEmpty(customShell) &&
+                    StringUtils.isNotEmpty(modelType) &&
+                    StringUtils.isNotEmpty(jobName) &&
+                    concurrency != 0 &&
+                    StringUtils.isNotEmpty(sourceType) &&
+                    StringUtils.isNotEmpty(targetType) &&
+                    StringUtils.isNotEmpty(sourceParams) &&
+                    StringUtils.isNotEmpty(targetParams);
+        } else if (SqoopJobType.CUSTOM.getDescp().equals(jobType)) {
+            sqoopParamsCheck = StringUtils.isNotEmpty(customShell) &&
+                    StringUtils.isEmpty(jobName);
+        }
+
+        return sqoopParamsCheck;
     }
 
     @Override
     public List<ResourceInfo> getResourceFilesList() {
-       return new ArrayList<>();
+        return new ArrayList<>();
     }
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/targets/TargetHiveParameter.java

@@ -41,6 +41,11 @@ public class TargetHiveParameter {
      * hive overwrite
      */
     private boolean hiveOverWrite;
+
+    /**
+     * hive target dir
+     */
+    private String hiveTargetDir;
     /**
      * replace delimiter
      */
@@ -117,4 +122,12 @@ public class TargetHiveParameter {
     public void setHivePartitionValue(String hivePartitionValue) {
         this.hivePartitionValue = hivePartitionValue;
     }
+
+    public String getHiveTargetDir() {
+        return hiveTargetDir;
+    }
+
+    public void setHiveTargetDir(String hiveTargetDir) {
+        this.hiveTargetDir = hiveTargetDir;
+    }
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/targets/TargetMysqlParameter.java

@@ -106,7 +106,7 @@ public class TargetMysqlParameter {
         this.preQuery = preQuery;
     }
 
-    public boolean isUpdate() {
+    public boolean getIsUpdate() {
         return isUpdate;
     }
 

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java

@@ -16,18 +16,32 @@
  */
 package org.apache.dolphinscheduler.common.utils;
 
+import static org.apache.dolphinscheduler.common.Constants.DATA_BASEDIR_PATH;
+import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS;
+import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE;
+import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.StringReader;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
+import java.util.Optional;
+
 import org.apache.commons.io.Charsets;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.*;
-import java.nio.charset.Charset;
-import java.nio.charset.UnsupportedCharsetException;
-
-import static org.apache.dolphinscheduler.common.Constants.*;
-
 /**
  * file utils
  */
@@ -36,6 +50,8 @@ public class FileUtils {
 
     public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH,"/tmp/dolphinscheduler");
 
+    public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>();
+
     /**
      * get file suffix
      *
@@ -118,7 +134,7 @@ public class FileUtils {
         String fileName = String.format("%s/exec/process/%s/%s/%s", DATA_BASEDIR, Integer.toString(projectId),
                 Integer.toString(processDefineId), Integer.toString(processInstanceId));
         File file = new File(fileName);
-        if (!file.getParentFile().exists()){
+        if (!file.getParentFile().exists()) {
             file.getParentFile().mkdirs();
         }
 
@@ -138,24 +154,40 @@ public class FileUtils {
      * @param userName user name
      * @throws IOException errors
      */
-    public static void createWorkDirAndUserIfAbsent(String execLocalPath, String userName) throws IOException{
+    public static void createWorkDirAndUserIfAbsent(String execLocalPath, String userName) throws IOException {
         //if work dir exists, first delete
         File execLocalPathFile = new File(execLocalPath);
 
-        if (execLocalPathFile.exists()){
+        if (execLocalPathFile.exists()) {
             org.apache.commons.io.FileUtils.forceDelete(execLocalPathFile);
         }
 
         //create work dir
         org.apache.commons.io.FileUtils.forceMkdir(execLocalPathFile);
-        logger.info("create dir success {}" , execLocalPath);
-
+        String mkdirLog = "create dir success " + execLocalPath;
+        LoggerUtils.logInfo(Optional.ofNullable(logger), mkdirLog);
+        LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), mkdirLog);
 
         //if not exists this user,then create
-        if (!OSUtils.getUserList().contains(userName)){
-            OSUtils.createUser(userName);
+        OSUtils.taskLoggerThreadLocal.set(taskLoggerThreadLocal.get());
+        try {
+            if (!OSUtils.getUserList().contains(userName)) {
+                boolean isSuccessCreateUser = OSUtils.createUser(userName);
+
+                String infoLog;
+                if (isSuccessCreateUser) {
+                    infoLog = String.format("create user name success %s", userName);
+                } else {
+                    infoLog = String.format("create user name fail %s", userName);
+                }
+                LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog);
+                LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog);
+            }
+        } catch (Throwable e) {
+            LoggerUtils.logError(Optional.ofNullable(logger), e);
+            LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), e);
         }
-        logger.info("create user name success {}", userName);
+        OSUtils.taskLoggerThreadLocal.remove();
     }
 
 

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java

@@ -21,6 +21,7 @@ import org.slf4j.Logger;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -93,4 +94,24 @@ public class LoggerUtils {
         }
         return appIds;
     }
+
+    public static void logError(Optional<Logger> optionalLogger
+            , String error) {
+        optionalLogger.ifPresent((Logger logger) -> logger.error(error));
+    }
+
+    public static void logError(Optional<Logger> optionalLogger
+            , Throwable e) {
+        optionalLogger.ifPresent((Logger logger) -> logger.error(e.getMessage(), e));
+    }
+
+    public static void logError(Optional<Logger> optionalLogger
+            , String error, Throwable e) {
+        optionalLogger.ifPresent((Logger logger) -> logger.error(error, e));
+    }
+
+    public static void logInfo(Optional<Logger> optionalLogger
+            , String info) {
+        optionalLogger.ifPresent((Logger logger) -> logger.info(info));
+    }
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java

@@ -16,9 +16,9 @@
  */
 package org.apache.dolphinscheduler.common.utils;
 
+import org.apache.commons.configuration.Configuration;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.shell.ShellExecutor;
-import org.apache.commons.configuration.Configuration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import oshi.SystemInfo;
@@ -36,10 +36,7 @@ import java.math.RoundingMode;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.text.DecimalFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
 import java.util.regex.Pattern;
 
 /**
@@ -50,6 +47,8 @@ public class OSUtils {
 
   private static final Logger logger = LoggerFactory.getLogger(OSUtils.class);
 
+  public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>();
+
   private static final SystemInfo SI = new SystemInfo();
   public static final String TWO_DECIMAL = "0.00";
 
@@ -236,7 +235,9 @@ public class OSUtils {
     try {
       String userGroup = OSUtils.getGroup();
       if (StringUtils.isEmpty(userGroup)) {
-        logger.error("{} group does not exist for this operating system.", userGroup);
+        String errorLog = String.format("%s group does not exist for this operating system.", userGroup);
+        LoggerUtils.logError(Optional.ofNullable(logger), errorLog);
+        LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), errorLog);
         return false;
       }
       if (isMacOS()) {
@@ -248,7 +249,8 @@ public class OSUtils {
       }
       return true;
     } catch (Exception e) {
-      logger.error(e.getMessage(), e);
+      LoggerUtils.logError(Optional.ofNullable(logger), e);
+      LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), e);
     }
 
     return false;
@@ -261,10 +263,14 @@ public class OSUtils {
    * @throws IOException in case of an I/O error
    */
   private static void createLinuxUser(String userName, String userGroup) throws IOException {
-    logger.info("create linux os user : {}", userName);
-    String cmd = String.format("sudo useradd -g %s %s", userGroup, userName);
+    String infoLog1 = String.format("create linux os user : %s", userName);
+    LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1);
+    LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1);
 
-    logger.info("execute cmd : {}", cmd);
+    String cmd = String.format("sudo useradd -g %s %s", userGroup, userName);
+    String infoLog2 = String.format("execute cmd : %s", cmd);
+    LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2);
+    LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2);
     OSUtils.exeCmd(cmd);
   }
 
@@ -275,13 +281,24 @@ public class OSUtils {
    * @throws IOException in case of an I/O error
    */
   private static void createMacUser(String userName, String userGroup) throws IOException {
-    logger.info("create mac os user : {}", userName);
-    String userCreateCmd = String.format("sudo sysadminctl -addUser %s -password %s", userName, userName);
-    String appendGroupCmd = String.format("sudo dseditgroup -o edit -a %s -t user %s", userName, userGroup);
 
-    logger.info("create user command : {}", userCreateCmd);
-    OSUtils.exeCmd(userCreateCmd);
-    logger.info("append user to group : {}", appendGroupCmd);
+    Optional<Logger> optionalLogger = Optional.ofNullable(logger);
+    Optional<Logger> optionalTaskLogger = Optional.ofNullable(taskLoggerThreadLocal.get());
+
+    String infoLog1 = String.format("create mac os user : %s", userName);
+    LoggerUtils.logInfo(optionalLogger, infoLog1);
+    LoggerUtils.logInfo(optionalTaskLogger, infoLog1);
+
+    String createUserCmd = String.format("sudo sysadminctl -addUser %s -password %s", userName, userName);
+    String infoLog2 = String.format("create user command : %s", createUserCmd);
+    LoggerUtils.logInfo(optionalLogger, infoLog2);
+    LoggerUtils.logInfo(optionalTaskLogger, infoLog2);
+    OSUtils.exeCmd(createUserCmd);
+
+    String appendGroupCmd = String.format("sudo dseditgroup -o edit -a %s -t user %s", userName, userGroup);
+    String infoLog3 = String.format("append user to group : %s", appendGroupCmd);
+    LoggerUtils.logInfo(optionalLogger, infoLog3);
+    LoggerUtils.logInfo(optionalTaskLogger, infoLog3);
     OSUtils.exeCmd(appendGroupCmd);
   }
 
@@ -292,14 +309,20 @@ public class OSUtils {
    * @throws IOException in case of an I/O error
    */
   private static void createWindowsUser(String userName, String userGroup) throws IOException {
-    logger.info("create windows os user : {}", userName);
-    String userCreateCmd = String.format("net user \"%s\" /add", userName);
-    String appendGroupCmd = String.format("net localgroup \"%s\" \"%s\" /add", userGroup, userName);
+    String infoLog1 = String.format("create windows os user : %s", userName);
+    LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1);
+    LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1);
 
-    logger.info("execute create user command : {}", userCreateCmd);
+    String userCreateCmd = String.format("net user \"%s\" /add", userName);
+    String infoLog2 = String.format("execute create user command : %s", userCreateCmd);
+    LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2);
+    LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2);
     OSUtils.exeCmd(userCreateCmd);
 
-    logger.info("execute append user to group : {}", appendGroupCmd);
+    String appendGroupCmd = String.format("net localgroup \"%s\" \"%s\" /add", userGroup, userName);
+    String infoLog3 = String.format("execute append user to group : %s", appendGroupCmd);
+    LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog3);
+    LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog3);
     OSUtils.exeCmd(appendGroupCmd);
   }
 
@@ -338,22 +361,12 @@ public class OSUtils {
    * @throws IOException errors
    */
   public static String exeCmd(String command) throws IOException {
-    BufferedReader br = null;
-
-    try {
-      Process p = Runtime.getRuntime().exec(command);
-      br = new BufferedReader(new InputStreamReader(p.getInputStream()));
-      String line;
-      StringBuilder sb = new StringBuilder();
-
-      while ((line = br.readLine()) != null) {
-        sb.append(line + "\n");
-      }
-
-      return sb.toString();
-    } finally {
-      IOUtils.closeQuietly(br);
+    StringTokenizer st = new StringTokenizer(command);
+    String[] cmdArray = new String[st.countTokens()];
+    for (int i = 0; st.hasMoreTokens(); i++) {
+      cmdArray[i] = st.nextToken();
     }
+    return exeShell(cmdArray);
   }
 
   /**
@@ -362,7 +375,7 @@ public class OSUtils {
    * @return result of execute the shell
    * @throws IOException errors
    */
-  public static String exeShell(String command) throws IOException {
+  public static String exeShell(String[] command) throws IOException {
     return ShellExecutor.execCommand(command);
   }
 

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java

@@ -14,9 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.common.utils;
 
-import com.alibaba.fastjson.JSON;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.CommandType;
 import org.apache.dolphinscheduler.common.enums.DataType;
@@ -24,233 +24,255 @@ import org.apache.dolphinscheduler.common.process.Property;
 import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
 import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
 import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.time.DateUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.sql.PreparedStatement;
-import java.text.ParseException;
-import java.util.*;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * parameter parse utils
  */
 public class ParameterUtils {
 
-  private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class);
+    private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class);
 
-  /**
-   * convert parameters place holders
-   *
-   * @param parameterString parameter
-   * @param parameterMap parameter map
-   * @return convert parameters place holders
-   */
-  public static String convertParameterPlaceholders(String parameterString, Map<String, String> parameterMap) {
-    if (StringUtils.isEmpty(parameterString) || parameterMap == null) {
-      return parameterString;
+    private static final String DATE_PARSE_PATTERN = "\\$\\[([^\\]]+)]";
+
+    private static final String DATE_START_PATTERN = "^[0-9]";
+
+    private ParameterUtils() {
+        throw new UnsupportedOperationException("Construct ParameterUtils");
     }
 
-    //Get current time, schedule execute time
-    String cronTimeStr = parameterMap.get(Constants.PARAMETER_DATETIME);
-
-    Date cronTime = null;
-
-    if (StringUtils.isNotEmpty(cronTimeStr)) {
-      try {
-        cronTime = DateUtils.parseDate(cronTimeStr, new String[]{Constants.PARAMETER_FORMAT_TIME});
-      } catch (ParseException e) {
-        logger.error("parse {} exception", cronTimeStr, e);
-      }
-    } else {
-      cronTime = new Date();
+    /**
+     * convert parameters place holders
+     *
+     * @param parameterString parameter
+     * @param parameterMap    parameter map
+     * @return convert parameters place holders
+     */
+    public static String convertParameterPlaceholders(String parameterString, Map<String, String> parameterMap) {
+        if (StringUtils.isEmpty(parameterString)) {
+            return parameterString;
+        }
+        Date cronTime;
+        if (parameterMap != null && !parameterMap.isEmpty()) {
+            // replace variable ${} form,refers to the replacement of system variables and custom variables
+            parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
+        }
+        if (parameterMap != null && null != parameterMap.get(Constants.PARAMETER_DATETIME)) {
+            //Get current time, schedule execute time
+            String cronTimeStr = parameterMap.get(Constants.PARAMETER_DATETIME);
+            cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME);
+        } else {
+            cronTime = new Date();
+        }
+        // replace time $[...] form, eg. $[yyyyMMdd]
+        if (cronTime != null) {
+            return dateTemplateParse(parameterString, cronTime);
+        }
+        return parameterString;
     }
 
-    // replace variable ${} form,refers to the replacement of system variables and custom variables
-    parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
+    /**
+     * new
+     * convert parameters place holders
+     *
+     * @param parameterString parameter
+     * @param parameterMap    parameter map
+     * @return convert parameters place holders
+     */
+    public static String convertParameterPlaceholders2(String parameterString, Map<String, String> parameterMap) {
+        if (StringUtils.isEmpty(parameterString)) {
+            return parameterString;
+        }
+        //Get current time, schedule execute time
+        String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME);
+        Date cronTime = null;
 
-    // replace time $[...] form, eg. $[yyyyMMdd]
-    if (cronTime != null) {
-      parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true);
+        if (StringUtils.isNotEmpty(cronTimeStr)) {
+            cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME);
+
+        } else {
+            cronTime = new Date();
+        }
+
+        // replace variable ${} form,refers to the replacement of system variables and custom variables
+        parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
+
+        // replace time $[...] form, eg. $[yyyyMMdd]
+        if (cronTime != null) {
+            return dateTemplateParse(parameterString, cronTime);
+        }
+        return parameterString;
     }
 
-    return parameterString;
-  }
-
-  /**
-   * new
-   * convert parameters place holders
-   *
-   * @param parameterString parameter
-   * @param parameterMap parameter map
-   * @return convert parameters place holders
-   */
-  public static String convertParameterPlaceholders2(String parameterString, Map<String, String> parameterMap) {
-    if (StringUtils.isEmpty(parameterString)) {
-      return parameterString;
-    }
-    //Get current time, schedule execute time
-    String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME);
-    Date cronTime = null;
-
-    if (StringUtils.isNotEmpty(cronTimeStr)) {
-      try {
-        cronTime = DateUtils.parseDate(cronTimeStr, new String[]{Constants.PARAMETER_FORMAT_TIME});
-
-      } catch (ParseException e) {
-        logger.error(String.format("parse %s exception", cronTimeStr), e);
-      }
-    } else {
-      cronTime = new Date();
+    /**
+     * set in parameter
+     *
+     * @param index    index
+     * @param stmt     preparedstatement
+     * @param dataType data type
+     * @param value    value
+     * @throws Exception errors
+     */
+    public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value) throws Exception {
+        if (dataType.equals(DataType.VARCHAR)) {
+            stmt.setString(index, value);
+        } else if (dataType.equals(DataType.INTEGER)) {
+            stmt.setInt(index, Integer.parseInt(value));
+        } else if (dataType.equals(DataType.LONG)) {
+            stmt.setLong(index, Long.parseLong(value));
+        } else if (dataType.equals(DataType.FLOAT)) {
+            stmt.setFloat(index, Float.parseFloat(value));
+        } else if (dataType.equals(DataType.DOUBLE)) {
+            stmt.setDouble(index, Double.parseDouble(value));
+        } else if (dataType.equals(DataType.DATE)) {
+            stmt.setDate(index, java.sql.Date.valueOf(value));
+        } else if (dataType.equals(DataType.TIME)) {
+            stmt.setString(index, value);
+        } else if (dataType.equals(DataType.TIMESTAMP)) {
+            stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value));
+        } else if (dataType.equals(DataType.BOOLEAN)) {
+            stmt.setBoolean(index, Boolean.parseBoolean(value));
+        }
     }
 
-    // replace variable ${} form,refers to the replacement of system variables and custom variables
-    parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
+    /**
+     * curing user define parameters
+     *
+     * @param globalParamMap  global param map
+     * @param globalParamList global param list
+     * @param commandType     command type
+     * @param scheduleTime    schedule time
+     * @return curing user define parameters
+     */
+    public static String curingGlobalParams(Map<String, String> globalParamMap, List<Property> globalParamList,
+                                            CommandType commandType, Date scheduleTime) {
 
-    // replace time $[...] form, eg. $[yyyyMMdd]
-    if (cronTime != null) {
-      parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true);
+        if (globalParamList == null || globalParamList.isEmpty()) {
+            return null;
+        }
 
-    }
-    return parameterString;
-  }
-
-
-  /**
-   *  set in parameter
-   * @param index index
-   * @param stmt preparedstatement
-   * @param dataType data type
-   * @param value value
-   * @throws Exception errors
-   */
-  public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value)throws Exception{
-    if (dataType.equals(DataType.VARCHAR)){
-      stmt.setString(index,value);
-    }else if (dataType.equals(DataType.INTEGER)){
-      stmt.setInt(index, Integer.parseInt(value));
-    }else if (dataType.equals(DataType.LONG)){
-      stmt.setLong(index, Long.parseLong(value));
-    }else if (dataType.equals(DataType.FLOAT)){
-      stmt.setFloat(index, Float.parseFloat(value));
-    }else if (dataType.equals(DataType.DOUBLE)){
-      stmt.setDouble(index, Double.parseDouble(value));
-    }else if (dataType.equals(DataType.DATE)){
-      stmt.setDate(index, java.sql.Date.valueOf(value));
-    }else if (dataType.equals(DataType.TIME)){
-      stmt.setString(index, value);
-    }else if (dataType.equals(DataType.TIMESTAMP)){
-      stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value));
-    }else if (dataType.equals(DataType.BOOLEAN)){
-      stmt.setBoolean(index,Boolean.parseBoolean(value));
-    }
-  }
-
-  /**
-   * curing user define parameters
-   *
-   * @param globalParamMap global param map
-   * @param globalParamList global param list
-   * @param commandType command type
-   * @param scheduleTime schedule time
-   * @return curing user define parameters
-   */
-  public static String curingGlobalParams(Map<String,String> globalParamMap, List<Property> globalParamList,
-                                   CommandType commandType, Date scheduleTime){
-
-    if (globalParamList == null || globalParamList.isEmpty()) {
-      return null;
-    }
-
-    Map<String, String> globalMap = new HashMap<>();
-    if (globalParamMap!= null){
-      globalMap.putAll(globalParamMap);
-    }
-    Map<String,String> allParamMap = new HashMap<>();
-    //If it is a complement, a complement time needs to be passed in, according to the task type
-    Map<String,String> timeParams = BusinessTimeUtils
+        Map<String, String> globalMap = new HashMap<>();
+        if (globalParamMap != null) {
+            globalMap.putAll(globalParamMap);
+        }
+        Map<String, String> allParamMap = new HashMap<>();
+        //If it is a complement, a complement time needs to be passed in, according to the task type
+        Map<String, String> timeParams = BusinessTimeUtils
             .getBusinessTime(commandType, scheduleTime);
 
-    if (timeParams != null) {
-      allParamMap.putAll(timeParams);
+        if (timeParams != null) {
+            allParamMap.putAll(timeParams);
+        }
+
+        allParamMap.putAll(globalMap);
+
+        Set<Map.Entry<String, String>> entries = allParamMap.entrySet();
+
+        Map<String, String> resolveMap = new HashMap<>();
+        for (Map.Entry<String, String> entry : entries) {
+            String val = entry.getValue();
+            if (val.startsWith("$")) {
+                String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap);
+                resolveMap.put(entry.getKey(), str);
+            }
+        }
+        globalMap.putAll(resolveMap);
+
+        for (Property property : globalParamList) {
+            String val = globalMap.get(property.getProp());
+            if (val != null) {
+                property.setValue(val);
+            }
+        }
+        return JSONUtils.toJsonString(globalParamList);
     }
 
-    allParamMap.putAll(globalMap);
+    /**
+     * handle escapes
+     *
+     * @param inputString input string
+     * @return string filter escapes
+     */
+    public static String handleEscapes(String inputString) {
 
-    Set<Map.Entry<String, String>> entries = allParamMap.entrySet();
-
-    Map<String,String> resolveMap = new HashMap<>();
-    for (Map.Entry<String,String> entry : entries){
-      String val = entry.getValue();
-      if (val.startsWith("$")){
-        String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap);
-        resolveMap.put(entry.getKey(),str);
-      }
+        if (StringUtils.isNotEmpty(inputString)) {
+            return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_");
+        }
+        return inputString;
     }
-    globalMap.putAll(resolveMap);
 
-    for (Property property : globalParamList){
-      String val = globalMap.get(property.getProp());
-      if (val != null){
-        property.setValue(val);
-      }
+    /**
+     * $[yyyyMMdd] replace schedule time
+     */
+    public static String replaceScheduleTime(String text, Date scheduleTime) {
+        Map<String, Property> paramsMap = new HashMap<>();
+        //if getScheduleTime null ,is current date
+        if (null == scheduleTime) {
+            scheduleTime = new Date();
+        }
+
+        String dateTime = org.apache.dolphinscheduler.common.utils.DateUtils.format(scheduleTime, Constants.PARAMETER_FORMAT_TIME);
+        Property p = new Property();
+        p.setValue(dateTime);
+        p.setProp(Constants.PARAMETER_SHECDULE_TIME);
+        paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
+        text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap));
+
+        return text;
     }
-    return JSON.toJSONString(globalParamList);
-  }
 
-
-  /**
-   * handle escapes
-   * @param inputString input string
-   * @return string filter escapes
-   */
-  public static String handleEscapes(String inputString){
-
-    if(StringUtils.isNotEmpty(inputString)){
-      return inputString.replace("%", "////%");
+    /**
+     * format convert
+     *
+     * @param paramsMap params map
+     * @return Map of converted
+     * see org.apache.dolphinscheduler.server.utils.ParamUtils.convert
+     */
+    public static Map<String, String> convert(Map<String, Property> paramsMap) {
+        Map<String, String> map = new HashMap<>();
+        Iterator<Map.Entry<String, Property>> iter = paramsMap.entrySet().iterator();
+        while (iter.hasNext()) {
+            Map.Entry<String, Property> en = iter.next();
+            map.put(en.getKey(), en.getValue().getValue());
+        }
+        return map;
     }
-    return inputString;
-  }
 
+    private static String dateTemplateParse(String templateStr, Date date) {
+        if (templateStr == null) {
+            return null;
+        }
+        Pattern pattern = Pattern.compile(DATE_PARSE_PATTERN);
 
-  /**
-   * $[yyyyMMdd] replace schedule time
-   * @param text
-   * @param scheduleTime
-   * @return
-   */
-  public static String replaceScheduleTime(String text, Date scheduleTime) {
-      Map<String, Property> paramsMap = new HashMap<>();
-      //if getScheduleTime null ,is current date
-      if (null == scheduleTime) {
-        scheduleTime = new Date();
-      }
+        StringBuffer newValue = new StringBuffer(templateStr.length());
 
-      String dateTime = org.apache.dolphinscheduler.common.utils.DateUtils.format(scheduleTime, Constants.PARAMETER_FORMAT_TIME);
-      Property p = new Property();
-      p.setValue(dateTime);
-      p.setProp(Constants.PARAMETER_SHECDULE_TIME);
-      paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
-      text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap));
+        Matcher matcher = pattern.matcher(templateStr);
 
-      return text;
-  }
+        while (matcher.find()) {
+            String key = matcher.group(1);
+            if (Pattern.matches(DATE_START_PATTERN, key)) {
+                continue;
+            }
+            String value = TimePlaceholderUtils.getPlaceHolderTime(key, date);
+            assert value != null;
+            matcher.appendReplacement(newValue, value);
+        }
 
+        matcher.appendTail(newValue);
 
-  /**
-   * format convert
-   * @param paramsMap params map
-   * @return Map of converted
-   * see org.apache.dolphinscheduler.server.utils.ParamUtils.convert
-   */
-  public static Map<String,String> convert(Map<String,Property> paramsMap){
-    Map<String,String> map = new HashMap<>();
-    Iterator<Map.Entry<String, Property>> iter = paramsMap.entrySet().iterator();
-    while (iter.hasNext()){
-      Map.Entry<String, Property> en = iter.next();
-      map.put(en.getKey(),en.getValue().getValue());
+        return newValue.toString();
     }
-    return map;
-  }
+
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java

@@ -14,11 +14,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.common.utils;
 
 public class StringUtils {
+
     public static final String EMPTY = "";
 
+    private StringUtils() {
+        throw new UnsupportedOperationException("Construct StringUtils");
+    }
+
     public static boolean isEmpty(final CharSequence cs) {
         return cs == null || cs.length() == 0;
     }
@@ -27,14 +33,26 @@ public class StringUtils {
         return !isEmpty(cs);
     }
 
-    public static boolean isBlank(String s){
+    public static boolean isBlank(String s) {
         if (isEmpty(s)) {
             return true;
         }
         return s.trim().length() == 0;
     }
 
-    public static boolean isNotBlank(String s){
+    public static boolean isNotBlank(String s) {
         return !isBlank(s);
     }
+
+    public static String replaceNRTtoUnderline(String src) {
+        if (isBlank(src)) {
+            return src;
+        } else {
+            return src.replaceAll("[\n|\r|\t]", "_");
+        }
+    }
+
+    public static String trim(String str) {
+        return str == null ? null : str.trim();
+    }
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java

@@ -14,11 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.common.utils.placeholder;
 
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.utils.DateUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -46,8 +47,8 @@ public class TimePlaceholderUtils {
      * Replaces all placeholders of format {@code ${name}} with the value returned
      * from the supplied {@link PropertyPlaceholderHelper.PlaceholderResolver}.
      *
-     * @param value                 the value containing the placeholders to be replaced
-     * @param date                  custom date
+     * @param value                          the value containing the placeholders to be replaced
+     * @param date                           custom date
      * @param ignoreUnresolvablePlaceholders ignore unresolvable placeholders
      * @return the supplied value with placeholders replaced inline
      */
@@ -59,11 +60,11 @@ public class TimePlaceholderUtils {
         return helper.replacePlaceholders(value, new TimePlaceholderResolver(value, date));
     }
 
-
     /**
      * Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix.
+     *
      * @param ignoreUnresolvablePlaceholders indicates whether unresolvable placeholders should
-     * be ignored ({@code true}) or cause an exception ({@code false})
+     *                                       be ignored ({@code true}) or cause an exception ({@code false})
      */
     private static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) {
         return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, ignoreUnresolvablePlaceholders);
@@ -89,7 +90,7 @@ public class TimePlaceholderUtils {
      * Change the sign in the expression to P (positive) N (negative)
      *
      * @param expression
-     * @return  eg. "-3+-6*(+8)-(-5) -> S3+S6*(P8)-(S5)"
+     * @return eg. "-3+-6*(+8)-(-5) -> S3+S6*(P8)-(S5)"
      */
     private static String convert(String expression) {
         char[] arr = expression.toCharArray();
@@ -262,7 +263,7 @@ public class TimePlaceholderUtils {
      * Placeholder replacement resolver
      */
     private static class TimePlaceholderResolver implements
-            PropertyPlaceholderHelper.PlaceholderResolver {
+        PropertyPlaceholderHelper.PlaceholderResolver {
 
         private final String value;
 
@@ -278,12 +279,28 @@ public class TimePlaceholderUtils {
             try {
                 return calculateTime(placeholderName, date);
             } catch (Exception ex) {
-                logger.error("resolve placeholder '{}' in [ {} ]" , placeholderName, value, ex);
+                logger.error("resolve placeholder '{}' in [ {} ]", placeholderName, value, ex);
                 return null;
             }
         }
     }
 
+    /**
+     * return the formatted date according to the corresponding date format
+     *
+     * @param expression date expression
+     * @param date       date
+     * @return reformat date
+     */
+    public static String getPlaceHolderTime(String expression, Date date) {
+        if (StringUtils.isBlank(expression)) {
+            return null;
+        }
+        if (null == date) {
+            return null;
+        }
+        return calculateTime(expression, date);
+    }
 
     /**
      * calculate time
@@ -320,9 +337,10 @@ public class TimePlaceholderUtils {
 
     /**
      * calculate time expresstion
+     *
      * @param expression expresstion
-     * @param date  date
-     * @return  map with date, date format
+     * @param date       date
+     * @return map with date, date format
      */
     public static Map.Entry<Date, String> calcTimeExpression(String expression, Date date) {
         Map.Entry<Date, String> resultEntry;
@@ -346,8 +364,9 @@ public class TimePlaceholderUtils {
 
     /**
      * get first day of month
+     *
      * @param expression expresstion
-     * @param date  date
+     * @param date       date
      * @return first day of month
      */
     public static Map.Entry<Date, String> calcMonthBegin(String expression, Date date) {
@@ -369,8 +388,9 @@ public class TimePlaceholderUtils {
 
     /**
      * get last day of month
+     *
      * @param expression expresstion
-     * @param date  date
+     * @param date       date
      * @return last day of month
      */
     public static Map.Entry<Date, String> calcMonthEnd(String expression, Date date) {
@@ -392,8 +412,9 @@ public class TimePlaceholderUtils {
 
     /**
      * get first day of week
+     *
      * @param expression expresstion
-     * @param date  date
+     * @param date       date
      * @return monday
      */
     public static Map.Entry<Date, String> calcWeekStart(String expression, Date date) {
@@ -414,8 +435,9 @@ public class TimePlaceholderUtils {
 
     /**
      * get last day of week
+     *
      * @param expression expresstion
-     * @param date  date
+     * @param date       date
      * @return last day of week
      */
     public static Map.Entry<Date, String> calcWeekEnd(String expression, Date date) {
@@ -437,8 +459,9 @@ public class TimePlaceholderUtils {
 
     /**
      * calc months expression
+     *
      * @param expression expresstion
-     * @param date  date
+     * @param date       date
      * @return calc months
      */
     public static Map.Entry<Date, String> calcMonths(String expression, Date date) {
@@ -461,7 +484,7 @@ public class TimePlaceholderUtils {
      * calculate time expression
      *
      * @param expression expresstion
-     * @param date  date
+     * @param date       date
      * @return calculate time expression with date,format
      */
     public static Map.Entry<Date, String> calcMinutes(String expression, Date date) {
@@ -471,7 +494,7 @@ public class TimePlaceholderUtils {
             if (Character.isDigit(expression.charAt(index + 1))) {
                 String addMinuteExpr = expression.substring(index + 1);
                 Date targetDate = org.apache.commons.lang.time.DateUtils
-                        .addMinutes(date, calcMinutes(addMinuteExpr));
+                    .addMinutes(date, calcMinutes(addMinuteExpr));
                 String dateFormat = expression.substring(0, index);
 
                 return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat);
@@ -482,7 +505,7 @@ public class TimePlaceholderUtils {
             if (Character.isDigit(expression.charAt(index + 1))) {
                 String addMinuteExpr = expression.substring(index + 1);
                 Date targetDate = org.apache.commons.lang.time.DateUtils
-                        .addMinutes(date, 0 - calcMinutes(addMinuteExpr));
+                    .addMinutes(date, 0 - calcMinutes(addMinuteExpr));
                 String dateFormat = expression.substring(0, index);
 
                 return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat);
@@ -512,7 +535,7 @@ public class TimePlaceholderUtils {
         } else {
 
             calcExpression = String.format("60*24*(%s)%s", minuteExpression.substring(0, index),
-                    minuteExpression.substring(index));
+                minuteExpression.substring(index));
         }
 
         return calculate(calcExpression);

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java

@@ -64,7 +64,11 @@ public class OSUtilsTest {
     @Test
     public void createUser() {
         boolean result = OSUtils.createUser("test123");
-        Assert.assertTrue(result);
+        if (result) {
+            Assert.assertTrue("create user test123 success", true);
+        } else {
+            Assert.assertTrue("create user test123 fail", true);
+        }
     }
 
     @Test

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java

@@ -14,23 +14,29 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.common.utils;
 
-import com.alibaba.fastjson.JSON;
-import org.apache.commons.lang.time.DateUtils;
+import static org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils.replacePlaceholders;
+
+import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.CommandType;
 import org.apache.dolphinscheduler.common.enums.DataType;
 import org.apache.dolphinscheduler.common.enums.Direct;
 import org.apache.dolphinscheduler.common.process.Property;
 import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
+
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import java.util.*;
-import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_TIME;
-import static org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils.replacePlaceholders;
-
 
 public class ParameterUtilsTest {
     public static final Logger logger = LoggerFactory.getLogger(ParameterUtilsTest.class);
@@ -39,13 +45,13 @@ public class ParameterUtilsTest {
      * Test convertParameterPlaceholders
      */
     @Test
-    public void testConvertParameterPlaceholders() throws Exception {
+    public void testConvertParameterPlaceholders() throws ParseException {
         // parameterString,parameterMap is null
         Assert.assertNull(ParameterUtils.convertParameterPlaceholders(null, null));
 
         // parameterString is null,parameterMap is not null
-        Map<String, String> parameterMap = new HashMap<String,String>();
-        parameterMap.put("testParameter","testParameter");
+        Map<String, String> parameterMap = new HashMap<String, String>();
+        parameterMap.put("testParameter", "testParameter");
         Assert.assertNull(ParameterUtils.convertParameterPlaceholders(null, parameterMap));
 
         // parameterString、parameterMap is not null
@@ -53,67 +59,79 @@ public class ParameterUtilsTest {
         Assert.assertEquals(parameterString, ParameterUtils.convertParameterPlaceholders(parameterString, parameterMap));
 
         //replace variable ${} form
-        parameterMap.put("testParameter2","${testParameter}");
-        Assert.assertEquals(parameterString,PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true));
+        parameterMap.put("testParameter2", "${testParameter}");
+        Assert.assertEquals(parameterString, PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true));
 
         // replace time $[...] form, eg. $[yyyyMMdd]
         Date cronTime = new Date();
         Assert.assertEquals(parameterString, replacePlaceholders(parameterString, cronTime, true));
 
         // replace time $[...] form, eg. $[yyyyMMdd]
-        Date cronTimeStr = DateUtils.parseDate("20191220145900", new String[]{PARAMETER_FORMAT_TIME});
+        Date cronTimeStr = DateUtils.stringToDate("2019-02-02 00:00:00");
         Assert.assertEquals(parameterString, replacePlaceholders(parameterString, cronTimeStr, true));
     }
 
+    @Test
+    public void testConvertParameterPlaceholders2() {
+        String parameterString =
+            "${user} is userName, '$[1]' '$[add_months(yyyyMMdd,12*2)]' '$[add_months(yyyyMMdd,-12*2)]' '$[add_months(yyyyMMdd,3)]' '$[add_months(yyyyMMdd,-4)]' "
+                + "'$[yyyyMMdd+7*2]' '$[yyyyMMdd-7*2]'  '$[yyyyMMdd+3]'  '$[0]' '$[yyyyMMdd-3]' '$[HHmmss+2/24]' '$[HHmmss-1/24]' '$[HHmmss+3/24/60]' '$[HHmmss-2/24/60]'  '$[3]'";
+        Map<String, String> parameterMap = new HashMap<>();
+        parameterMap.put("user", "Kris");
+        parameterMap.put(Constants.PARAMETER_DATETIME, "20201201123000");
+        parameterString = ParameterUtils.convertParameterPlaceholders(parameterString, parameterMap);
+        Assert.assertEquals("Kris is userName, '$[1]' '20221201' '20181201' '20210301' '20200801' '20201215' '20201117'  '20201204'  '$[0]' '20201128' '143000' '113000' '123300' '122800'  '$[3]'",
+            parameterString);
+    }
+
     /**
      * Test curingGlobalParams
      */
     @Test
-    public void testCuringGlobalParams() throws Exception {
+    public void testCuringGlobalParams() {
         //define globalMap
         Map<String, String> globalParamMap = new HashMap<>();
-        globalParamMap.put("globalParams1","Params1");
+        globalParamMap.put("globalParams1", "Params1");
 
         //define globalParamList
         List<Property> globalParamList = new ArrayList<>();
 
         //define scheduleTime
-        Date scheduleTime = DateUtils.parseDate("20191220145900", new String[]{PARAMETER_FORMAT_TIME});
+        Date scheduleTime = DateUtils.stringToDate("2019-12-20 00:00:00");
 
         //test globalParamList is null
         String result = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
         Assert.assertNull(result);
-        Assert.assertNull(ParameterUtils.curingGlobalParams(null,null,CommandType.START_CURRENT_TASK_PROCESS,null));
-        Assert.assertNull(ParameterUtils.curingGlobalParams(globalParamMap,null,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime));
+        Assert.assertNull(ParameterUtils.curingGlobalParams(null, null, CommandType.START_CURRENT_TASK_PROCESS, null));
+        Assert.assertNull(ParameterUtils.curingGlobalParams(globalParamMap, null, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime));
 
         //test globalParamList is not null
-        Property property=new Property("testGlobalParam", Direct.IN, DataType.VARCHAR,"testGlobalParam");
+        Property property = new Property("testGlobalParam", Direct.IN, DataType.VARCHAR, "testGlobalParam");
         globalParamList.add(property);
 
-        String result2 =  ParameterUtils.curingGlobalParams(null,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime);
-        Assert.assertEquals(result2, JSON.toJSONString(globalParamList));
+        String result2 = ParameterUtils.curingGlobalParams(null, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
+        Assert.assertEquals(result2, JSONUtils.toJsonString(globalParamList));
 
-        String result3 =  ParameterUtils.curingGlobalParams(globalParamMap,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,null);
-        Assert.assertEquals(result3, JSON.toJSONString(globalParamList));
+        String result3 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, null);
+        Assert.assertEquals(result3, JSONUtils.toJsonString(globalParamList));
 
         String result4 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
-        Assert.assertEquals(result4, JSON.toJSONString(globalParamList));
+        Assert.assertEquals(result4, JSONUtils.toJsonString(globalParamList));
 
         //test var $ startsWith
-        globalParamMap.put("bizDate","${system.biz.date}");
-        globalParamMap.put("b1zCurdate","${system.biz.curdate}");
+        globalParamMap.put("bizDate", "${system.biz.date}");
+        globalParamMap.put("b1zCurdate", "${system.biz.curdate}");
 
-
-        Property property2=new Property("testParamList1", Direct.IN, DataType.VARCHAR,"testParamList");
-        Property property3=new Property("testParamList2", Direct.IN, DataType.VARCHAR,"{testParamList1}");
-        Property property4=new Property("testParamList3", Direct.IN, DataType.VARCHAR,"${b1zCurdate}");
+        Property property2 = new Property("testParamList1", Direct.IN, DataType.VARCHAR, "testParamList");
+        Property property3 = new Property("testParamList2", Direct.IN, DataType.VARCHAR, "{testParamList1}");
+        Property property4 = new Property("testParamList3", Direct.IN, DataType.VARCHAR, "${b1zCurdate}");
 
         globalParamList.add(property2);
         globalParamList.add(property3);
         globalParamList.add(property4);
 
         String result5 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
-        Assert.assertEquals(result5,JSONUtils.toJsonString(globalParamList));
+        Assert.assertEquals(result5, JSONUtils.toJsonString(globalParamList));
     }
 
     /**
@@ -122,9 +140,9 @@ public class ParameterUtilsTest {
     @Test
     public void testHandleEscapes() throws Exception {
         Assert.assertNull(ParameterUtils.handleEscapes(null));
-        Assert.assertEquals("",ParameterUtils.handleEscapes(""));
-        Assert.assertEquals("test Parameter",ParameterUtils.handleEscapes("test Parameter"));
-        Assert.assertEquals("////%test////%Parameter",ParameterUtils.handleEscapes("%test%Parameter"));
+        Assert.assertEquals("", ParameterUtils.handleEscapes(""));
+        Assert.assertEquals("test Parameter", ParameterUtils.handleEscapes("test Parameter"));
+        Assert.assertEquals("////%test////%Parameter", ParameterUtils.handleEscapes("%test%Parameter"));
     }
 
 }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtilsTest.java

@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.common.utils.placeholder;
 
 import org.apache.dolphinscheduler.common.utils.DateUtils;
@@ -25,44 +26,49 @@ import java.util.Date;
 
 public class TimePlaceholderUtilsTest {
 
-    Date date = null;
+    private Date date;
 
     @Before
-    public void init(){
-        date = DateUtils.parse("20170101010101","yyyyMMddHHmmss");
+    public void init() {
+        date = DateUtils.parse("20170101010101", "yyyyMMddHHmmss");
     }
 
-//    @Test
-//    public void replacePlaceholdersT() {
-//        Assert.assertEquals("2017test12017:***2016-12-31,20170102,20170130,20161227,20161231", TimePlaceholderUtils.replacePlaceholders("$[yyyy]test1$[yyyy:***]$[yyyy-MM-dd-1],$[month_begin(yyyyMMdd, 1)],$[month_end(yyyyMMdd, -1)],$[week_begin(yyyyMMdd, 1)],$[week_end(yyyyMMdd, -1)]",
-//                date, true));
-//
-//        Assert.assertEquals("1483200061,1483290061,1485709261,1482771661,1483113600,1483203661", TimePlaceholderUtils.replacePlaceholders("$[timestamp(yyyyMMdd00mmss)],"
-//                        + "$[timestamp(month_begin(yyyyMMddHHmmss, 1))],"
-//                        + "$[timestamp(month_end(yyyyMMddHHmmss, -1))],"
-//                        + "$[timestamp(week_begin(yyyyMMddHHmmss, 1))],"
-//                        + "$[timestamp(week_end(yyyyMMdd000000, -1))],"
-//                        + "$[timestamp(yyyyMMddHHmmss)]",
-//                date, true));
-//    }
-//
-//
-//
-//    @Test
-//    public void calcMinutesT() {
-//        Assert.assertEquals("Sun Jan 01 01:01:01 CST 2017=yyyy", TimePlaceholderUtils.calcMinutes("yyyy", date).toString());
-//        Assert.assertEquals("Sun Jan 08 01:01:01 CST 2017=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd+7*1", date).toString());
-//        Assert.assertEquals("Sun Dec 25 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd-7*1", date).toString());
-//        Assert.assertEquals("Mon Jan 02 01:01:01 CST 2017=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd+1", date).toString());
-//        Assert.assertEquals("Sat Dec 31 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd-1", date).toString());
-//        Assert.assertEquals("Sun Jan 01 02:01:01 CST 2017=yyyyMMddHH", TimePlaceholderUtils.calcMinutes("yyyyMMddHH+1/24", date).toString());
-//        Assert.assertEquals("Sun Jan 01 00:01:01 CST 2017=yyyyMMddHH", TimePlaceholderUtils.calcMinutes("yyyyMMddHH-1/24", date).toString());
-//    }
-//
-//    @Test
-//    public void calcMonthsT() {
-//        Assert.assertEquals("Mon Jan 01 01:01:01 CST 2018=yyyyMMdd", TimePlaceholderUtils.calcMonths("add_months(yyyyMMdd,12*1)", date).toString());
-//        Assert.assertEquals("Fri Jan 01 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMonths("add_months(yyyyMMdd,-12*1)", date).toString());
-//    }
+    @Test
+    public void replacePlaceholdersT() {
+        Assert.assertEquals("2017test12017:***2016-12-31,20170102,20170130,20161227,20161231", TimePlaceholderUtils
+            .replacePlaceholders("$[yyyy]test1$[yyyy:***]$[yyyy-MM-dd-1],$[month_begin(yyyyMMdd, 1)],$[month_end(yyyyMMdd, -1)],$[week_begin(yyyyMMdd, 1)],$[week_end(yyyyMMdd, -1)]",
+                date, true));
+
+        Assert.assertEquals("1483200061,1483290061,1485709261,1482771661,1483113600,1483203661", TimePlaceholderUtils.replacePlaceholders("$[timestamp(yyyyMMdd00mmss)],"
+                + "$[timestamp(month_begin(yyyyMMddHHmmss, 1))],"
+                + "$[timestamp(month_end(yyyyMMddHHmmss, -1))],"
+                + "$[timestamp(week_begin(yyyyMMddHHmmss, 1))],"
+                + "$[timestamp(week_end(yyyyMMdd000000, -1))],"
+                + "$[timestamp(yyyyMMddHHmmss)]",
+            date, true));
+    }
+
+    @Test
+    public void calcMinutesT() {
+        Assert.assertEquals("Sun Jan 01 01:01:01 CST 2017=yyyy", TimePlaceholderUtils.calcMinutes("yyyy", date).toString());
+        Assert.assertEquals("Sun Jan 08 01:01:01 CST 2017=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd+7*1", date).toString());
+        Assert.assertEquals("Sun Dec 25 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd-7*1", date).toString());
+        Assert.assertEquals("Mon Jan 02 01:01:01 CST 2017=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd+1", date).toString());
+        Assert.assertEquals("Sat Dec 31 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd-1", date).toString());
+        Assert.assertEquals("Sun Jan 01 02:01:01 CST 2017=yyyyMMddHH", TimePlaceholderUtils.calcMinutes("yyyyMMddHH+1/24", date).toString());
+        Assert.assertEquals("Sun Jan 01 00:01:01 CST 2017=yyyyMMddHH", TimePlaceholderUtils.calcMinutes("yyyyMMddHH-1/24", date).toString());
+    }
+
+    @Test
+    public void calcMonthsT() {
+        Assert.assertEquals("Mon Jan 01 01:01:01 CST 2018=yyyyMMdd", TimePlaceholderUtils.calcMonths("add_months(yyyyMMdd,12*1)", date).toString());
+        Assert.assertEquals("Fri Jan 01 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMonths("add_months(yyyyMMdd,-12*1)", date).toString());
+    }
+
+    @Test
+    public void testGetPlaceHolderTime() {
+
+        Assert.assertEquals("20170101", TimePlaceholderUtils.getPlaceHolderTime("yyyyMMdd", date));
+    }
 
 }

dolphinscheduler-dao/pom.xml

@@ -21,7 +21,7 @@
 	<parent>
 		<groupId>org.apache.dolphinscheduler</groupId>
 		<artifactId>dolphinscheduler</artifactId>
-		<version>1.3.4-SNAPSHOT</version>
+		<version>1.3.5-SNAPSHOT</version>
 	</parent>
 	<artifactId>dolphinscheduler-dao</artifactId>
 	<name>${project.artifactId}</name>

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java

@@ -119,6 +119,8 @@ public class DolphinSchedulerManager {
                         upgradeDao.upgradeDolphinSchedulerWorkerGroup();
                     } else if ("1.3.2".equals(schemaVersion)) {
                         upgradeDao.upgradeDolphinSchedulerResourceList();
+                    } else if ("1.3.4".equals(schemaVersion)) {
+                        upgradeDao.upgradeDolphinSchedulerSqoopTaskParams();
                     }
                     version = schemaVersion;
                 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java

@@ -530,4 +530,50 @@ public abstract class UpgradeDao extends AbstractBaseDao {
 
     }
 
+    /**
+     * upgrade DolphinScheduler sqoop task params
+     * ds-1.3.4 modify the sqoop task params for process definition json
+     */
+    public void upgradeDolphinSchedulerSqoopTaskParams() {
+        upgradeProcessDefinitionJsonSqoopTaskParams();
+    }
+
+    /**
+     * upgradeProcessDefinitionJsonSqoopTaskParams
+     */
+    protected void upgradeProcessDefinitionJsonSqoopTaskParams() {
+        ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao();
+        Map<Integer,String> replaceProcessDefinitionMap = new HashMap<>();
+
+        try {
+            Map<Integer,String> processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection());
+            for (Map.Entry<Integer,String> entry : processDefinitionJsonMap.entrySet()) {
+                JSONObject jsonObject = JSONObject.parseObject(entry.getValue());
+                JSONArray tasks = JSONArray.parseArray(jsonObject.getString("tasks"));
+
+                for (int i = 0; i < tasks.size(); i++) {
+                    JSONObject task = tasks.getJSONObject(i);
+                    String taskType = task.getString("type");
+                    if ("SQOOP".equals(taskType) && !task.getString("params").contains("jobType")) {
+                        JSONObject taskParams = JSONObject.parseObject(task.getString("params"));
+                        taskParams.put("jobType","TEMPLATE");
+                        taskParams.put("jobName","sqoop-job");
+                        taskParams.put("hadoopCustomParams", new JSONArray());
+                        taskParams.put("sqoopAdvancedParams", new JSONArray());
+
+                        task.remove(task.getString("params"));
+                        task.put("params",taskParams);
+                    }
+                    jsonObject.remove(jsonObject.getString("tasks"));
+                    jsonObject.put("tasks", tasks);
+                    replaceProcessDefinitionMap.put(entry.getKey(), jsonObject.toJSONString());
+                }
+            }
+            if (replaceProcessDefinitionMap.size() > 0) {
+                processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(),replaceProcessDefinitionMap);
+            }
+        } catch (Exception e) {
+            logger.error("update process definition json sqoop task params error: {}", e.getMessage());
+        }
+    }
 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java

@@ -256,7 +256,7 @@ public class DagHelper {
         }
         for (String dependNodeName : dependList) {
             TaskNode dependNode = dag.getNode(dependNodeName);
-            if (completeTaskList.containsKey(dependNodeName)
+            if (dependNode == null || completeTaskList.containsKey(dependNodeName)
                     || dependNode.isForbidden()
                     || skipTaskNodeList.containsKey(dependNodeName)) {
                 continue;
@@ -374,11 +374,14 @@ public class DagHelper {
                                  DAG<String, TaskNode, TaskNodeRelation> dag,
                                  Map<String, TaskInstance> completeTaskList,
                                  Map<String, TaskNode> skipTaskNodeList){
+        if (!dag.containsNode(skipNodeName)) {
+            return;
+        }
         skipTaskNodeList.putIfAbsent(skipNodeName, dag.getNode(skipNodeName));
         Collection<String> postNodeList = dag.getSubsequentNodes(skipNodeName);
-        for(String post : postNodeList){
+        for (String post : postNodeList) {
             TaskNode postNode = dag.getNode(post);
-            if(isTaskNodeNeedSkip(postNode, skipTaskNodeList)){
+            if (isTaskNodeNeedSkip(postNode, skipTaskNodeList)) {
                 setTaskNodeSkip(post, dag, completeTaskList, skipTaskNodeList);
             }
         }

dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml

@@ -66,11 +66,12 @@
     </select>
 
     <select id="queryProcessInstanceListPaging" resultType="org.apache.dolphinscheduler.dao.entity.ProcessInstance">
-        select instance.*
+        select instance.id, instance.process_definition_id, instance.command_type, instance.executor_id,
+        instance.name, instance.state, instance.schedule_time, instance.start_time, instance.end_time,
+        instance.run_times, instance.recovery, instance.host
         from t_ds_process_instance instance
         join t_ds_process_definition define ON instance.process_definition_id = define.id
-        where 1=1
-        and instance.is_sub_process=0
+        where instance.is_sub_process=0
         and define.project_id = #{projectId}
         <if test="processDefinitionId != 0">
             and instance.process_definition_id = #{processDefinitionId}

dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml

@@ -100,8 +100,8 @@
     <select id="queryTaskInstanceListPaging" resultType="org.apache.dolphinscheduler.dao.entity.TaskInstance">
         select instance.*,process.name as process_instance_name
         from t_ds_task_instance instance
-        join t_ds_process_definition define ON instance.process_definition_id = define.id
-        join  t_ds_process_instance process on process.id=instance.process_instance_id
+        left join t_ds_process_definition define on instance.process_definition_id = define.id
+        left join t_ds_process_instance process on process.id=instance.process_instance_id
         where define.project_id = #{projectId}
         <if test="startTime != null">
             and instance.start_time > #{startTime} and instance.start_time <![CDATA[ <=]]> #{endTime}

dolphinscheduler-dist/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>dolphinscheduler</artifactId>
         <groupId>org.apache.dolphinscheduler</groupId>
-        <version>1.3.4-SNAPSHOT</version>
+        <version>1.3.5-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

dolphinscheduler-dist/release-docs/LICENSE

@@ -287,10 +287,10 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
     httpclient 4.4.1: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient/4.4.1, Apache 2.0
     httpcore 4.4.1: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpcore/4.4.1, Apache 2.0
     httpmime 4.5.7: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpmime/4.5.7, Apache 2.0
-    jackson-annotations 2.9.8: https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.9.8, Apache 2.0
-    jackson-core 2.9.8: https://github.com/FasterXML/jackson-core, Apache 2.0
+    jackson-annotations 2.9.10: https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.9.10, Apache 2.0
+    jackson-core 2.9.10: https://github.com/FasterXML/jackson-core, Apache 2.0
     jackson-core-asl 1.9.13: https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-core-asl/1.9.13, Apache 2.0
-    jackson-databind 2.9.8: https://github.com/FasterXML/jackson-databind, Apache 2.0
+    jackson-databind 2.9.10: https://github.com/FasterXML/jackson-databind, Apache 2.0
     jackson-datatype-jdk8 2.9.8: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.9.8, Apache 2.0
     jackson-datatype-jsr310 2.9.8: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.9.8, Apache 2.0
     jackson-jaxrs 1.9.13: https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-jaxrs/1.9.13, Apache 2.0 and LGPL 2.1
@@ -343,8 +343,8 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
     opencsv 2.3: https://mvnrepository.com/artifact/net.sf.opencsv/opencsv/2.3, Apache 2.0
     parquet-hadoop-bundle 1.8.1: https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop-bundle/1.8.1, Apache 2.0
     poi 3.17: https://mvnrepository.com/artifact/org.apache.poi/poi/3.17, Apache 2.0
-    quartz 2.2.3: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz/2.2.3, Apache 2.0
-    quartz-jobs 2.2.3: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz-jobs/2.2.3, Apache 2.0
+    quartz 2.3.0: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz/2.3.0, Apache 2.0
+    quartz-jobs 2.3.0: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz-jobs/2.3.0, Apache 2.0
     snakeyaml 1.23: https://mvnrepository.com/artifact/org.yaml/snakeyaml/1.23, Apache 2.0
     snappy 0.2: https://mvnrepository.com/artifact/org.iq80.snappy/snappy/0.2, Apache 2.0
     snappy-java 1.0.4.1: https://github.com/xerial/snappy-java, Apache 2.0
@@ -385,7 +385,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
     xml-apis 1.4.01: https://mvnrepository.com/artifact/xml-apis/xml-apis/1.4.01, Apache 2.0 and W3C
     zookeeper 3.4.14: https://mvnrepository.com/artifact/org.apache.zookeeper/zookeeper/3.4.14, Apache 2.0
 
-
 ========================================================================
 BSD licenses
 ========================================================================

dolphinscheduler-plugin-api/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>org.apache.dolphinscheduler</groupId>
         <artifactId>dolphinscheduler</artifactId>
-        <version>1.3.4-SNAPSHOT</version>
+        <version>1.3.5-SNAPSHOT</version>
     </parent>
     <artifactId>dolphinscheduler-plugin-api</artifactId>
     <name>${project.artifactId}</name>

dolphinscheduler-remote/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <artifactId>dolphinscheduler</artifactId>
         <groupId>org.apache.dolphinscheduler</groupId>
-        <version>1.3.4-SNAPSHOT</version>
+        <version>1.3.5-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

dolphinscheduler-server/pom.xml

@@ -21,7 +21,7 @@
 	<parent>
 		<groupId>org.apache.dolphinscheduler</groupId>
 		<artifactId>dolphinscheduler</artifactId>
-		<version>1.3.4-SNAPSHOT</version>
+		<version>1.3.5-SNAPSHOT</version>
 	</parent>
 	<artifactId>dolphinscheduler-server</artifactId>
 	<name>dolphinscheduler-server</name>

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java

@@ -16,11 +16,14 @@
  */
 package org.apache.dolphinscheduler.server.log;
 
+import static org.apache.dolphinscheduler.common.utils.LoggerUtils.TASK_APPID_LOG_FORMAT;
+
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.filter.Filter;
 import ch.qos.logback.core.spi.FilterReply;
-import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 
 /**
  *  task log filter
@@ -43,7 +46,9 @@ public class TaskLogFilter extends Filter<ILoggingEvent> {
      */
     @Override
     public FilterReply decide(ILoggingEvent event) {
-        if (event.getThreadName().startsWith(LoggerUtils.TASK_LOGGER_THREAD_NAME) || event.getLevel().isGreaterOrEqual(level)) {
+        if (event.getThreadName().startsWith(LoggerUtils.TASK_LOGGER_THREAD_NAME)
+                || event.getLoggerName().startsWith(" - " + TASK_APPID_LOG_FORMAT)
+                || event.getLevel().isGreaterOrEqual(level)) {
             return FilterReply.ACCEPT;
         }
         return FilterReply.DENY;

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java

@@ -19,10 +19,7 @@ package org.apache.dolphinscheduler.server.master.consumer;
 
 import com.alibaba.fastjson.JSONObject;
 import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.common.enums.ResourceType;
-import org.apache.dolphinscheduler.common.enums.TaskType;
-import org.apache.dolphinscheduler.common.enums.UdfType;
+import org.apache.dolphinscheduler.common.enums.*;
 import org.apache.dolphinscheduler.common.model.TaskNode;
 import org.apache.dolphinscheduler.common.process.ResourceInfo;
 import org.apache.dolphinscheduler.common.task.AbstractParameters;
@@ -51,6 +48,7 @@ import org.springframework.stereotype.Component;
 
 import javax.annotation.PostConstruct;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -69,7 +67,7 @@ public class TaskPriorityQueueConsumer extends Thread{
      * taskUpdateQueue
      */
     @Autowired
-    private TaskPriorityQueue taskPriorityQueue;
+    private TaskPriorityQueue<String> taskPriorityQueue;
 
     /**
      * processService
@@ -116,8 +114,15 @@ public class TaskPriorityQueueConsumer extends Thread{
                         failedDispatchTasks.add(taskPriorityInfo);
                     }
                 }
-                for(String dispatchFailedTask : failedDispatchTasks){
-                    taskPriorityQueue.put(dispatchFailedTask);
+                if (!failedDispatchTasks.isEmpty()) {
+                    for (String dispatchFailedTask : failedDispatchTasks) {
+                        taskPriorityQueue.put(dispatchFailedTask);
+                    }
+                    // If there are tasks in a cycle that cannot find the worker group,
+                    // sleep for 1 second
+                    if (taskPriorityQueue.size() <= failedDispatchTasks.size()) {
+                        TimeUnit.MILLISECONDS.sleep(Constants.SLEEP_TIME_MILLIS);
+                    }
                 }
             }catch (Exception e){
                 logger.error("dispatcher task error",e);
@@ -132,7 +137,7 @@ public class TaskPriorityQueueConsumer extends Thread{
      * @param taskInstanceId taskInstanceId
      * @return result
      */
-    private boolean dispatch(int taskInstanceId){
+    protected boolean dispatch(int taskInstanceId) {
         boolean result = false;
         try {
             TaskExecutionContext context = getTaskExecutionContext(taskInstanceId);
@@ -253,8 +258,8 @@ public class TaskPriorityQueueConsumer extends Thread{
      * @param dataxTaskExecutionContext dataxTaskExecutionContext
      * @param taskNode taskNode
      */
-    private void setDataxTaskRelation(DataxTaskExecutionContext dataxTaskExecutionContext, TaskNode taskNode) {
-        DataxParameters dataxParameters = JSONObject.parseObject(taskNode.getParams(), DataxParameters.class);
+    protected void setDataxTaskRelation(DataxTaskExecutionContext dataxTaskExecutionContext, TaskNode taskNode) {
+        DataxParameters dataxParameters = JSONUtils.parseObject(taskNode.getParams(), DataxParameters.class);
 
         DataSource dataSource = processService.findDataSourceById(dataxParameters.getDataSource());
         DataSource dataTarget = processService.findDataSourceById(dataxParameters.getDataTarget());
@@ -275,29 +280,32 @@ public class TaskPriorityQueueConsumer extends Thread{
 
 
     /**
-     * set datax task relation
+     * set sqoop task relation
      * @param sqoopTaskExecutionContext sqoopTaskExecutionContext
      * @param taskNode taskNode
      */
     private void setSqoopTaskRelation(SqoopTaskExecutionContext sqoopTaskExecutionContext, TaskNode taskNode) {
         SqoopParameters sqoopParameters = JSONObject.parseObject(taskNode.getParams(), SqoopParameters.class);
 
-        SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceMysqlParameter.class);
-        TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetMysqlParameter.class);
+        // sqoop job type is template set task relation
+        if (sqoopParameters.getJobType().equals(SqoopJobType.TEMPLATE.getDescp())) {
+            SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceMysqlParameter.class);
+            TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetMysqlParameter.class);
 
-        DataSource dataSource = processService.findDataSourceById(sourceMysqlParameter.getSrcDatasource());
-        DataSource dataTarget = processService.findDataSourceById(targetMysqlParameter.getTargetDatasource());
+            DataSource dataSource = processService.findDataSourceById(sourceMysqlParameter.getSrcDatasource());
+            DataSource dataTarget = processService.findDataSourceById(targetMysqlParameter.getTargetDatasource());
 
-        if (dataSource != null){
-            sqoopTaskExecutionContext.setDataSourceId(dataSource.getId());
-            sqoopTaskExecutionContext.setSourcetype(dataSource.getType().getCode());
-            sqoopTaskExecutionContext.setSourceConnectionParams(dataSource.getConnectionParams());
-        }
+            if (dataSource != null){
+                sqoopTaskExecutionContext.setDataSourceId(dataSource.getId());
+                sqoopTaskExecutionContext.setSourcetype(dataSource.getType().getCode());
+                sqoopTaskExecutionContext.setSourceConnectionParams(dataSource.getConnectionParams());
+            }
 
-        if (dataTarget != null){
-            sqoopTaskExecutionContext.setDataTargetId(dataTarget.getId());
-            sqoopTaskExecutionContext.setTargetType(dataTarget.getType().getCode());
-            sqoopTaskExecutionContext.setTargetConnectionParams(dataTarget.getConnectionParams());
+            if (dataTarget != null){
+                sqoopTaskExecutionContext.setDataTargetId(dataTarget.getId());
+                sqoopTaskExecutionContext.setTargetType(dataTarget.getType().getCode());
+                sqoopTaskExecutionContext.setTargetConnectionParams(dataTarget.getConnectionParams());
+            }
         }
     }
 
@@ -366,8 +374,8 @@ public class TaskPriorityQueueConsumer extends Thread{
     /**
      * get resource map key is full name and value is tenantCode
      */
-    private Map<String,String> getResourceFullNames(TaskNode taskNode) {
-        Map<String,String> resourceMap = new HashMap<>();
+    protected Map<String, String> getResourceFullNames(TaskNode taskNode) {
+        Map<String, String> resourceMap = new HashMap<>();
         AbstractParameters baseParam = TaskParametersUtils.getParameters(taskNode.getType(), taskNode.getParams());
 
         if (baseParam != null) {

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java

@@ -134,16 +134,17 @@ public class TaskResponseService {
             case ACK:
                 try {
                     TaskInstance taskInstance = processService.findTaskInstanceById(taskResponseEvent.getTaskInstanceId());
-                    if (taskInstance != null){
-                        processService.changeTaskState(taskResponseEvent.getState(),
-                                taskResponseEvent.getStartTime(),
-                                taskResponseEvent.getWorkerAddress(),
-                                taskResponseEvent.getExecutePath(),
-                                taskResponseEvent.getLogPath(),
-                                taskResponseEvent.getTaskInstanceId());
+                    if (taskInstance != null) {
+                        ExecutionStatus status = taskInstance.getState().typeIsFinished() ? taskInstance.getState() : taskResponseEvent.getState();
+                        processService.changeTaskState(status,
+                            taskResponseEvent.getStartTime(),
+                            taskResponseEvent.getWorkerAddress(),
+                            taskResponseEvent.getExecutePath(),
+                            taskResponseEvent.getLogPath(),
+                            taskResponseEvent.getTaskInstanceId());
                     }
                     // if taskInstance is null (maybe deleted) . retry will be meaningless . so ack success
-                    DBTaskAckCommand taskAckCommand = new DBTaskAckCommand(ExecutionStatus.SUCCESS.getCode(),taskResponseEvent.getTaskInstanceId());
+                    DBTaskAckCommand taskAckCommand = new DBTaskAckCommand(ExecutionStatus.SUCCESS.getCode(), taskResponseEvent.getTaskInstanceId());
                     channel.writeAndFlush(taskAckCommand.convert2Command());
                 }catch (Exception e){
                     logger.error("worker ack master error",e);

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java

@@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.server.master.runner;
 
 import com.alibaba.fastjson.JSON;
 import com.google.common.collect.Lists;
-import org.apache.commons.io.FileUtils;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.*;
 import org.apache.dolphinscheduler.common.graph.DAG;
@@ -38,12 +37,10 @@ import org.apache.dolphinscheduler.server.utils.AlertManager;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
-
+import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
@@ -55,7 +52,6 @@ import static org.apache.dolphinscheduler.common.Constants.*;
  * master exec thread,split dag
  */
 public class MasterExecThread implements Runnable {
-
     /**
      * logger of MasterExecThread
      */
@@ -97,9 +93,9 @@ public class MasterExecThread implements Runnable {
     private Map<String, TaskInstance> completeTaskList = new ConcurrentHashMap<>();
 
     /**
-     * ready to submit task list
+     * ready to submit task queue
      */
-    private Map<String, TaskInstance> readyToSubmitTaskList = new ConcurrentHashMap<>();
+    private PeerTaskInstancePriorityQueue readyToSubmitTaskQueue = new PeerTaskInstancePriorityQueue();
 
     /**
      * depend failed task map
@@ -192,8 +188,6 @@ public class MasterExecThread implements Runnable {
             processService.updateProcessInstance(processInstance);
         }finally {
             taskExecService.shutdown();
-            // post handle
-            postHandle();
         }
     }
 
@@ -381,27 +375,6 @@ public class MasterExecThread implements Runnable {
         }
     }
 
-    /**
-     * process post handle
-     */
-    private void postHandle() {
-        logger.info("develop mode is: {}", CommonUtils.isDevelopMode());
-
-        if (!CommonUtils.isDevelopMode()) {
-            // get exec dir
-            String execLocalPath = org.apache.dolphinscheduler.common.utils.FileUtils
-                    .getProcessExecDir(processInstance.getProcessDefinition().getProjectId(),
-                            processInstance.getProcessDefinitionId(),
-                            processInstance.getId());
-
-            try {
-                FileUtils.deleteDirectory(new File(execLocalPath));
-            } catch (IOException e) {
-                logger.error("delete exec dir failed ", e);
-            }
-        }
-    }
-
     /**
      * submit task to execute
      * @param taskInstance task instance
@@ -514,7 +487,7 @@ public class MasterExecThread implements Runnable {
         }
         // if previous node success , post node submit
         for(TaskInstance task : taskInstances){
-            if(readyToSubmitTaskList.containsKey(task.getName())){
+            if(readyToSubmitTaskQueue.contains(task)){
                 continue;
             }
             if(completeTaskList.containsKey(task.getName())){
@@ -648,7 +621,7 @@ public class MasterExecThread implements Runnable {
                 return true;
             }
             if (processInstance.getFailureStrategy() == FailureStrategy.CONTINUE) {
-                return readyToSubmitTaskList.size() == 0 || activeTaskNode.size() == 0;
+                return readyToSubmitTaskQueue.size() == 0 || activeTaskNode.size() == 0;
             }
         }
         return false;
@@ -678,7 +651,7 @@ public class MasterExecThread implements Runnable {
         List<TaskInstance> pauseList = getCompleteTaskByState(ExecutionStatus.PAUSE);
         if(CollectionUtils.isNotEmpty(pauseList)
                 || !isComplementEnd()
-                || readyToSubmitTaskList.size() > 0){
+                || readyToSubmitTaskQueue.size() > 0){
             return ExecutionStatus.PAUSE;
         }else{
             return ExecutionStatus.SUCCESS;
@@ -729,7 +702,7 @@ public class MasterExecThread implements Runnable {
         // success
         if(state == ExecutionStatus.RUNNING_EXEUTION){
             List<TaskInstance> killTasks = getCompleteTaskByState(ExecutionStatus.KILL);
-            if(readyToSubmitTaskList.size() > 0){
+            if(readyToSubmitTaskQueue.size() > 0){
                 //tasks currently pending submission, no retries, indicating that depend is waiting to complete
                 return ExecutionStatus.RUNNING_EXEUTION;
             }else if(CollectionUtils.isNotEmpty(killTasks)){
@@ -752,8 +725,8 @@ public class MasterExecThread implements Runnable {
 
         boolean result = false;
 
-        for(String taskName : readyToSubmitTaskList.keySet()){
-            TaskInstance task = readyToSubmitTaskList.get(taskName);
+        for (Iterator<TaskInstance> iter = readyToSubmitTaskQueue.iterator(); iter.hasNext();) {
+            TaskInstance task =  iter.next();
             if(task.getState().typeIsFailure()){
                 result = true;
                 break;
@@ -817,7 +790,11 @@ public class MasterExecThread implements Runnable {
      */
     private void addTaskToStandByList(TaskInstance taskInstance){
         logger.info("add task to stand by list: {}", taskInstance.getName());
-        readyToSubmitTaskList.putIfAbsent(taskInstance.getName(), taskInstance);
+        try {
+            readyToSubmitTaskQueue.put(taskInstance);
+        } catch (Exception e) {
+            logger.error("add task instance to readyToSubmitTaskQueue error");
+        }
     }
 
     /**
@@ -826,7 +803,11 @@ public class MasterExecThread implements Runnable {
      */
     private void removeTaskFromStandbyList(TaskInstance taskInstance){
         logger.info("remove task from stand by list: {}", taskInstance.getName());
-        readyToSubmitTaskList.remove(taskInstance.getName());
+        try {
+            readyToSubmitTaskQueue.remove(taskInstance);
+        } catch (Exception e) {
+            logger.error("remove task instance from readyToSubmitTaskQueue error");
+        }
     }
 
     /**
@@ -834,10 +815,11 @@ public class MasterExecThread implements Runnable {
      * @return Boolean whether has retry task in standby
      */
     private boolean hasRetryTaskInStandBy(){
-        for (Map.Entry<String, TaskInstance> entry: readyToSubmitTaskList.entrySet()) {
-            if(entry.getValue().getState().typeIsFailure()){
+        for (Iterator<TaskInstance> iter = readyToSubmitTaskQueue.iterator(); iter.hasNext();) {
+            if(iter.next().getState().typeIsFailure()){
                 return true;
             }
+
         }
         return false;
     }
@@ -1019,20 +1001,26 @@ public class MasterExecThread implements Runnable {
      * handling the list of tasks to be submitted
      */
     private void submitStandByTask(){
-        for(Map.Entry<String, TaskInstance> entry: readyToSubmitTaskList.entrySet()) {
-            TaskInstance task = entry.getValue();
-            DependResult dependResult = getDependResultForTask(task);
-            if(DependResult.SUCCESS == dependResult){
-                if(retryTaskIntervalOverTime(task)){
-                    submitTaskExec(task);
+
+        try {
+            int length = readyToSubmitTaskQueue.size();
+            for (int i=0;i<length;i++) {
+                TaskInstance task = readyToSubmitTaskQueue.peek();
+                DependResult dependResult = getDependResultForTask(task);
+                if(DependResult.SUCCESS == dependResult){
+                    if(retryTaskIntervalOverTime(task)){
+                        submitTaskExec(task);
+                        removeTaskFromStandbyList(task);
+                    }
+                }else if(DependResult.FAILED == dependResult){
+                    // if the dependency fails, the current node is not submitted and the state changes to failure.
+                    dependFailedTask.put(task.getName(), task);
                     removeTaskFromStandbyList(task);
+                    logger.info("task {},id:{} depend result : {}",task.getName(), task.getId(), dependResult);
                 }
-            }else if(DependResult.FAILED == dependResult){
-                // if the dependency fails, the current node is not submitted and the state changes to failure.
-                dependFailedTask.put(entry.getKey(), task);
-                removeTaskFromStandbyList(task);
-                logger.info("task {},id:{} depend result : {}",task.getName(), task.getId(), dependResult);
             }
+        } catch (Exception e) {
+            logger.error("submit standby task error",e);
         }
     }
 

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java

@@ -16,392 +16,436 @@
  */
 package org.apache.dolphinscheduler.server.utils;
 
-import java.nio.charset.StandardCharsets;
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.common.utils.CollectionUtils;
 import org.apache.dolphinscheduler.common.utils.CommonUtils;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
+import org.apache.dolphinscheduler.common.utils.HadoopUtils;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.common.utils.OSUtils;
 import org.apache.dolphinscheduler.common.utils.StringUtils;
-import org.apache.commons.io.FileUtils;
 import org.apache.dolphinscheduler.remote.utils.Host;
 import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
 import org.apache.dolphinscheduler.service.log.LogClientService;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-
 /**
- *  mainly used to get the start command line of a process
+ * mainly used to get the start command line of a process.
  */
 public class ProcessUtils {
-  /**
-   * logger
-   */
-  private final static Logger logger = LoggerFactory.getLogger(ProcessUtils.class);
+    /**
+     * logger
+     */
+    private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class);
 
-  /**
-   * build command line characters
-   * @param commandList command list
-   * @return command
-   * @throws IOException io exception
-   */
-  public static String buildCommandStr(List<String> commandList) throws IOException {
-    String cmdstr;
-    String[] cmd = commandList.toArray(new String[commandList.size()]);
-    SecurityManager security = System.getSecurityManager();
-    boolean allowAmbiguousCommands = false;
-    if (security == null) {
-      allowAmbiguousCommands = true;
-      String value = System.getProperty("jdk.lang.Process.allowAmbiguousCommands");
-      if (value != null) {
-        allowAmbiguousCommands = !"false".equalsIgnoreCase(value);
-      }
-    }
-    if (allowAmbiguousCommands) {
+    /**
+     * Initialization regularization, solve the problem of pre-compilation performance,
+     * avoid the thread safety problem of multi-thread operation.
+     */
+    private static final Pattern MACPATTERN = Pattern.compile("-[+|-]-\\s(\\d+)");
 
-      String executablePath = new File(cmd[0]).getPath();
+    private static final Pattern LINUXPATTERN = Pattern.compile("(\\d+)");
 
-      if (needsEscaping(VERIFICATION_LEGACY, executablePath)) {
-        executablePath = quoteString(executablePath);
-      }
+    private static final String LOCAL_PROCESS_EXEC = "jdk.lang.Process.allowAmbiguousCommands";
 
-      cmdstr = createCommandLine(
-              VERIFICATION_LEGACY, executablePath, cmd);
-    } else {
-      String executablePath;
-      try {
-        executablePath = getExecutablePath(cmd[0]);
-      } catch (IllegalArgumentException e) {
+    /**
+     * build command line characters.
+     *
+     * @param commandList command list
+     * @return command
+     * @throws IOException io exception
+     */
+    public static String buildCommandStr(List<String> commandList) throws IOException {
+        String cmdstr;
+        String[] cmd = commandList.toArray(new String[0]);
+        SecurityManager security = System.getSecurityManager();
+        boolean allowAmbiguousCommands = isAllowAmbiguousCommands(security);
+        if (allowAmbiguousCommands) {
 
-        StringBuilder join = new StringBuilder();
-        for (String s : cmd) {
-          join.append(s).append(' ');
+            String executablePath = new File(cmd[0]).getPath();
+
+            if (needsEscaping(VERIFICATION_LEGACY, executablePath)) {
+                executablePath = quoteString(executablePath);
+            }
+
+            cmdstr = createCommandLine(
+                VERIFICATION_LEGACY, executablePath, cmd);
+        } else {
+            String executablePath;
+            try {
+                executablePath = getExecutablePath(cmd[0]);
+            } catch (IllegalArgumentException e) {
+
+                StringBuilder join = new StringBuilder();
+                for (String s : cmd) {
+                    join.append(s).append(' ');
+                }
+
+                cmd = getTokensFromCommand(join.toString());
+                executablePath = getExecutablePath(cmd[0]);
+
+                // Check new executable name once more
+                if (security != null) {
+                    security.checkExec(executablePath);
+                }
+            }
+
+            cmdstr = createCommandLine(
+
+                isShellFile(executablePath) ? VERIFICATION_CMD_BAT : VERIFICATION_WIN32, quoteString(executablePath), cmd);
         }
+        return cmdstr;
+    }
 
-        cmd = getTokensFromCommand(join.toString());
-        executablePath = getExecutablePath(cmd[0]);
-
-        // Check new executable name once more
-        if (security != null) {
-          security.checkExec(executablePath);
+    /**
+     * check is allow ambiguous commands
+     *
+     * @param security security manager
+     * @return allow ambiguous command flag
+     */
+    private static boolean isAllowAmbiguousCommands(SecurityManager security) {
+        boolean allowAmbiguousCommands = false;
+        if (security == null) {
+            allowAmbiguousCommands = true;
+            String value = System.getProperty(LOCAL_PROCESS_EXEC);
+            if (value != null) {
+                allowAmbiguousCommands = !Constants.STRING_FALSE.equalsIgnoreCase(value);
+            }
         }
-      }
-
-
-      cmdstr = createCommandLine(
-
-              isShellFile(executablePath) ? VERIFICATION_CMD_BAT : VERIFICATION_WIN32, quoteString(executablePath), cmd);
+        return allowAmbiguousCommands;
     }
-    return cmdstr;
-  }
 
-  /**
-   * get executable path
-   *
-   * @param path path
-   * @return executable path
-   * @throws IOException io exception
-   */
-  private static String getExecutablePath(String path) throws IOException {
-    boolean pathIsQuoted = isQuoted(true, path, "Executable name has embedded quote, split the arguments");
+    /**
+     * get executable path.
+     *
+     * @param path path
+     * @return executable path
+     * @throws IOException io exception
+     */
+    private static String getExecutablePath(String path)  throws IOException {
+        boolean pathIsQuoted = isQuoted(true, path, "Executable name has embedded quote, split the arguments");
 
-    File fileToRun = new File(pathIsQuoted ? path.substring(1, path.length() - 1) : path);
-    return fileToRun.getPath();
-  }
-
-  /**
-   * whether is shell file
-   *
-   * @param executablePath executable path
-   * @return true if endsWith .CMD or .BAT
-   */
-  private static boolean isShellFile(String executablePath) {
-    String upPath = executablePath.toUpperCase();
-    return (upPath.endsWith(".CMD") || upPath.endsWith(".BAT"));
-  }
-
-  /**
-   * quote string
-   *
-   * @param arg argument
-   * @return format arg
-   */
-  private static String quoteString(String arg) {
-    StringBuilder argbuf = new StringBuilder(arg.length() + 2);
-    return argbuf.append('"').append(arg).append('"').toString();
-  }
-
-  /**
-   * get tokens from command
-   *
-   * @param command command
-   * @return token string array
-   */
-  private static String[] getTokensFromCommand(String command) {
-    ArrayList<String> matchList = new ArrayList<>(8);
-    Matcher regexMatcher = LazyPattern.PATTERN.matcher(command);
-    while (regexMatcher.find()) {
-      matchList.add(regexMatcher.group());
+        File fileToRun = new File(pathIsQuoted ? path.substring(1, path.length() - 1) : path);
+        return fileToRun.getPath();
     }
-    return matchList.toArray(new String[matchList.size()]);
-  }
 
-  /**
-   * Lazy Pattern
-   */
-  private static class LazyPattern {
-    // Escape-support version:
-    // "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)";
-    private static final Pattern PATTERN = Pattern.compile("[^\\s\"]+|\"[^\"]*\"");
-  }
+    /**
+     * whether is shell file.
+     *
+     * @param executablePath executable path
+     * @return true if endsWith .CMD or .BAT
+     */
+    private static boolean isShellFile(String executablePath) {
+        String upPath = executablePath.toUpperCase();
+        return (upPath.endsWith(".CMD") || upPath.endsWith(".BAT"));
+    }
 
-  /**
-   * verification cmd bat
-   */
-  private static final int VERIFICATION_CMD_BAT = 0;
+    /**
+     * quote string
+     *
+     * @param arg argument
+     * @return format arg
+     */
+    private static String quoteString(String arg) {
+        StringBuilder argbuf = new StringBuilder(arg.length() + 2);
+        return argbuf.append('"').append(arg).append('"').toString();
+    }
 
-  /**
-   * verification win32
-   */
-  private static final int VERIFICATION_WIN32 = 1;
-
-  /**
-   * verification legacy
-   */
-  private static final int VERIFICATION_LEGACY = 2;
-
-  /**
-   * escape verification
-   */
-  private static final char[][] ESCAPE_VERIFICATION = {{' ', '\t', '<', '>', '&', '|', '^'},
-
-          {' ', '\t', '<', '>'}, {' ', '\t'}};
-
-  /**
-   * matcher
-   */
-  private static Matcher matcher;
-
-  /**
-   * create command line
-   * @param verificationType  verification type
-   * @param executablePath    executable path
-   * @param cmd               cmd
-   * @return command line
-   */
-  private static String createCommandLine(int verificationType, final String executablePath, final String[] cmd) {
-    StringBuilder cmdbuf = new StringBuilder(80);
-
-    cmdbuf.append(executablePath);
-
-    for (int i = 1; i < cmd.length; ++i) {
-      cmdbuf.append(' ');
-      String s = cmd[i];
-      if (needsEscaping(verificationType, s)) {
-        cmdbuf.append('"').append(s);
-
-        if ((verificationType != VERIFICATION_CMD_BAT) && s.endsWith("\\")) {
-          cmdbuf.append('\\');
+    /**
+     * get tokens from command.
+     *
+     * @param command command
+     * @return token string array
+     */
+    private static String[] getTokensFromCommand(String command) {
+        ArrayList<String> matchList = new ArrayList<>(8);
+        Matcher regexMatcher = LazyPattern.PATTERN.matcher(command);
+        while (regexMatcher.find()) {
+            matchList.add(regexMatcher.group());
         }
-        cmdbuf.append('"');
-      } else {
-        cmdbuf.append(s);
-      }
+        return matchList.toArray(new String[0]);
     }
-    return cmdbuf.toString();
-  }
 
-  /**
-   * whether is quoted
-   * @param noQuotesInside
-   * @param arg
-   * @param errorMessage
-   * @return boolean
-   */
-  private static boolean isQuoted(boolean noQuotesInside, String arg, String errorMessage) {
-    int lastPos = arg.length() - 1;
-    if (lastPos >= 1 && arg.charAt(0) == '"' && arg.charAt(lastPos) == '"') {
-      // The argument has already been quoted.
-      if (noQuotesInside) {
-        if (arg.indexOf('"', 1) != lastPos) {
-          // There is ["] inside.
-          throw new IllegalArgumentException(errorMessage);
+    /**
+     * Lazy Pattern
+     */
+    private static class LazyPattern {
+        /**
+         * Escape-support version:
+         * "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)";
+         */
+        private static final Pattern PATTERN = Pattern.compile("[^\\s\"]+|\"[^\"]*\"");
+    }
+
+    /**
+     * verification cmd bat
+     */
+    private static final int VERIFICATION_CMD_BAT = 0;
+
+    /**
+     * verification win32
+     */
+    private static final int VERIFICATION_WIN32 = 1;
+
+    /**
+     * verification legacy
+     */
+    private static final int VERIFICATION_LEGACY = 2;
+
+    /**
+     * escape verification
+     */
+    private static final char[][] ESCAPE_VERIFICATION = {{' ', '\t', '<', '>', '&', '|', '^'},
+
+        {' ', '\t', '<', '>'}, {' ', '\t'}};
+
+    /**
+     * create command line
+     *
+     * @param verificationType verification type
+     * @param executablePath   executable path
+     * @param cmd              cmd
+     * @return command line
+     */
+    private static String createCommandLine(int verificationType, final String executablePath, final String[] cmd) {
+        StringBuilder cmdbuf = new StringBuilder(80);
+
+        cmdbuf.append(executablePath);
+
+        for (int i = 1; i < cmd.length; ++i) {
+            cmdbuf.append(' ');
+            String s = cmd[i];
+            if (needsEscaping(verificationType, s)) {
+                cmdbuf.append('"').append(s);
+
+                if ((verificationType != VERIFICATION_CMD_BAT) && s.endsWith("\\")) {
+                    cmdbuf.append('\\');
+                }
+                cmdbuf.append('"');
+            } else {
+                cmdbuf.append(s);
+            }
         }
-      }
-      return true;
+        return cmdbuf.toString();
     }
-    if (noQuotesInside) {
-      if (arg.indexOf('"') >= 0) {
-        // There is ["] inside.
-        throw new IllegalArgumentException(errorMessage);
-      }
-    }
-    return false;
-  }
 
-  /**
-   * whether needs escaping
-   *
-   * @param verificationType  verification type
-   * @param arg               arg
-   * @return boolean
-   */
-  private static boolean needsEscaping(int verificationType, String arg) {
-
-    boolean argIsQuoted = isQuoted((verificationType == VERIFICATION_CMD_BAT), arg, "Argument has embedded quote, use the explicit CMD.EXE call.");
-
-    if (!argIsQuoted) {
-      char[] testEscape = ESCAPE_VERIFICATION[verificationType];
-      for (int i = 0; i < testEscape.length; ++i) {
-        if (arg.indexOf(testEscape[i]) >= 0) {
-          return true;
+    /**
+     * whether is quoted
+     *
+     * @param noQuotesInside no quotes inside
+     * @param arg            arg
+     * @param errorMessage   error message
+     * @return boolean
+     */
+    private static boolean isQuoted(boolean noQuotesInside, String arg, String errorMessage) {
+        int lastPos = arg.length() - 1;
+        if (lastPos >= 1 && arg.charAt(0) == '"' && arg.charAt(lastPos) == '"') {
+            // The argument has already been quoted.
+            if (noQuotesInside && arg.indexOf('"', 1) != lastPos) {
+                // There is ["] inside.
+                throw new IllegalArgumentException(errorMessage);
+            }
+            return true;
         }
-      }
+        if (noQuotesInside && arg.indexOf('"') >= 0) {
+            // There is ["] inside.
+            throw new IllegalArgumentException(errorMessage);
+        }
+        return false;
     }
-    return false;
-  }
 
-  /**
-   * kill yarn application
-   *
-   * @param appIds      app id list
-   * @param logger      logger
-   * @param tenantCode  tenant code
-   * @param executePath     execute path
-   * @throws IOException io exception
-   */
-  public static void cancelApplication(List<String> appIds, Logger logger, String tenantCode,String executePath)
-          throws IOException {
-    if (appIds.size() > 0) {
-      String appid = appIds.get(appIds.size() - 1);
-      String commandFile = String
-              .format("%s/%s.kill", executePath, appid);
-      String cmd = "yarn application -kill " + appid;
-      try {
+    /**
+     * whether needs escaping
+     *
+     * @param verificationType verification type
+     * @param arg              arg
+     * @return boolean
+     */
+    private static boolean needsEscaping(int verificationType, String arg) {
+
+        boolean argIsQuoted = isQuoted((verificationType == VERIFICATION_CMD_BAT), arg, "Argument has embedded quote, use the explicit CMD.EXE call.");
+
+        if (!argIsQuoted) {
+            char[] testEscape = ESCAPE_VERIFICATION[verificationType];
+            for (char c : testEscape) {
+                if (arg.indexOf(c) >= 0) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    /**
+     * kill yarn application
+     *
+     * @param appIds      app id list
+     * @param logger      logger
+     * @param tenantCode  tenant code
+     * @param executePath execute path
+     */
+    public static void cancelApplication(List<String> appIds, Logger logger, String tenantCode, String executePath) {
+        if (CollectionUtils.isNotEmpty(appIds)) {
+
+            for (String appId : appIds) {
+                try {
+                    ExecutionStatus applicationStatus = HadoopUtils.getInstance().getApplicationStatus(appId);
+
+                    if (!applicationStatus.typeIsFinished()) {
+                        String commandFile = String
+                            .format("%s/%s.kill", executePath, appId);
+                        String cmd = "yarn application -kill " + appId;
+                        execYarnKillCommand(logger, tenantCode, appId, commandFile, cmd);
+                    }
+                } catch (Exception e) {
+                    logger.error(String.format("Get yarn application app id [%s] status failed: [%s]", appId, e.getMessage()));
+                }
+            }
+        }
+    }
+
+    /**
+     * build kill command for yarn application
+     *
+     * @param logger      logger
+     * @param tenantCode  tenant code
+     * @param appId       app id
+     * @param commandFile command file
+     * @param cmd         cmd
+     */
+    private static void execYarnKillCommand(Logger logger, String tenantCode, String appId, String commandFile, String cmd) {
+        try {
+            StringBuilder sb = new StringBuilder();
+            sb.append("#!/bin/sh\n");
+            sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n");
+            sb.append("cd $BASEDIR\n");
+            if (CommonUtils.getSystemEnvPath() != null) {
+                sb.append("source ").append(CommonUtils.getSystemEnvPath()).append("\n");
+            }
+            sb.append("\n\n");
+            sb.append(cmd);
+
+            File f = new File(commandFile);
+
+            if (!f.exists()) {
+                FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8);
+            }
+
+            String runCmd = String.format("%s %s", Constants.SH, commandFile);
+            if (StringUtils.isNotEmpty(tenantCode)) {
+                runCmd = "sudo -u " + tenantCode + " " + runCmd;
+            }
+
+            logger.info("kill cmd:{}", runCmd);
+            OSUtils.exeCmd(runCmd);
+        } catch (Exception e) {
+            logger.error(String.format("Kill yarn application app id [%s] failed: [%s]", appId, e.getMessage()));
+        }
+    }
+
+    /**
+     * kill tasks according to different task types
+     *
+     * @param taskExecutionContext taskExecutionContext
+     */
+    public static void kill(TaskExecutionContext taskExecutionContext) {
+        try {
+            int processId = taskExecutionContext.getProcessId();
+            if (processId == 0) {
+                logger.error("process kill failed, process id :{}, task id:{}",
+                    processId, taskExecutionContext.getTaskInstanceId());
+                return;
+            }
+
+            String cmd = String.format("sudo kill -9 %s", getPidsStr(processId));
+
+            logger.info("process id:{}, cmd:{}", processId, cmd);
+
+            OSUtils.exeCmd(cmd);
+
+            // find log and kill yarn job
+            killYarnJob(taskExecutionContext);
+
+        } catch (Exception e) {
+            logger.error("kill task failed", e);
+        }
+    }
+
+    /**
+     * get pids str
+     *
+     * @param processId process id
+     * @return pids pid String
+     * @throws Exception exception
+     */
+    public static String getPidsStr(int processId) throws Exception {
         StringBuilder sb = new StringBuilder();
-        sb.append("#!/bin/sh\n");
-        sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n");
-        sb.append("cd $BASEDIR\n");
-        if (CommonUtils.getSystemEnvPath() != null) {
-          sb.append("source " + CommonUtils.getSystemEnvPath() + "\n");
-        }
-        sb.append("\n\n");
-        sb.append(cmd);
-
-        File f = new File(commandFile);
-
-        if (!f.exists()) {
-          FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8);
+        Matcher mat = null;
+        // pstree pid get sub pids
+        if (OSUtils.isMacOS()) {
+            String pids = OSUtils.exeCmd(String.format("%s -sp %d", Constants.PSTREE, processId));
+            if (null != pids) {
+                mat = MACPATTERN.matcher(pids);
+            }
+        } else {
+            String pids = OSUtils.exeCmd(String.format("%s -p %d", Constants.PSTREE, processId));
+            mat = LINUXPATTERN.matcher(pids);
         }
 
-        String runCmd = "sh " + commandFile;
-        if (StringUtils.isNotEmpty(tenantCode)) {
-          runCmd = "sudo -u " + tenantCode + " " + runCmd;
+        if (null != mat) {
+            while (mat.find()) {
+                sb.append(mat.group(1)).append(" ");
+            }
         }
 
-        logger.info("kill cmd:{}", runCmd);
-
-        Runtime.getRuntime().exec(runCmd);
-      } catch (Exception e) {
-        logger.error("kill application error", e);
-      }
-    }
-  }
-
-  /**
-   * kill tasks according to different task types
-   *
-   * @param taskExecutionContext  taskExecutionContext
-   */
-  public static void kill(TaskExecutionContext taskExecutionContext) {
-    try {
-      int processId = taskExecutionContext.getProcessId();
-      if(processId == 0 ){
-        logger.error("process kill failed, process id :{}, task id:{}",
-                processId, taskExecutionContext.getTaskInstanceId());
-        return ;
-      }
-
-      String cmd = String.format("sudo kill -9 %s", getPidsStr(processId));
-
-      logger.info("process id:{}, cmd:{}", processId, cmd);
-
-      OSUtils.exeCmd(cmd);
-
-      // find log and kill yarn job
-      killYarnJob(taskExecutionContext);
-
-    } catch (Exception e) {
-      logger.error("kill task failed", e);
-    }
-  }
-
-  /**
-   * get pids str
-   *
-   * @param processId process id
-   * @return pids
-   * @throws Exception exception
-   */
-  public static String getPidsStr(int processId)throws Exception{
-    StringBuilder sb = new StringBuilder();
-    Matcher mat;
-    // pstree pid get sub pids
-    if (OSUtils.isMacOS()) {
-      String pids = OSUtils.exeCmd("pstree -sp " + processId);
-      mat = Pattern.compile("-[+|-]-\\s(\\d+)").matcher(pids);
-    } else {
-      String pids = OSUtils.exeCmd("pstree -p " + processId);
-      mat = Pattern.compile("(\\d+)").matcher(pids);
+        return sb.toString().trim();
     }
 
-    while (mat.find()){
-      sb.append(mat.group(1)).append(" ");
-    }
-    return sb.toString().trim();
-  }
+    /**
+     * find logs and kill yarn tasks
+     *
+     * @param taskExecutionContext taskExecutionContext
+     */
+    public static void killYarnJob(TaskExecutionContext taskExecutionContext) {
+        try {
+            Thread.sleep(Constants.SLEEP_TIME_MILLIS);
+            LogClientService logClient = null;
+            String log;
+            try {
+                logClient = new LogClientService();
+                log = logClient.viewLog(Host.of(taskExecutionContext.getHost()).getIp(),
+                    Constants.RPC_PORT,
+                    taskExecutionContext.getLogPath());
+            } finally {
+                if (logClient != null) {
+                    logClient.close();
+                }
+            }
+            if (StringUtils.isNotEmpty(log)) {
+                List<String> appIds = LoggerUtils.getAppIds(log, logger);
+                String workerDir = taskExecutionContext.getExecutePath();
+                if (StringUtils.isEmpty(workerDir)) {
+                    logger.error("task instance work dir is empty");
+                    throw new RuntimeException("task instance work dir is empty");
+                }
+                if (CollectionUtils.isNotEmpty(appIds)) {
+                    cancelApplication(appIds, logger, taskExecutionContext.getTenantCode(), taskExecutionContext.getExecutePath());
+                }
+            }
 
-  /**
-   * find logs and kill yarn tasks
-   *
-   * @param taskExecutionContext  taskExecutionContext
-   */
-  public static void killYarnJob(TaskExecutionContext taskExecutionContext) {
-    try {
-      Thread.sleep(Constants.SLEEP_TIME_MILLIS);
-      LogClientService logClient = null;
-      String log = null;
-      try {
-        logClient = new LogClientService();
-        log = logClient.viewLog(Host.of(taskExecutionContext.getHost()).getIp(),
-                Constants.RPC_PORT,
-                taskExecutionContext.getLogPath());
-      } finally {
-        if(logClient != null){
-          logClient.close();
+        } catch (Exception e) {
+            logger.error("kill yarn job failure", e);
         }
-      }
-      if (StringUtils.isNotEmpty(log)) {
-        List<String> appIds = LoggerUtils.getAppIds(log, logger);
-        String workerDir = taskExecutionContext.getExecutePath();
-        if (StringUtils.isEmpty(workerDir)) {
-          logger.error("task instance work dir is empty");
-          throw new RuntimeException("task instance work dir is empty");
-        }
-        if (appIds.size() > 0) {
-          cancelApplication(appIds, logger, taskExecutionContext.getTenantCode(), taskExecutionContext.getExecutePath());
-        }
-      }
-
-    } catch (Exception e) {
-      logger.error("kill yarn job failure",e);
     }
-  }
 }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/TaskExecutionContextCacheManager.java

@@ -42,7 +42,16 @@ public interface TaskExecutionContextCacheManager {
 
     /**
      * remove taskInstance by taskInstanceId
+     *
      * @param taskInstanceId taskInstanceId
      */
     void removeByTaskInstanceId(Integer taskInstanceId);
+
+    /**
+     * If the value for the specified key is present and non-null,then perform the update，otherwise it will return false
+     *
+     * @param taskExecutionContext taskExecutionContext
+     * @return status
+     */
+    boolean updateTaskExecutionContext(TaskExecutionContext taskExecutionContext);
 }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/impl/TaskExecutionContextCacheManagerImpl.java

@@ -59,10 +59,17 @@ public class TaskExecutionContextCacheManagerImpl implements TaskExecutionContex
 
     /**
      * remove taskInstance by taskInstanceId
+     *
      * @param taskInstanceId taskInstanceId
      */
     @Override
     public void removeByTaskInstanceId(Integer taskInstanceId) {
         taskExecutionContextCache.remove(taskInstanceId);
     }
+
+    @Override
+    public boolean updateTaskExecutionContext(TaskExecutionContext taskExecutionContext) {
+        taskExecutionContextCache.computeIfPresent(taskExecutionContext.getTaskInstanceId(), (k, v) -> taskExecutionContext);
+        return taskExecutionContextCache.containsKey(taskExecutionContext.getTaskInstanceId());
+    }
 }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java

@@ -25,10 +25,10 @@ import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.Event;
 import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.common.enums.TaskType;
-import org.apache.dolphinscheduler.common.utils.OSUtils;
-import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.common.utils.FileUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.common.utils.OSUtils;
 import org.apache.dolphinscheduler.common.utils.Preconditions;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
@@ -37,14 +37,19 @@ import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand;
 import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
 import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
 import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
 import org.apache.dolphinscheduler.server.worker.cache.ResponceCache;
+import org.apache.dolphinscheduler.server.worker.cache.TaskExecutionContextCacheManager;
+import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl;
 import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
 import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.Date;
+import java.util.Optional;
 import java.util.concurrent.ExecutorService;
 
 /**
@@ -54,41 +59,73 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
 
     private final Logger logger = LoggerFactory.getLogger(TaskExecuteProcessor.class);
 
-
     /**
      *  thread executor service
      */
     private final ExecutorService workerExecService;
 
     /**
-     *  worker config
+     * worker config
      */
     private final WorkerConfig workerConfig;
 
     /**
-     *  task callback service
+     * task callback service
      */
     private final TaskCallbackService taskCallbackService;
 
-    public TaskExecuteProcessor(){
+    /**
+     * taskExecutionContextCacheManager
+     */
+    private TaskExecutionContextCacheManager taskExecutionContextCacheManager;
+
+    public TaskExecuteProcessor() {
         this.taskCallbackService = SpringApplicationContext.getBean(TaskCallbackService.class);
         this.workerConfig = SpringApplicationContext.getBean(WorkerConfig.class);
         this.workerExecService = ThreadUtils.newDaemonFixedThreadExecutor("Worker-Execute-Thread", workerConfig.getWorkerExecThreads());
+        this.taskExecutionContextCacheManager = SpringApplicationContext.getBean(TaskExecutionContextCacheManagerImpl.class);
+    }
+
+    /**
+     * Pre-cache task to avoid extreme situations when kill task. There is no such task in the cache
+     *
+     * @param taskExecutionContext task
+     */
+    private void setTaskCache(TaskExecutionContext taskExecutionContext) {
+        TaskExecutionContext preTaskCache = new TaskExecutionContext();
+        preTaskCache.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
+        taskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext);
     }
 
     @Override
     public void process(Channel channel, Command command) {
         Preconditions.checkArgument(CommandType.TASK_EXECUTE_REQUEST == command.getType(),
-                String.format("invalid command type : %s", command.getType()));
+            String.format("invalid command type : %s", command.getType()));
 
         TaskExecuteRequestCommand taskRequestCommand = FastJsonSerializer.deserialize(
-                command.getBody(), TaskExecuteRequestCommand.class);
+            command.getBody(), TaskExecuteRequestCommand.class);
 
         logger.info("received command : {}", taskRequestCommand);
 
-        String contextJson = taskRequestCommand.getTaskExecutionContext();
+        if (taskRequestCommand == null) {
+            logger.error("task execute request command is null");
+            return;
+        }
 
+        String contextJson = taskRequestCommand.getTaskExecutionContext();
         TaskExecutionContext taskExecutionContext = JSONObject.parseObject(contextJson, TaskExecutionContext.class);
+
+        if (taskExecutionContext == null) {
+            logger.error("task execution context is null");
+            return;
+        }
+        setTaskCache(taskExecutionContext);
+        // custom logger
+        Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
+                taskExecutionContext.getProcessDefineId(),
+                taskExecutionContext.getProcessInstanceId(),
+                taskExecutionContext.getTaskInstanceId()));
+
         taskExecutionContext.setHost(OSUtils.getHost() + ":" + workerConfig.getListenPort());
         taskExecutionContext.setStartTime(new Date());
         taskExecutionContext.setLogPath(getTaskLogPath(taskExecutionContext));
@@ -97,18 +134,24 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
         String execLocalPath = getExecLocalPath(taskExecutionContext);
         logger.info("task instance  local execute path : {} ", execLocalPath);
 
+        FileUtils.taskLoggerThreadLocal.set(taskLogger);
         try {
             FileUtils.createWorkDirAndUserIfAbsent(execLocalPath, taskExecutionContext.getTenantCode());
-        } catch (Exception ex){
-            logger.error(String.format("create execLocalPath : %s", execLocalPath), ex);
+        } catch (Throwable ex) {
+            String errorLog = String.format("create execLocalPath : %s", execLocalPath);
+            LoggerUtils.logError(Optional.ofNullable(logger), errorLog, ex);
+            LoggerUtils.logError(Optional.ofNullable(taskLogger), errorLog, ex);
+            taskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
         }
+        FileUtils.taskLoggerThreadLocal.remove();
+
         taskCallbackService.addRemoteChannel(taskExecutionContext.getTaskInstanceId(),
                 new NettyRemoteChannel(channel, command.getOpaque()));
 
         this.doAck(taskExecutionContext);
 
         // submit task
-        workerExecService.submit(new TaskExecuteThread(taskExecutionContext, taskCallbackService));
+        workerExecService.submit(new TaskExecuteThread(taskExecutionContext, taskCallbackService, taskLogger));
     }
 
     private void doAck(TaskExecutionContext taskExecutionContext){

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java

@@ -102,15 +102,17 @@ public class TaskKillProcessor implements NettyRequestProcessor {
      * @return kill result
      */
     private Pair<Boolean, List<String>> doKill(TaskKillRequestCommand killCommand){
-        List<String> appIds = Collections.EMPTY_LIST;
+        List<String> appIds = Collections.emptyList();
         try {
-            TaskExecutionContext taskExecutionContext = taskExecutionContextCacheManager.getByTaskInstanceId(killCommand.getTaskInstanceId());
+            int taskInstanceId = killCommand.getTaskInstanceId();
+            TaskExecutionContext taskExecutionContext = taskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId);
 
             Integer processId = taskExecutionContext.getProcessId();
 
-            if (processId == null || processId.equals(0)){
-                logger.error("process kill failed, process id :{}, task id:{}", processId, killCommand.getTaskInstanceId());
-                return Pair.of(false, appIds);
+            if (processId.equals(0)) {
+                taskExecutionContextCacheManager.removeByTaskInstanceId(taskInstanceId);
+                logger.info("the task has not been executed and has been cancelled, task id:{}", taskInstanceId);
+                return Pair.of(true, appIds);
             }
 
             String cmd = String.format("sudo kill -9 %s", ProcessUtils.getPidsStr(taskExecutionContext.getProcessId()));

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java

@@ -14,8 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.dolphinscheduler.server.worker.runner;
 
+package org.apache.dolphinscheduler.server.worker.runner;
 
 import com.alibaba.fastjson.JSONObject;
 import org.apache.commons.collections.MapUtils;
@@ -26,7 +26,7 @@ import org.apache.dolphinscheduler.common.process.Property;
 import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
 import org.apache.dolphinscheduler.common.utils.CommonUtils;
 import org.apache.dolphinscheduler.common.utils.HadoopUtils;
-import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand;
 import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
 import org.apache.dolphinscheduler.server.worker.cache.ResponceCache;
@@ -36,18 +36,21 @@ import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService;
 import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
 import org.apache.dolphinscheduler.server.worker.task.TaskManager;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import static ch.qos.logback.classic.ClassicConstants.FINALIZE_SESSION_MARKER;
-
-
 /**
- *  task scheduler thread
+ * task scheduler thread
  */
 public class TaskExecuteThread implements Runnable {
 
@@ -57,17 +60,17 @@ public class TaskExecuteThread implements Runnable {
     private final Logger logger = LoggerFactory.getLogger(TaskExecuteThread.class);
 
     /**
-     *  task instance
+     * task instance
      */
     private TaskExecutionContext taskExecutionContext;
 
     /**
-     *  abstract task
+     * abstract task
      */
     private AbstractTask task;
 
     /**
-     *  task callback service
+     * task callback service
      */
     private TaskCallbackService taskCallbackService;
 
@@ -76,15 +79,23 @@ public class TaskExecuteThread implements Runnable {
      */
     private TaskExecutionContextCacheManager taskExecutionContextCacheManager;
 
+    /**
+     * task logger
+     */
+    private Logger taskLogger;
+
     /**
      *  constructor
      * @param taskExecutionContext taskExecutionContext
      * @param taskCallbackService taskCallbackService
      */
-    public TaskExecuteThread(TaskExecutionContext taskExecutionContext, TaskCallbackService taskCallbackService){
+    public TaskExecuteThread(TaskExecutionContext taskExecutionContext
+            , TaskCallbackService taskCallbackService
+            , Logger taskLogger) {
         this.taskExecutionContext = taskExecutionContext;
         this.taskCallbackService = taskCallbackService;
         this.taskExecutionContextCacheManager = SpringApplicationContext.getBean(TaskExecutionContextCacheManagerImpl.class);
+        this.taskLogger = taskLogger;
     }
 
     @Override
@@ -113,14 +124,7 @@ public class TaskExecuteThread implements Runnable {
                     taskExecutionContext.getProcessInstanceId(),
                     taskExecutionContext.getTaskInstanceId()));
 
-            // custom logger
-            Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
-                    taskExecutionContext.getProcessDefineId(),
-                    taskExecutionContext.getProcessInstanceId(),
-                    taskExecutionContext.getTaskInstanceId()));
-
-            task = TaskManager.newTask(taskExecutionContext,
-                    taskLogger);
+            task = TaskManager.newTask(taskExecutionContext, taskLogger);
 
             // task init
             task.init();
@@ -135,7 +139,7 @@ public class TaskExecuteThread implements Runnable {
             responseCommand.setProcessId(task.getProcessId());
             responseCommand.setAppIds(task.getAppIds());
             logger.info("task instance id : {},task final status : {}", taskExecutionContext.getTaskInstanceId(), task.getExitStatus());
-        }catch (Exception e){
+        } catch (Exception e) {
             logger.error("task scheduler failure", e);
             kill();
             responseCommand.setStatus(ExecutionStatus.FAILURE.getCode());
@@ -144,18 +148,47 @@ public class TaskExecuteThread implements Runnable {
             responseCommand.setAppIds(task.getAppIds());
         } finally {
             taskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
-            ResponceCache.get().cache(taskExecutionContext.getTaskInstanceId(),responseCommand.convert2Command(),Event.RESULT);
+            ResponceCache.get().cache(taskExecutionContext.getTaskInstanceId(), responseCommand.convert2Command(), Event.RESULT);
             taskCallbackService.sendResult(taskExecutionContext.getTaskInstanceId(), responseCommand.convert2Command());
-
+            clearTaskExecPath();
         }
     }
 
+    /**
+     * when task finish, clear execute path.
+     */
+    private void clearTaskExecPath() {
+        logger.info("develop mode is: {}", CommonUtils.isDevelopMode());
+
+        if (!CommonUtils.isDevelopMode()) {
+            // get exec dir
+            String execLocalPath = taskExecutionContext.getExecutePath();
+
+            if (StringUtils.isEmpty(execLocalPath)) {
+                logger.warn("task: {} exec local path is empty.", taskExecutionContext.getTaskName());
+                return;
+            }
+
+            if ("/".equals(execLocalPath)) {
+                logger.warn("task: {} exec local path is '/'， direct deletion is not allowed", taskExecutionContext.getTaskName());
+                return;
+            }
+
+            try {
+                org.apache.commons.io.FileUtils.deleteDirectory(new File(execLocalPath));
+                logger.info("exec local path: {} cleared.", execLocalPath);
+            } catch (IOException e) {
+                logger.error("delete exec dir failed : {}", e.getMessage(), e);
+            }
+        }
+
+    }
+
     /**
      * get global paras map
-     * @return
      */
     private Map<String, String> getGlobalParamsMap() {
-        Map<String,String> globalParamsMap = new HashMap<>(16);
+        Map<String, String> globalParamsMap = new HashMap<>(16);
 
         // global params string
         String globalParamsStr = taskExecutionContext.getGlobalParams();
@@ -168,17 +201,17 @@ public class TaskExecuteThread implements Runnable {
 
     /**
      * set task timeout
+     *
      * @param taskExecutionContext TaskExecutionContext
-     * @param taskNode
      */
     private void setTaskTimeout(TaskExecutionContext taskExecutionContext, TaskNode taskNode) {
         // the default timeout is the maximum value of the integer
         taskExecutionContext.setTaskTimeout(Integer.MAX_VALUE);
         TaskTimeoutParameter taskTimeoutParameter = taskNode.getTaskTimeoutParameter();
-        if (taskTimeoutParameter.getEnable()){
+        if (taskTimeoutParameter.getEnable()) {
             // get timeout strategy
             taskExecutionContext.setTaskTimeoutStrategy(taskTimeoutParameter.getStrategy().getCode());
-            switch (taskTimeoutParameter.getStrategy()){
+            switch (taskTimeoutParameter.getStrategy()) {
                 case WARN:
                     break;
                 case FAILED:
@@ -199,38 +232,32 @@ public class TaskExecuteThread implements Runnable {
         }
     }
 
-
     /**
-     *  kill task
+     * kill task
      */
-    public void kill(){
-        if (task != null){
+    public void kill() {
+        if (task != null) {
             try {
                 task.cancelApplication(true);
-            }catch (Exception e){
-                logger.error(e.getMessage(),e);
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
             }
         }
     }
 
-
     /**
      * download resource file
-     *
-     * @param execLocalPath
-     * @param projectRes
-     * @param logger
      */
     private void downloadResource(String execLocalPath,
-                                  Map<String,String> projectRes,
+                                  Map<String, String> projectRes,
                                   Logger logger) throws Exception {
-        if (MapUtils.isEmpty(projectRes)){
+        if (MapUtils.isEmpty(projectRes)) {
             return;
         }
 
         Set<Map.Entry<String, String>> resEntries = projectRes.entrySet();
 
-        for (Map.Entry<String,String> resource : resEntries) {
+        for (Map.Entry<String, String> resource : resEntries) {
             String fullName = resource.getKey();
             String tenantCode = resource.getValue();
             File resFile = new File(execLocalPath, fullName);
@@ -241,8 +268,8 @@ public class TaskExecuteThread implements Runnable {
 
                     logger.info("get resource file from hdfs :{}", resHdfsPath);
                     HadoopUtils.getInstance().copyHdfsToLocal(resHdfsPath, execLocalPath + File.separator + fullName, false, true);
-                }catch (Exception e){
-                    logger.error(e.getMessage(),e);
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
                     throw new RuntimeException(e.getMessage());
                 }
             } else {

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java

@@ -47,6 +47,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_FAILURE;
+import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_KILL;
 import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_SUCCESS;
 
 /**
@@ -160,12 +161,18 @@ public abstract class AbstractCommandExecutor {
      * @return CommandExecuteResult
      * @throws Exception if error throws Exception
      */
-    public CommandExecuteResult run(String execCommand) throws Exception{
+    public CommandExecuteResult run(String execCommand) throws Exception {
 
         CommandExecuteResult result = new CommandExecuteResult();
 
-
+        int taskInstanceId = taskExecutionContext.getTaskInstanceId();
+        // If the task has been killed, then the task in the cache is null
+        if (null == taskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId)) {
+            result.setExitStatusCode(EXIT_CODE_KILL);
+            return result;
+        }
         if (StringUtils.isEmpty(execCommand)) {
+            taskExecutionContextCacheManager.removeByTaskInstanceId(taskInstanceId);
             return result;
         }
 
@@ -187,7 +194,12 @@ public abstract class AbstractCommandExecutor {
 
         // cache processId
         taskExecutionContext.setProcessId(processId);
-        taskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext);
+        boolean updateTaskExecutionContextStatus = taskExecutionContextCacheManager.updateTaskExecutionContext(taskExecutionContext);
+        if (Boolean.FALSE.equals(updateTaskExecutionContextStatus)) {
+            ProcessUtils.kill(taskExecutionContext);
+            result.setExitStatusCode(EXIT_CODE_KILL);
+            return result;
+        }
 
         // print process id
         logger.info("process start, process id is: {}", processId);

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java

@@ -16,6 +16,7 @@
  */
 package org.apache.dolphinscheduler.server.worker.task.shell;
 
+import static java.util.Calendar.DAY_OF_MONTH;
 
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.CommandType;
@@ -40,6 +41,8 @@ import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Date;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 
@@ -131,25 +134,26 @@ public class ShellTask extends AbstractTask {
      *  combining local and global parameters
      */
     Map<String, Property> paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()),
-            taskExecutionContext.getDefinedParams(),
-            shellParameters.getLocalParametersMap(),
-            CommandType.of(taskExecutionContext.getCmdTypeIfComplement()),
-            taskExecutionContext.getScheduleTime());
-    if (paramsMap != null){
-      script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
-    }
-    // new
+        taskExecutionContext.getDefinedParams(),
+        shellParameters.getLocalParametersMap(),
+        CommandType.of(taskExecutionContext.getCmdTypeIfComplement()),
+        taskExecutionContext.getScheduleTime());
     // replace variable TIME with $[YYYYmmddd...] in shell file when history run job and batch complement job
-    if (paramsMap != null) {
-      if (taskExecutionContext.getScheduleTime() != null) {
-        String dateTime = DateUtils.format(taskExecutionContext.getScheduleTime(), Constants.PARAMETER_FORMAT_TIME);
-        Property p = new Property();
-        p.setValue(dateTime);
-        p.setProp(Constants.PARAMETER_SHECDULE_TIME);
-        paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
+    if (taskExecutionContext.getScheduleTime() != null) {
+      if (paramsMap == null) {
+        paramsMap = new HashMap<>();
       }
-      script = ParameterUtils.convertParameterPlaceholders2(script, ParamUtils.convert(paramsMap));
+      Date date = taskExecutionContext.getScheduleTime();
+      if (CommandType.COMPLEMENT_DATA.getCode() == taskExecutionContext.getCmdTypeIfComplement()) {
+        date = DateUtils.add(taskExecutionContext.getScheduleTime(), DAY_OF_MONTH, 1);
+      }
+      String dateTime = DateUtils.format(date, Constants.PARAMETER_FORMAT_TIME);
+      Property p = new Property();
+      p.setValue(dateTime);
+      p.setProp(Constants.PARAMETER_DATETIME);
+      paramsMap.put(Constants.PARAMETER_DATETIME, p);
     }
+    script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
 
     shellParameters.setRawScript(script);
 

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopConstants.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.task.sqoop;
+
+public final class SqoopConstants {
+
+    private SqoopConstants() {
+    }
+
+    //sqoop general param
+    public static final String SQOOP = "sqoop";
+    public static final String SQOOP_MR_JOB_NAME = "mapred.job.name";
+    public static final String SQOOP_PARALLELISM = "-m";
+    public static final String FIELDS_TERMINATED_BY = "--fields-terminated-by";
+    public static final String LINES_TERMINATED_BY = "--lines-terminated-by";
+    public static final String FIELD_NULL_PLACEHOLDER = "--null-non-string 'NULL' --null-string 'NULL'";
+
+    //sqoop db
+    public static final String DB_CONNECT = "--connect";
+    public static final String DB_USERNAME = "--username";
+    public static final String DB_PWD = "--password";
+    public static final String TABLE = "--table";
+    public static final String COLUMNS = "--columns";
+    public static final String QUERY_WHERE = "where";
+    public static final String QUERY = "--query";
+    public static final String QUERY_CONDITION = "AND \\$CONDITIONS";
+    public static final String QUERY_WITHOUT_CONDITION = "WHERE \\$CONDITIONS";
+    public static final String MAP_COLUMN_HIVE = "--map-column-hive";
+    public static final String MAP_COLUMN_JAVA = "--map-column-java";
+
+
+    //sqoop hive source
+    public static final String HCATALOG_DATABASE = "--hcatalog-database";
+    public static final String HCATALOG_TABLE = "--hcatalog-table";
+    public static final String HCATALOG_PARTITION_KEYS = "--hcatalog-partition-keys";
+    public static final String HCATALOG_PARTITION_VALUES = "--hcatalog-partition-values";
+
+    //sqoop hdfs
+    public static final String HDFS_EXPORT_DIR = "--export-dir";
+    public static final String TARGET_DIR = "--target-dir";
+    public static final String COMPRESSION_CODEC = "--compression-codec";
+
+    //sqoop hive
+    public static final String HIVE_IMPORT = "--hive-import";
+    public static final String HIVE_DATABASE = "--hive-database";
+    public static final String HIVE_TABLE = "--hive-table";
+    public static final String CREATE_HIVE_TABLE = "--create-hive-table";
+    public static final String HIVE_DROP_IMPORT_DELIMS = "--hive-drop-import-delims";
+    public static final String HIVE_OVERWRITE = "--hive-overwrite";
+    public static final String DELETE_TARGET_DIR = "--delete-target-dir";
+    public static final String HIVE_DELIMS_REPLACEMENT = "--hive-delims-replacement";
+    public static final String HIVE_PARTITION_KEY = "--hive-partition-key";
+    public static final String HIVE_PARTITION_VALUE = "--hive-partition-value";
+
+    //sqoop update model
+    public static final String UPDATE_KEY = "--update-key";
+    public static final String UPDATE_MODE = "--update-mode";
+
+
+}

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java

@@ -14,63 +14,73 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.server.worker.task.sqoop;
 
-import com.alibaba.fastjson.JSON;
 import org.apache.dolphinscheduler.common.enums.CommandType;
 import org.apache.dolphinscheduler.common.process.Property;
 import org.apache.dolphinscheduler.common.task.AbstractParameters;
 import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.common.utils.ParameterUtils;
 import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
 import org.apache.dolphinscheduler.server.utils.ParamUtils;
 import org.apache.dolphinscheduler.server.worker.task.AbstractYarnTask;
 import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator;
-import org.slf4j.Logger;
+
 import java.util.Map;
 
+import org.slf4j.Logger;
+
 /**
  * sqoop task extends the shell task
  */
 public class SqoopTask extends AbstractYarnTask {
 
+    /**
+     * sqoop task params
+     */
     private SqoopParameters sqoopParameters;
 
     /**
      * taskExecutionContext
      */
-    private TaskExecutionContext taskExecutionContext;
+    private final TaskExecutionContext sqoopTaskExecutionContext;
 
-    public SqoopTask(TaskExecutionContext taskExecutionContext, Logger logger){
-        super(taskExecutionContext,logger);
-        this.taskExecutionContext = taskExecutionContext;
+    public SqoopTask(TaskExecutionContext taskExecutionContext, Logger logger) {
+        super(taskExecutionContext, logger);
+        this.sqoopTaskExecutionContext = taskExecutionContext;
     }
 
     @Override
-    public void init() throws Exception {
-        logger.info("sqoop task params {}", taskExecutionContext.getTaskParams());
+    public void init() {
+        logger.info("sqoop task params {}", sqoopTaskExecutionContext.getTaskParams());
         sqoopParameters =
-                JSON.parseObject(taskExecutionContext.getTaskParams(),SqoopParameters.class);
-        if (!sqoopParameters.checkParameters()) {
-            throw new RuntimeException("sqoop task params is not valid");
+            JSONUtils.parseObject(sqoopTaskExecutionContext.getTaskParams(), SqoopParameters.class);
+        //check sqoop task params
+        if (null == sqoopParameters) {
+            throw new IllegalArgumentException("Sqoop Task params is null");
         }
 
+        if (!sqoopParameters.checkParameters()) {
+            throw new IllegalArgumentException("Sqoop Task params check fail");
+        }
     }
 
     @Override
-    protected String buildCommand() throws Exception {
+    protected String buildCommand() {
         //get sqoop scripts
         SqoopJobGenerator generator = new SqoopJobGenerator();
-        String script = generator.generateSqoopJob(sqoopParameters,taskExecutionContext);
+        String script = generator.generateSqoopJob(sqoopParameters, sqoopTaskExecutionContext);
 
-        Map<String, Property> paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()),
-                taskExecutionContext.getDefinedParams(),
-                sqoopParameters.getLocalParametersMap(),
-                CommandType.of(taskExecutionContext.getCmdTypeIfComplement()),
-                taskExecutionContext.getScheduleTime());
+        Map<String, Property> paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(sqoopTaskExecutionContext.getDefinedParams()),
+            sqoopTaskExecutionContext.getDefinedParams(),
+            sqoopParameters.getLocalParametersMap(),
+            CommandType.of(sqoopTaskExecutionContext.getCmdTypeIfComplement()),
+            sqoopTaskExecutionContext.getScheduleTime());
 
-        if(paramsMap != null){
-            String resultScripts = ParameterUtils.convertParameterPlaceholders(script,  ParamUtils.convert(paramsMap));
+        if (paramsMap != null) {
+            String resultScripts = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
             logger.info("sqoop script: {}", resultScripts);
             return resultScripts;
         }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/CommonGenerator.java

@@ -14,9 +14,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.server.worker.task.sqoop.generator;
 
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.process.Property;
 import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.apache.dolphinscheduler.common.utils.CollectionUtils;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopConstants;
+
+import java.util.List;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -25,21 +33,54 @@ import org.slf4j.LoggerFactory;
  */
 public class CommonGenerator {
 
-    private Logger logger = LoggerFactory.getLogger(getClass());
+    private static final Logger logger = LoggerFactory.getLogger(CommonGenerator.class);
 
     public String generate(SqoopParameters sqoopParameters) {
-        StringBuilder result = new StringBuilder();
-        try{
-            result.append("sqoop ")
-                    .append(sqoopParameters.getModelType());
-            if(sqoopParameters.getConcurrency() >0){
-                result.append(" -m ")
-                        .append(sqoopParameters.getConcurrency());
+
+        StringBuilder commonSb = new StringBuilder();
+
+        try {
+            //sqoop task model
+            commonSb.append(SqoopConstants.SQOOP)
+                .append(Constants.SPACE)
+                .append(sqoopParameters.getModelType());
+
+            //sqoop map-reduce job name
+            commonSb.append(Constants.SPACE).append(Constants.D).append(Constants.SPACE)
+                .append(String.format("%s%s%s", SqoopConstants.SQOOP_MR_JOB_NAME,
+                    Constants.EQUAL_SIGN, sqoopParameters.getJobName()));
+
+            //hadoop custom param
+            List<Property> hadoopCustomParams = sqoopParameters.getHadoopCustomParams();
+            if (CollectionUtils.isNotEmpty(hadoopCustomParams)) {
+                StringBuilder hadoopCustomParamStr = new StringBuilder();
+                for (Property hadoopCustomParam : hadoopCustomParams) {
+                    hadoopCustomParamStr.append(Constants.D)
+                            .append(Constants.SPACE).append(hadoopCustomParam.getProp())
+                            .append(Constants.EQUAL_SIGN).append(hadoopCustomParam.getValue())
+                            .append(Constants.SPACE);
+                }
+                commonSb.append(Constants.SPACE).append(hadoopCustomParamStr.substring(0, hadoopCustomParamStr.length() - 1));
             }
-        }catch (Exception e){
-            logger.error(e.getMessage());
+
+            //sqoop custom params
+            List<Property> sqoopAdvancedParams = sqoopParameters.getSqoopAdvancedParams();
+            if (CollectionUtils.isNotEmpty(sqoopAdvancedParams)) {
+                for (Property sqoopAdvancedParam : sqoopAdvancedParams) {
+                    commonSb.append(Constants.SPACE).append(sqoopAdvancedParam.getProp())
+                        .append(Constants.SPACE).append(sqoopAdvancedParam.getValue());
+                }
+            }
+
+            //sqoop parallelism
+            if (sqoopParameters.getConcurrency() > 0) {
+                commonSb.append(Constants.SPACE).append(SqoopConstants.SQOOP_PARALLELISM)
+                    .append(Constants.SPACE).append(sqoopParameters.getConcurrency());
+            }
+        } catch (Exception e) {
+            logger.error(String.format("Sqoop task general param build failed: [%s]", e.getMessage()));
         }
 
-        return result.toString();
+        return commonSb.toString();
     }
 }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ISourceGenerator.java

@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.server.worker.task.sqoop.generator;
 
 import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
@@ -26,9 +27,10 @@ public interface ISourceGenerator {
 
     /**
      * generate the source script
-     * @param sqoopParameters sqoopParameters
+     *
+     * @param sqoopParameters      sqoopParameters
      * @param taskExecutionContext taskExecutionContext
      * @return source script
      */
-    String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext);
+    String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext);
 }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ITargetGenerator.java

@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.server.worker.task.sqoop.generator;
 
 import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
@@ -26,9 +27,10 @@ public interface ITargetGenerator {
 
     /**
      * generate the target script
-     * @param sqoopParameters sqoopParameters
+     *
+     * @param sqoopParameters      sqoopParameters
      * @param taskExecutionContext taskExecutionContext
      * @return target script
      */
-    String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext);
+    String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext);
 }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/SqoopJobGenerator.java

@@ -14,8 +14,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.server.worker.task.sqoop.generator;
 
+import org.apache.dolphinscheduler.common.enums.SqoopJobType;
 import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
 import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
 import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources.HdfsSourceGenerator;
@@ -45,40 +47,51 @@ public class SqoopJobGenerator {
     /**
      * common script generator
      */
-    private CommonGenerator commonGenerator;
+    private final CommonGenerator commonGenerator;
 
-    public SqoopJobGenerator(){
+    public SqoopJobGenerator() {
         commonGenerator = new CommonGenerator();
     }
 
-    private void createSqoopJobGenerator(String sourceType,String targetType){
+    private void createSqoopJobGenerator(String sourceType, String targetType) {
         sourceGenerator = createSourceGenerator(sourceType);
         targetGenerator = createTargetGenerator(targetType);
     }
 
     /**
      * get the final sqoop scripts
-     * @param sqoopParameters
-     * @return
+     *
+     * @param sqoopParameters sqoop params
+     * @return sqoop scripts
      */
-    public String generateSqoopJob(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext){
-        createSqoopJobGenerator(sqoopParameters.getSourceType(),sqoopParameters.getTargetType());
-        if(sourceGenerator == null || targetGenerator == null){
-            return null;
+    public String generateSqoopJob(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext) {
+
+        String sqoopScripts = "";
+
+        if (SqoopJobType.TEMPLATE.getDescp().equals(sqoopParameters.getJobType())) {
+            createSqoopJobGenerator(sqoopParameters.getSourceType(), sqoopParameters.getTargetType());
+            if (sourceGenerator == null || targetGenerator == null) {
+                throw new RuntimeException("sqoop task source type or target type is null");
+            }
+
+            sqoopScripts = String.format("%s%s%s", commonGenerator.generate(sqoopParameters),
+                sourceGenerator.generate(sqoopParameters, taskExecutionContext),
+                targetGenerator.generate(sqoopParameters, taskExecutionContext));
+        } else if (SqoopJobType.CUSTOM.getDescp().equals(sqoopParameters.getJobType())) {
+            sqoopScripts = sqoopParameters.getCustomShell().replaceAll("\\r\\n", "\n");
         }
 
-        return commonGenerator.generate(sqoopParameters)
-                + sourceGenerator.generate(sqoopParameters,taskExecutionContext)
-                + targetGenerator.generate(sqoopParameters,taskExecutionContext);
+        return sqoopScripts;
     }
 
     /**
      * get the source generator
-     * @param sourceType
-     * @return
+     *
+     * @param sourceType sqoop source type
+     * @return sqoop source generator
      */
-    private ISourceGenerator createSourceGenerator(String sourceType){
-        switch (sourceType){
+    private ISourceGenerator createSourceGenerator(String sourceType) {
+        switch (sourceType) {
             case MYSQL:
                 return new MysqlSourceGenerator();
             case HIVE:
@@ -92,11 +105,12 @@ public class SqoopJobGenerator {
 
     /**
      * get the target generator
-     * @param targetType
-     * @return
+     *
+     * @param targetType sqoop target type
+     * @return sqoop target generator
      */
-    private ITargetGenerator createTargetGenerator(String targetType){
-        switch (targetType){
+    private ITargetGenerator createTargetGenerator(String targetType) {
+        switch (targetType) {
             case MYSQL:
                 return new MysqlTargetGenerator();
             case HIVE:

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java

@@ -14,14 +14,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources;
 
+import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
 import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHdfsParameter;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.common.utils.StringUtils;
 import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopConstants;
 import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ISourceGenerator;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,28 +34,30 @@ import org.slf4j.LoggerFactory;
  */
 public class HdfsSourceGenerator implements ISourceGenerator {
 
-    private Logger logger = LoggerFactory.getLogger(getClass());
+    private static final Logger logger = LoggerFactory.getLogger(HdfsSourceGenerator.class);
 
     @Override
-    public String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext) {
-        StringBuilder result = new StringBuilder();
-        try{
-            SourceHdfsParameter sourceHdfsParameter
-                    = JSONUtils.parseObject(sqoopParameters.getSourceParams(),SourceHdfsParameter.class);
+    public String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext) {
 
-            if(sourceHdfsParameter != null){
-                if(StringUtils.isNotEmpty(sourceHdfsParameter.getExportDir())){
-                    result.append(" --export-dir ")
-                            .append(sourceHdfsParameter.getExportDir());
-                }else{
-                    throw new Exception("--export-dir is null");
+        StringBuilder hdfsSourceSb = new StringBuilder();
+
+        try {
+            SourceHdfsParameter sourceHdfsParameter
+                = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHdfsParameter.class);
+
+            if (null != sourceHdfsParameter) {
+                if (StringUtils.isNotEmpty(sourceHdfsParameter.getExportDir())) {
+                    hdfsSourceSb.append(Constants.SPACE).append(SqoopConstants.HDFS_EXPORT_DIR)
+                        .append(Constants.SPACE).append(sourceHdfsParameter.getExportDir());
+                } else {
+                    throw new IllegalArgumentException("Sqoop hdfs export dir is null");
                 }
 
             }
-        }catch (Exception e){
-            logger.error("get hdfs source failed",e);
+        } catch (Exception e) {
+            logger.error(String.format("Sqoop hdfs source parmas build failed: [%s]", e.getMessage()));
         }
 
-        return result.toString();
+        return hdfsSourceSb.toString();
     }
 }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java

@@ -14,14 +14,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
 import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHiveParameter;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
 import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopConstants;
 import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ISourceGenerator;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,33 +34,40 @@ import org.slf4j.LoggerFactory;
  */
 public class HiveSourceGenerator implements ISourceGenerator {
 
-    private Logger logger = LoggerFactory.getLogger(getClass());
+    private static final Logger logger = LoggerFactory.getLogger(HiveSourceGenerator.class);
 
     @Override
-    public String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext) {
-        StringBuilder sb = new StringBuilder();
-        try{
+    public String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext) {
+
+        StringBuilder hiveSourceSb = new StringBuilder();
+
+        try {
             SourceHiveParameter sourceHiveParameter
-                    = JSONUtils.parseObject(sqoopParameters.getSourceParams(),SourceHiveParameter.class);
-            if(sourceHiveParameter != null){
-                if(StringUtils.isNotEmpty(sourceHiveParameter.getHiveDatabase())){
-                    sb.append(" --hcatalog-database ").append(sourceHiveParameter.getHiveDatabase());
+                = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHiveParameter.class);
+
+            if (null != sourceHiveParameter) {
+                if (StringUtils.isNotEmpty(sourceHiveParameter.getHiveDatabase())) {
+                    hiveSourceSb.append(Constants.SPACE).append(SqoopConstants.HCATALOG_DATABASE)
+                        .append(Constants.SPACE).append(sourceHiveParameter.getHiveDatabase());
                 }
 
-                if(StringUtils.isNotEmpty(sourceHiveParameter.getHiveTable())){
-                    sb.append(" --hcatalog-table ").append(sourceHiveParameter.getHiveTable());
+                if (StringUtils.isNotEmpty(sourceHiveParameter.getHiveTable())) {
+                    hiveSourceSb.append(Constants.SPACE).append(SqoopConstants.HCATALOG_TABLE)
+                        .append(Constants.SPACE).append(sourceHiveParameter.getHiveTable());
                 }
 
-                if(StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionKey())&&
-                        StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionValue())){
-                    sb.append(" --hcatalog-partition-keys ").append(sourceHiveParameter.getHivePartitionKey())
-                            .append(" --hcatalog-partition-values ").append(sourceHiveParameter.getHivePartitionValue());
+                if (StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionKey())
+                    && StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionValue())) {
+                    hiveSourceSb.append(Constants.SPACE).append(SqoopConstants.HCATALOG_PARTITION_KEYS)
+                        .append(Constants.SPACE).append(sourceHiveParameter.getHivePartitionKey())
+                        .append(Constants.SPACE).append(SqoopConstants.HCATALOG_PARTITION_VALUES)
+                        .append(Constants.SPACE).append(sourceHiveParameter.getHivePartitionValue());
                 }
             }
-        }catch (Exception e){
-            logger.error(e.getMessage());
+        } catch (Exception e) {
+            logger.error(String.format("Sqoop hive source params build failed: [%s]", e.getMessage()));
         }
 
-        return sb.toString();
+        return hiveSourceSb.toString();
     }
 }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java

@@ -14,106 +14,124 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.DbType;
-import org.apache.dolphinscheduler.common.enums.QueryType;
+import org.apache.dolphinscheduler.common.enums.SqoopQueryType;
 import org.apache.dolphinscheduler.common.process.Property;
 import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
 import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceMysqlParameter;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
 import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
 import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
 import org.apache.dolphinscheduler.server.entity.SqoopTaskExecutionContext;
 import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopConstants;
 import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ISourceGenerator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.util.List;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * mysql source generator
  */
 public class MysqlSourceGenerator implements ISourceGenerator {
 
-    private Logger logger = LoggerFactory.getLogger(getClass());
+    private static final Logger logger = LoggerFactory.getLogger(MysqlSourceGenerator.class);
 
     @Override
-    public String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext) {
-        StringBuilder result = new StringBuilder();
-        try {
-            SourceMysqlParameter sourceMysqlParameter
-                    = JSONUtils.parseObject(sqoopParameters.getSourceParams(),SourceMysqlParameter.class);
+    public String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext) {
 
+        StringBuilder mysqlSourceSb = new StringBuilder();
+
+        try {
+            SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceMysqlParameter.class);
             SqoopTaskExecutionContext sqoopTaskExecutionContext = taskExecutionContext.getSqoopTaskExecutionContext();
 
-            if(sourceMysqlParameter != null){
+            if (null != sourceMysqlParameter) {
                 BaseDataSource baseDataSource = DataSourceFactory.getDatasource(DbType.of(sqoopTaskExecutionContext.getSourcetype()),
-                        sqoopTaskExecutionContext.getSourceConnectionParams());
-                if(baseDataSource != null){
-                    result.append(" --connect ")
-                            .append(baseDataSource.getJdbcUrl())
-                            .append(" --username ")
-                            .append(baseDataSource.getUser())
-                            .append(" --password ")
-                            .append(baseDataSource.getPassword());
+                    sqoopTaskExecutionContext.getSourceConnectionParams());
 
-                    if(sourceMysqlParameter.getSrcQueryType() == QueryType.FORM.ordinal()){
-                        if(StringUtils.isNotEmpty(sourceMysqlParameter.getSrcTable())){
-                            result.append(" --table ").append(sourceMysqlParameter.getSrcTable());
+                if (null != baseDataSource) {
+
+                    mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.DB_CONNECT)
+                        .append(Constants.SPACE).append(Constants.DOUBLE_QUOTES).append(baseDataSource.getJdbcUrl()).append(Constants.DOUBLE_QUOTES)
+                        .append(Constants.SPACE).append(SqoopConstants.DB_USERNAME)
+                        .append(Constants.SPACE).append(baseDataSource.getUser())
+                        .append(Constants.SPACE).append(SqoopConstants.DB_PWD)
+                        .append(Constants.SPACE).append(Constants.DOUBLE_QUOTES).append(baseDataSource.getPassword()).append(Constants.DOUBLE_QUOTES);
+
+                    //sqoop table & sql query
+                    if (sourceMysqlParameter.getSrcQueryType() == SqoopQueryType.FORM.getCode()) {
+                        if (StringUtils.isNotEmpty(sourceMysqlParameter.getSrcTable())) {
+                            mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.TABLE)
+                                .append(Constants.SPACE).append(sourceMysqlParameter.getSrcTable());
                         }
 
-                        if(StringUtils.isNotEmpty(sourceMysqlParameter.getSrcColumns())){
-                            result.append(" --columns ").append(sourceMysqlParameter.getSrcColumns());
+                        if (StringUtils.isNotEmpty(sourceMysqlParameter.getSrcColumns())) {
+                            mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.COLUMNS)
+                                .append(Constants.SPACE).append(sourceMysqlParameter.getSrcColumns());
                         }
+                    } else if (sourceMysqlParameter.getSrcQueryType() == SqoopQueryType.SQL.getCode()
+                        && StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())) {
 
-                    }else if(sourceMysqlParameter.getSrcQueryType() == QueryType.SQL.ordinal()
-                            && StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())){
                         String srcQuery = sourceMysqlParameter.getSrcQuerySql();
-                        if(srcQuery.toLowerCase().contains("where")){
-                            srcQuery += " AND "+"$CONDITIONS";
-                        }else{
-                            srcQuery += " WHERE $CONDITIONS";
-                        }
-                        result.append(" --query \'").append(srcQuery).append("\'");
+                        mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.QUERY)
+                            .append(Constants.SPACE).append(Constants.DOUBLE_QUOTES).append(srcQuery);
 
-                    }
-
-                    List<Property>  mapColumnHive = sourceMysqlParameter.getMapColumnHive();
-
-                    if(mapColumnHive != null && !mapColumnHive.isEmpty()){
-                        StringBuilder columnMap = new StringBuilder();
-                        for(Property item:mapColumnHive){
-                            columnMap.append(item.getProp()).append("=").append(item.getValue()).append(",");
-                        }
-
-                        if(StringUtils.isNotEmpty(columnMap.toString())){
-                            result.append(" --map-column-hive ")
-                                    .append(columnMap.substring(0,columnMap.length() - 1));
+                        if (srcQuery.toLowerCase().contains(SqoopConstants.QUERY_WHERE)) {
+                            mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.QUERY_CONDITION).append(Constants.DOUBLE_QUOTES);
+                        } else {
+                            mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.QUERY_WITHOUT_CONDITION).append(Constants.DOUBLE_QUOTES);
                         }
                     }
 
-                    List<Property>  mapColumnJava = sourceMysqlParameter.getMapColumnJava();
+                    //sqoop hive map column
+                    List<Property> mapColumnHive = sourceMysqlParameter.getMapColumnHive();
 
-                    if(mapColumnJava != null && !mapColumnJava.isEmpty()){
+                    if (null != mapColumnHive && !mapColumnHive.isEmpty()) {
                         StringBuilder columnMap = new StringBuilder();
-                        for(Property item:mapColumnJava){
-                            columnMap.append(item.getProp()).append("=").append(item.getValue()).append(",");
+                        for (Property item : mapColumnHive) {
+                            if (!item.getProp().isEmpty()) {
+                                columnMap.append(item.getProp()).append(Constants.EQUAL_SIGN)
+                                        .append(item.getValue()).append(Constants.COMMA);
+                            }
                         }
 
-                        if(StringUtils.isNotEmpty(columnMap.toString())){
-                            result.append(" --map-column-java ")
-                                    .append(columnMap.substring(0,columnMap.length() - 1));
+                        if (StringUtils.isNotEmpty(columnMap.toString())) {
+                            mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.MAP_COLUMN_HIVE)
+                                .append(Constants.SPACE).append(columnMap.substring(0, columnMap.length() - 1));
+                        }
+                    }
+
+                    //sqoop map column java
+                    List<Property> mapColumnJava = sourceMysqlParameter.getMapColumnJava();
+
+                    if (null != mapColumnJava && !mapColumnJava.isEmpty()) {
+                        StringBuilder columnJavaMap = new StringBuilder();
+                        for (Property item : mapColumnJava) {
+                            if (!item.getProp().isEmpty()) {
+                                columnJavaMap.append(item.getProp()).append(Constants.EQUAL_SIGN)
+                                        .append(item.getValue()).append(Constants.COMMA);
+                            }
+                        }
+
+                        if (StringUtils.isNotEmpty(columnJavaMap.toString())) {
+                            mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.MAP_COLUMN_JAVA)
+                                .append(Constants.SPACE).append(columnJavaMap.substring(0, columnJavaMap.length() - 1));
                         }
                     }
                 }
             }
-        }catch (Exception e){
-            logger.error(e.getMessage());
+        } catch (Exception e) {
+            logger.error(String.format("Sqoop task mysql source params build failed: [%s]", e.getMessage()));
         }
 
-        return result.toString();
+        return mysqlSourceSb.toString();
     }
 }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java

@@ -14,14 +14,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
 import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHdfsParameter;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
 import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopConstants;
 import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ITargetGenerator;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,47 +34,53 @@ import org.slf4j.LoggerFactory;
  */
 public class HdfsTargetGenerator implements ITargetGenerator {
 
-    private Logger logger = LoggerFactory.getLogger(getClass());
+    private static final Logger logger = LoggerFactory.getLogger(HdfsTargetGenerator.class);
 
     @Override
-    public String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext) {
-        StringBuilder result = new StringBuilder();
-        try{
+    public String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext) {
+
+        StringBuilder hdfsTargetSb = new StringBuilder();
+
+        try {
             TargetHdfsParameter targetHdfsParameter =
-                    JSONUtils.parseObject(sqoopParameters.getTargetParams(),TargetHdfsParameter.class);
+                JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHdfsParameter.class);
 
-            if(targetHdfsParameter != null){
+            if (null != targetHdfsParameter) {
 
-                if(StringUtils.isNotEmpty(targetHdfsParameter.getTargetPath())){
-                    result.append(" --target-dir ").append(targetHdfsParameter.getTargetPath());
+                if (StringUtils.isNotEmpty(targetHdfsParameter.getTargetPath())) {
+                    hdfsTargetSb.append(Constants.SPACE).append(SqoopConstants.TARGET_DIR)
+                        .append(Constants.SPACE).append(targetHdfsParameter.getTargetPath());
                 }
 
-                if(StringUtils.isNotEmpty(targetHdfsParameter.getCompressionCodec())){
-                    result.append(" --compression-codec ").append(targetHdfsParameter.getCompressionCodec());
+                if (StringUtils.isNotEmpty(targetHdfsParameter.getCompressionCodec())) {
+                    hdfsTargetSb.append(Constants.SPACE).append(SqoopConstants.COMPRESSION_CODEC)
+                        .append(Constants.SPACE).append(targetHdfsParameter.getCompressionCodec());
                 }
 
-                if(StringUtils.isNotEmpty(targetHdfsParameter.getFileType())){
-                    result.append(" ").append(targetHdfsParameter.getFileType());
+                if (StringUtils.isNotEmpty(targetHdfsParameter.getFileType())) {
+                    hdfsTargetSb.append(Constants.SPACE).append(targetHdfsParameter.getFileType());
                 }
 
-                if(targetHdfsParameter.isDeleteTargetDir()){
-                    result.append(" --delete-target-dir");
+                if (targetHdfsParameter.isDeleteTargetDir()) {
+                    hdfsTargetSb.append(Constants.SPACE).append(SqoopConstants.DELETE_TARGET_DIR);
                 }
 
-                if(StringUtils.isNotEmpty(targetHdfsParameter.getFieldsTerminated())){
-                    result.append(" --fields-terminated-by '").append(targetHdfsParameter.getFieldsTerminated()).append("'");
+                if (StringUtils.isNotEmpty(targetHdfsParameter.getFieldsTerminated())) {
+                    hdfsTargetSb.append(Constants.SPACE).append(SqoopConstants.FIELDS_TERMINATED_BY)
+                        .append(Constants.SPACE).append(Constants.SINGLE_QUOTES).append(targetHdfsParameter.getFieldsTerminated()).append(Constants.SINGLE_QUOTES);
                 }
 
-                if(StringUtils.isNotEmpty(targetHdfsParameter.getLinesTerminated())){
-                    result.append(" --lines-terminated-by '").append(targetHdfsParameter.getLinesTerminated()).append("'");
+                if (StringUtils.isNotEmpty(targetHdfsParameter.getLinesTerminated())) {
+                    hdfsTargetSb.append(Constants.SPACE).append(SqoopConstants.LINES_TERMINATED_BY)
+                        .append(Constants.SPACE).append(Constants.SINGLE_QUOTES).append(targetHdfsParameter.getLinesTerminated()).append(Constants.SINGLE_QUOTES);
                 }
 
-                result.append(" --null-non-string 'NULL' --null-string 'NULL'");
+                hdfsTargetSb.append(Constants.SPACE).append(SqoopConstants.FIELD_NULL_PLACEHOLDER);
             }
-        }catch(Exception e){
-            logger.error(e.getMessage());
+        } catch (Exception e) {
+            logger.error(String.format("Sqoop hdfs target params build failed: [%s]", e.getMessage()));
         }
 
-        return result.toString();
+        return hdfsTargetSb.toString();
     }
 }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java

@@ -14,14 +14,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
 import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHiveParameter;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
 import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopConstants;
 import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ITargetGenerator;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,57 +34,63 @@ import org.slf4j.LoggerFactory;
  */
 public class HiveTargetGenerator implements ITargetGenerator {
 
-    private Logger logger = LoggerFactory.getLogger(getClass());
+    private static final Logger logger = LoggerFactory.getLogger(HiveTargetGenerator.class);
 
     @Override
-    public String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext) {
+    public String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext) {
 
-        StringBuilder result = new StringBuilder();
+        StringBuilder hiveTargetSb = new StringBuilder();
 
-        try{
+        try {
             TargetHiveParameter targetHiveParameter =
-                    JSONUtils.parseObject(sqoopParameters.getTargetParams(),TargetHiveParameter.class);
-            if(targetHiveParameter != null){
+                JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHiveParameter.class);
+            if (null != targetHiveParameter) {
+                hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.HIVE_IMPORT);
 
-                result.append(" --hive-import ");
-
-                if(StringUtils.isNotEmpty(targetHiveParameter.getHiveDatabase())&&
-                        StringUtils.isNotEmpty(targetHiveParameter.getHiveTable())){
-                    result.append(" --hive-table ")
-                            .append(targetHiveParameter.getHiveDatabase())
-                            .append(".")
-                            .append(targetHiveParameter.getHiveTable());
+                if (StringUtils.isNotEmpty(targetHiveParameter.getHiveDatabase())
+                    && StringUtils.isNotEmpty(targetHiveParameter.getHiveTable())) {
+                    hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.HIVE_DATABASE)
+                        .append(Constants.SPACE).append(targetHiveParameter.getHiveDatabase())
+                        .append(Constants.SPACE).append(SqoopConstants.HIVE_TABLE)
+                        .append(Constants.SPACE).append(targetHiveParameter.getHiveTable());
                 }
 
-                if(targetHiveParameter.isCreateHiveTable()){
-                    result.append(" --create-hive-table");
+                if (targetHiveParameter.isCreateHiveTable()) {
+                    hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.CREATE_HIVE_TABLE);
                 }
 
-                if(targetHiveParameter.isDropDelimiter()){
-                    result.append(" --hive-drop-import-delims");
+                if (targetHiveParameter.isDropDelimiter()) {
+                    hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.HIVE_DROP_IMPORT_DELIMS);
                 }
 
-                if(targetHiveParameter.isHiveOverWrite()){
-                    result.append(" --hive-overwrite -delete-target-dir");
+                if (targetHiveParameter.isHiveOverWrite()) {
+                    hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.HIVE_OVERWRITE)
+                        .append(Constants.SPACE).append(SqoopConstants.DELETE_TARGET_DIR);
                 }
 
-                if(StringUtils.isNotEmpty(targetHiveParameter.getReplaceDelimiter())){
-                    result.append(" --hive-delims-replacement ").append(targetHiveParameter.getReplaceDelimiter());
+                if (StringUtils.isNotEmpty(targetHiveParameter.getHiveTargetDir())) {
+                    hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.TARGET_DIR)
+                            .append(Constants.SPACE).append(targetHiveParameter.getHiveTargetDir());
                 }
 
-                if(StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionKey())&&
-                        StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionValue())){
-                    result.append(" --hive-partition-key ")
-                            .append(targetHiveParameter.getHivePartitionKey())
-                            .append(" --hive-partition-value ")
-                            .append(targetHiveParameter.getHivePartitionValue());
+                if (StringUtils.isNotEmpty(targetHiveParameter.getReplaceDelimiter())) {
+                    hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.HIVE_DELIMS_REPLACEMENT)
+                        .append(Constants.SPACE).append(targetHiveParameter.getReplaceDelimiter());
+                }
+
+                if (StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionKey())
+                    && StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionValue())) {
+                    hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.HIVE_PARTITION_KEY)
+                        .append(Constants.SPACE).append(targetHiveParameter.getHivePartitionKey())
+                        .append(Constants.SPACE).append(SqoopConstants.HIVE_PARTITION_VALUE)
+                        .append(Constants.SPACE).append(targetHiveParameter.getHivePartitionValue());
                 }
 
             }
-        }catch(Exception e){
-            logger.error(e.getMessage());
+        } catch (Exception e) {
+            logger.error(String.format("Sqoop hive target params build failed: [%s]", e.getMessage()));
         }
 
-        return result.toString();
+        return hiveTargetSb.toString();
     }
 }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java

@@ -14,21 +14,22 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.DbType;
 import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
 import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetMysqlParameter;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
 import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
 import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
-import org.apache.dolphinscheduler.dao.entity.DataSource;
 import org.apache.dolphinscheduler.server.entity.SqoopTaskExecutionContext;
 import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopConstants;
 import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ITargetGenerator;
-import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
-import org.apache.dolphinscheduler.service.process.ProcessService;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -37,59 +38,74 @@ import org.slf4j.LoggerFactory;
  */
 public class MysqlTargetGenerator implements ITargetGenerator {
 
-    private Logger logger = LoggerFactory.getLogger(getClass());
+    private static final Logger logger = LoggerFactory.getLogger(MysqlTargetGenerator.class);
 
     @Override
-    public String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext) {
+    public String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext) {
 
-        StringBuilder result = new StringBuilder();
-        try{
+        StringBuilder mysqlTargetSb = new StringBuilder();
 
+        try {
             TargetMysqlParameter targetMysqlParameter =
-                    JSONUtils.parseObject(sqoopParameters.getTargetParams(),TargetMysqlParameter.class);
+                JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetMysqlParameter.class);
 
             SqoopTaskExecutionContext sqoopTaskExecutionContext = taskExecutionContext.getSqoopTaskExecutionContext();
 
-            if(targetMysqlParameter != null && targetMysqlParameter.getTargetDatasource() != 0){
+            if (null != targetMysqlParameter && targetMysqlParameter.getTargetDatasource() != 0) {
 
                 // get datasource
                 BaseDataSource baseDataSource = DataSourceFactory.getDatasource(DbType.of(sqoopTaskExecutionContext.getTargetType()),
-                        sqoopTaskExecutionContext.getTargetConnectionParams());
+                    sqoopTaskExecutionContext.getTargetConnectionParams());
 
-                if(baseDataSource != null){
-                    result.append(" --connect ")
-                            .append(baseDataSource.getJdbcUrl())
-                            .append(" --username ")
-                            .append(baseDataSource.getUser())
-                            .append(" --password ")
-                            .append(baseDataSource.getPassword())
-                            .append(" --table ")
-                            .append(targetMysqlParameter.getTargetTable());
+                if (null != baseDataSource) {
 
-                    if(StringUtils.isNotEmpty(targetMysqlParameter.getTargetColumns())){
-                        result.append(" --columns ").append(targetMysqlParameter.getTargetColumns());
+                    mysqlTargetSb.append(Constants.SPACE).append(SqoopConstants.DB_CONNECT)
+                        .append(Constants.SPACE).append(Constants.DOUBLE_QUOTES).append(baseDataSource.getJdbcUrl()).append(Constants.DOUBLE_QUOTES)
+                        .append(Constants.SPACE).append(SqoopConstants.DB_USERNAME)
+                        .append(Constants.SPACE).append(baseDataSource.getUser())
+                        .append(Constants.SPACE).append(SqoopConstants.DB_PWD)
+                        .append(Constants.SPACE).append(Constants.DOUBLE_QUOTES).append(baseDataSource.getPassword()).append(Constants.DOUBLE_QUOTES)
+                        .append(Constants.SPACE).append(SqoopConstants.TABLE)
+                        .append(Constants.SPACE).append(targetMysqlParameter.getTargetTable());
+
+                    if (StringUtils.isNotEmpty(targetMysqlParameter.getTargetColumns())) {
+                        mysqlTargetSb.append(Constants.SPACE).append(SqoopConstants.COLUMNS)
+                            .append(Constants.SPACE).append(targetMysqlParameter.getTargetColumns());
                     }
 
-                    if(StringUtils.isNotEmpty(targetMysqlParameter.getFieldsTerminated())){
-                        result.append(" --fields-terminated-by '").append(targetMysqlParameter.getFieldsTerminated()).append("'");
+                    if (StringUtils.isNotEmpty(targetMysqlParameter.getFieldsTerminated())) {
+                        mysqlTargetSb.append(Constants.SPACE).append(SqoopConstants.FIELDS_TERMINATED_BY);
+                        if (targetMysqlParameter.getFieldsTerminated().contains("'")) {
+                            mysqlTargetSb.append(Constants.SPACE).append(targetMysqlParameter.getFieldsTerminated());
+
+                        } else {
+                            mysqlTargetSb.append(Constants.SPACE).append(Constants.SINGLE_QUOTES).append(targetMysqlParameter.getFieldsTerminated()).append(Constants.SINGLE_QUOTES);
+                        }
                     }
 
-                    if(StringUtils.isNotEmpty(targetMysqlParameter.getLinesTerminated())){
-                        result.append(" --lines-terminated-by '").append(targetMysqlParameter.getLinesTerminated()).append("'");
+                    if (StringUtils.isNotEmpty(targetMysqlParameter.getLinesTerminated())) {
+                        mysqlTargetSb.append(Constants.SPACE).append(SqoopConstants.LINES_TERMINATED_BY);
+                        if (targetMysqlParameter.getLinesTerminated().contains(Constants.SINGLE_QUOTES)) {
+                            mysqlTargetSb.append(Constants.SPACE).append(targetMysqlParameter.getLinesTerminated());
+                        } else {
+                            mysqlTargetSb.append(Constants.SPACE).append(Constants.SINGLE_QUOTES).append(targetMysqlParameter.getLinesTerminated()).append(Constants.SINGLE_QUOTES);
+                        }
                     }
 
-                    if(targetMysqlParameter.isUpdate()
-                            && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey())
-                            && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())){
-                        result.append(" --update-key ").append(targetMysqlParameter.getTargetUpdateKey())
-                              .append(" --update-mode ").append(targetMysqlParameter.getTargetUpdateMode());
+                    if (targetMysqlParameter.getIsUpdate()
+                        && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey())
+                        && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())) {
+                        mysqlTargetSb.append(Constants.SPACE).append(SqoopConstants.UPDATE_KEY)
+                            .append(Constants.SPACE).append(targetMysqlParameter.getTargetUpdateKey())
+                            .append(Constants.SPACE).append(SqoopConstants.UPDATE_MODE)
+                            .append(Constants.SPACE).append(targetMysqlParameter.getTargetUpdateMode());
                     }
                 }
             }
-        }catch (Exception e){
-            logger.error(e.getMessage());
+        } catch (Exception e) {
+            logger.error(String.format("Sqoop mysql target params build failed: [%s]", e.getMessage()));
         }
 
-        return result.toString();
+        return mysqlTargetSb.toString();
     }
 }

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java

@@ -17,11 +17,13 @@
 
 package org.apache.dolphinscheduler.server.master.consumer;
 
-import org.apache.dolphinscheduler.common.enums.CommandType;
-import org.apache.dolphinscheduler.common.enums.DbType;
-import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.common.enums.Priority;
+import org.apache.dolphinscheduler.common.enums.*;
+import org.apache.dolphinscheduler.common.model.TaskNode;
+import org.apache.dolphinscheduler.common.thread.Stopper;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.dao.entity.*;
+import org.apache.dolphinscheduler.server.entity.DataxTaskExecutionContext;
+import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher;
 import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
@@ -32,9 +34,10 @@ import org.apache.dolphinscheduler.server.zk.SpringZKServer;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue;
-import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl;
 import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
 import org.apache.dolphinscheduler.service.zk.ZookeeperConfig;
+import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -43,7 +46,11 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 
+import java.util.ArrayList;
 import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 
 @RunWith(SpringJUnit4ClassRunner.class)
@@ -66,23 +73,21 @@ public class TaskPriorityQueueConsumerTest {
     private ExecutorDispatcher dispatcher;
 
     @Before
-    public void init(){
+    public void init() {
 
         Tenant tenant = new Tenant();
         tenant.setId(1);
         tenant.setTenantCode("journey");
-        tenant.setTenantName("journey");
         tenant.setDescription("journey");
         tenant.setQueueId(1);
         tenant.setCreateTime(new Date());
         tenant.setUpdateTime(new Date());
 
-        Mockito.when(processService.getTenantForProcess(1,2)).thenReturn(tenant);
+        Mockito.doReturn(tenant).when(processService).getTenantForProcess(1, 2);
 
-        Mockito.when(processService.queryUserQueueByProcessInstanceId(1)).thenReturn("default");
+        Mockito.doReturn("default").when(processService).queryUserQueueByProcessInstanceId(1);
     }
 
-
     @Test
     public void testSHELLTask() throws Exception {
         TaskInstance taskInstance = new TaskInstance();
@@ -91,12 +96,31 @@ public class TaskPriorityQueueConsumerTest {
         taskInstance.setProcessDefinitionId(1);
         taskInstance.setProcessInstanceId(1);
         taskInstance.setState(ExecutionStatus.KILL);
-        taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-55201\",\"maxRetryTimes\":0,\"name\":\"测试任务\",\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SHELL\",\"workerGroup\":\"default\"}");
+        taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\","
+                + "\"conditionsTask\":false,"
+                + "\"depList\":[],"
+                + "\"dependence\":\"{}\","
+                + "\"forbidden\":false,"
+                + "\"id\":\"tasks-55201\","
+                + "\"maxRetryTimes\":0,"
+                + "\"name\":\"测试任务\","
+                + "\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\","
+                + "\"preTasks\":\"[]\","
+                + "\"retryInterval\":1,"
+                + "\"runFlag\":\"NORMAL\","
+                + "\"taskInstancePriority\":\"MEDIUM\","
+                + "\"taskTimeoutParameter\":{\"enable\":false,"
+                + "\"interval\":0},"
+                + "\"timeout\":\"{\\\"enable\\\":false,"
+                + "\\\"strategy\\\":\\\"\\\"}\","
+                + "\"type\":\"SHELL\","
+                + "\"workerGroup\":\"default\"}");
         taskInstance.setProcessInstancePriority(Priority.MEDIUM);
         taskInstance.setWorkerGroup("default");
         taskInstance.setExecutorId(2);
 
         ProcessInstance processInstance = new ProcessInstance();
+        processInstance.setId(1);
         processInstance.setTenantId(1);
         processInstance.setCommandType(CommandType.START_PROCESS);
         taskInstance.setProcessInstance(processInstance);
@@ -106,12 +130,15 @@ public class TaskPriorityQueueConsumerTest {
         processDefinition.setProjectId(1);
         taskInstance.setProcessDefine(processDefinition);
 
-        Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance);
+        Mockito.doReturn(taskInstance).when(processService).getTaskInstanceDetailByTaskId(1);
+        Mockito.doReturn(taskInstance).when(processService).findTaskInstanceById(1);
+
         taskPriorityQueue.put("2_1_2_1_default");
 
-        Thread.sleep(10000);
-    }
+        TimeUnit.SECONDS.sleep(10);
 
+        Assert.assertNotNull(taskInstance);
+    }
 
     @Test
     public void testSQLTask() throws Exception {
@@ -121,7 +148,13 @@ public class TaskPriorityQueueConsumerTest {
         taskInstance.setProcessDefinitionId(1);
         taskInstance.setProcessInstanceId(1);
         taskInstance.setState(ExecutionStatus.KILL);
-        taskInstance.setTaskJson("{\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-3655\",\"maxRetryTimes\":0,\"name\":\"UDF测试\",\"params\":\"{\\\"postStatements\\\":[],\\\"connParams\\\":\\\"\\\",\\\"receiversCc\\\":\\\"\\\",\\\"udfs\\\":\\\"1\\\",\\\"type\\\":\\\"HIVE\\\",\\\"title\\\":\\\"test\\\",\\\"sql\\\":\\\"select id,name,ds,zodia(ds) from t_journey_user\\\",\\\"preStatements\\\":[],\\\"sqlType\\\":0,\\\"receivers\\\":\\\"825193156@qq.com\\\",\\\"datasource\\\":3,\\\"showType\\\":\\\"TABLE\\\",\\\"localParams\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SQL\"}");
+        taskInstance.setTaskJson("{\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-3655\",\"maxRetryTimes\":0,\"name\":\"UDF测试\","
+                + "\"params\":\"{\\\"postStatements\\\":[],\\\"connParams\\\":\\\"\\\",\\\"receiversCc\\\":\\\"\\\",\\\"udfs\\\":\\\"1\\\",\\\"type\\\":\\\"HIVE\\\",\\\"title\\\":\\\"test\\\","
+                + "\\\"sql\\\":\\\"select id,name,ds,zodia(ds) from t_journey_user\\\",\\\"preStatements\\\":[],"
+                + "\\\"sqlType\\\":0,\\\"receivers\\\":\\\"825193156@qq.com\\\",\\\"datasource\\\":3,\\\"showType\\\":\\\"TABLE\\\",\\\"localParams\\\":[]}\","
+                + "\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\","
+                + "\"taskInstancePriority\":\"MEDIUM\","
+                + "\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SQL\"}");
         taskInstance.setProcessInstancePriority(Priority.MEDIUM);
         taskInstance.setWorkerGroup("default");
         taskInstance.setExecutorId(2);
@@ -135,8 +168,7 @@ public class TaskPriorityQueueConsumerTest {
         processDefinition.setUserId(2);
         processDefinition.setProjectId(1);
         taskInstance.setProcessDefine(processDefinition);
-
-        Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance);
+        Mockito.doReturn(taskInstance).when(processService).getTaskInstanceDetailByTaskId(1);
         taskPriorityQueue.put("2_1_2_1_default");
 
         DataSource dataSource = new DataSource();
@@ -144,16 +176,20 @@ public class TaskPriorityQueueConsumerTest {
         dataSource.setName("sqlDatasource");
         dataSource.setType(DbType.MYSQL);
         dataSource.setUserId(2);
-        dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\",\"database\":\"dolphinscheduler_qiaozhanwei\",\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\",\"user\":\"root\",\"password\":\"root@123\"}");
+        dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\","
+                + "\"database\":\"dolphinscheduler_qiaozhanwei\","
+                + "\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\","
+                + "\"user\":\"root\","
+                + "\"password\":\"root@123\"}");
         dataSource.setCreateTime(new Date());
         dataSource.setUpdateTime(new Date());
 
-        Mockito.when(processService.findDataSourceById(1)).thenReturn(dataSource);
+        Mockito.doReturn(dataSource).when(processService).findDataSourceById(1);
 
-        Thread.sleep(10000);
+        TimeUnit.SECONDS.sleep(10);
+        Assert.assertNotNull(taskInstance);
     }
 
-
     @Test
     public void testDataxTask() throws Exception {
         TaskInstance taskInstance = new TaskInstance();
@@ -162,7 +198,26 @@ public class TaskPriorityQueueConsumerTest {
         taskInstance.setProcessDefinitionId(1);
         taskInstance.setProcessInstanceId(1);
         taskInstance.setState(ExecutionStatus.KILL);
-        taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-97625\",\"maxRetryTimes\":0,\"name\":\"MySQL数据相互导入\",\"params\":\"{\\\"targetTable\\\":\\\"pv2\\\",\\\"postStatements\\\":[],\\\"jobSpeedRecord\\\":1000,\\\"customConfig\\\":0,\\\"dtType\\\":\\\"MYSQL\\\",\\\"dsType\\\":\\\"MYSQL\\\",\\\"jobSpeedByte\\\":0,\\\"dataSource\\\":80,\\\"dataTarget\\\":80,\\\"sql\\\":\\\"SELECT dt,count FROM pv\\\",\\\"preStatements\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"DATAX\",\"workerGroup\":\"default\"}");
+        taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\","
+                + "\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\","
+                + "\"forbidden\":false,\"id\":\"tasks-97625\","
+                + "\"maxRetryTimes\":0,\"name\":\"MySQL数据相互导入\","
+                + "\"params\":\"{\\\"targetTable\\\":\\\"pv2\\\","
+                + "    \\\"postStatements\\\":[],"
+                + "    \\\"jobSpeedRecord\\\":1000,"
+                + "    \\\"customConfig\\\":0,"
+                + "    \\\"dtType\\\":\\\"MYSQL\\\","
+                + "    \\\"dsType\\\":\\\"MYSQL\\\","
+                + "    \\\"jobSpeedByte\\\":0,"
+                + "    \\\"dataSource\\\":80,"
+                + "    \\\"dataTarget\\\":80,"
+                + "    \\\"sql\\\":\\\"SELECT dt,count FROM pv\\\","
+                + "    \\\"preStatements\\\":[]}\","
+                + "\"preTasks\":\"[]\","
+                + "\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\","
+                + "\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\","
+                + "\"type\":\"DATAX\","
+                + "\"workerGroup\":\"default\"}");
         taskInstance.setProcessInstancePriority(Priority.MEDIUM);
         taskInstance.setWorkerGroup("default");
         taskInstance.setExecutorId(2);
@@ -176,27 +231,26 @@ public class TaskPriorityQueueConsumerTest {
         processDefinition.setUserId(2);
         processDefinition.setProjectId(1);
         taskInstance.setProcessDefine(processDefinition);
-
-        Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance);
+        Mockito.doReturn(taskInstance).when(processService).getTaskInstanceDetailByTaskId(1);
         taskPriorityQueue.put("2_1_2_1_default");
 
-
-
         DataSource dataSource = new DataSource();
         dataSource.setId(80);
         dataSource.setName("datax");
         dataSource.setType(DbType.MYSQL);
         dataSource.setUserId(2);
-        dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\",\"database\":\"dolphinscheduler_qiaozhanwei\",\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\",\"user\":\"root\",\"password\":\"root@123\"}");
+        dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\","
+                + "\"database\":\"dolphinscheduler_qiaozhanwei\","
+                + "\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\","
+                + "\"user\":\"root\","
+                + "\"password\":\"root@123\"}");
         dataSource.setCreateTime(new Date());
         dataSource.setUpdateTime(new Date());
-
-        Mockito.when(processService.findDataSourceById(80)).thenReturn(dataSource);
-
-        Thread.sleep(10000);
+        Mockito.doReturn(dataSource).when(processService).findDataSourceById(80);
+        TimeUnit.SECONDS.sleep(10);
+        Assert.assertNotNull(taskInstance);
     }
 
-
     @Test
     public void testSqoopTask() throws Exception {
         TaskInstance taskInstance = new TaskInstance();
@@ -205,7 +259,32 @@ public class TaskPriorityQueueConsumerTest {
         taskInstance.setProcessDefinitionId(1);
         taskInstance.setProcessInstanceId(1);
         taskInstance.setState(ExecutionStatus.KILL);
-        taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-63634\",\"maxRetryTimes\":0,\"name\":\"MySQL数据导入HDSF\",\"params\":\"{\\\"sourceType\\\":\\\"MYSQL\\\",\\\"targetType\\\":\\\"HDFS\\\",\\\"targetParams\\\":\\\"{\\\\\\\"targetPath\\\\\\\":\\\\\\\"/test/datatest\\\\\\\",\\\\\\\"deleteTargetDir\\\\\\\":true,\\\\\\\"fileType\\\\\\\":\\\\\\\"--as-textfile\\\\\\\",\\\\\\\"compressionCodec\\\\\\\":\\\\\\\"\\\\\\\",\\\\\\\"fieldsTerminated\\\\\\\":\\\\\\\",\\\\\\\",\\\\\\\"linesTerminated\\\\\\\":\\\\\\\"\\\\\\\\\\\\\\\\n\\\\\\\"}\\\",\\\"modelType\\\":\\\"import\\\",\\\"sourceParams\\\":\\\"{\\\\\\\"srcType\\\\\\\":\\\\\\\"MYSQL\\\\\\\",\\\\\\\"srcDatasource\\\\\\\":1,\\\\\\\"srcTable\\\\\\\":\\\\\\\"t_ds_user\\\\\\\",\\\\\\\"srcQueryType\\\\\\\":\\\\\\\"0\\\\\\\",\\\\\\\"srcQuerySql\\\\\\\":\\\\\\\"\\\\\\\",\\\\\\\"srcColumnType\\\\\\\":\\\\\\\"0\\\\\\\",\\\\\\\"srcColumns\\\\\\\":\\\\\\\"\\\\\\\",\\\\\\\"srcConditionList\\\\\\\":[],\\\\\\\"mapColumnHive\\\\\\\":[],\\\\\\\"mapColumnJava\\\\\\\":[]}\\\",\\\"localParams\\\":[],\\\"concurrency\\\":1}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SQOOP\",\"workerGroup\":\"default\"}");
+        taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\","
+                + "\"forbidden\":false,\"id\":\"tasks-63634\","
+                + "\"maxRetryTimes\":0,\"name\":\"MySQL数据导入HDSF\","
+                + "\"params\":\"{\\\"sourceType\\\":\\\"MYSQL\\\","
+                + "    \\\"targetType\\\":\\\"HDFS\\\","
+                + "    \\\"targetParams\\\":\\\"{\\\\\\\"targetPath\\\\\\\":\\\\\\\"/test/datatest\\\\\\\","
+                + "        \\\\\\\"deleteTargetDir\\\\\\\":true,\\\\\\\"fileType\\\\\\\":\\\\\\\"--as-textfile\\\\\\\","
+                + "        \\\\\\\"compressionCodec\\\\\\\":\\\\\\\"\\\\\\\","
+                + "        \\\\\\\"fieldsTerminated\\\\\\\":\\\\\\\",\\\\\\\","
+                + "        \\\\\\\"linesTerminated\\\\\\\":\\\\\\\"\\\\\\\\\\\\\\\\n\\\\\\\"}\\\","
+                + "    \\\"modelType\\\":\\\"import\\\","
+                + "    \\\"sourceParams\\\":\\\"{\\\\\\\"srcType\\\\\\\":\\\\\\\"MYSQL\\\\\\\","
+                + "        \\\\\\\"srcDatasource\\\\\\\":1,\\\\\\\"srcTable\\\\\\\":\\\\\\\"t_ds_user\\\\\\\","
+                + "        \\\\\\\"srcQueryType\\\\\\\":\\\\\\\"0\\\\\\\","
+                + "        \\\\\\\"srcQuerySql\\\\\\\":\\\\\\\"\\\\\\\","
+                + "        \\\\\\\"srcColumnType\\\\\\\":\\\\\\\"0\\\\\\\","
+                + "        \\\\\\\"srcColumns\\\\\\\":\\\\\\\"\\\\\\\","
+                + "        \\\\\\\"srcConditionList\\\\\\\":[],\\\\\\\"mapColumnHive\\\\\\\":[],\\\\\\\"mapColumnJava\\\\\\\":[]}\\\","
+                + "    \\\"localParams\\\":[],\\\"concurrency\\\":1}\","
+                + "\"preTasks\":\"[]\","
+                + "\"retryInterval\":1,"
+                + "\"runFlag\":\"NORMAL\","
+                + "\"taskInstancePriority\":\"MEDIUM\","
+                + "\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\","
+                + "\"type\":\"SQOOP\","
+                + "\"workerGroup\":\"default\"}");
         taskInstance.setProcessInstancePriority(Priority.MEDIUM);
         taskInstance.setWorkerGroup("default");
         taskInstance.setExecutorId(2);
@@ -219,44 +298,331 @@ public class TaskPriorityQueueConsumerTest {
         processDefinition.setUserId(2);
         processDefinition.setProjectId(1);
         taskInstance.setProcessDefine(processDefinition);
-
-        Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance);
+        Mockito.doReturn(taskInstance).when(processService).getTaskInstanceDetailByTaskId(1);
         taskPriorityQueue.put("2_1_2_1_default");
 
-
-
         DataSource dataSource = new DataSource();
         dataSource.setId(1);
         dataSource.setName("datax");
         dataSource.setType(DbType.MYSQL);
         dataSource.setUserId(2);
-        dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\",\"database\":\"dolphinscheduler_qiaozhanwei\",\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\",\"user\":\"root\",\"password\":\"root@123\"}");
+        dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\","
+                + "\"database\":\"dolphinscheduler_qiaozhanwei\","
+                + "\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\","
+                + "\"user\":\"root\","
+                + "\"password\":\"root@123\"}");
         dataSource.setCreateTime(new Date());
         dataSource.setUpdateTime(new Date());
-
-        Mockito.when(processService.findDataSourceById(1)).thenReturn(dataSource);
-        Thread.sleep(10000);
+        Mockito.doReturn(dataSource).when(processService).findDataSourceById(1);
+        TimeUnit.SECONDS.sleep(10);
+        Assert.assertNotNull(taskInstance);
     }
 
-
     @Test
-    public void testTaskInstanceIsFinalState(){
+    public void testTaskInstanceIsFinalState() {
         TaskInstance taskInstance = new TaskInstance();
         taskInstance.setId(1);
         taskInstance.setTaskType("SHELL");
         taskInstance.setProcessDefinitionId(1);
         taskInstance.setProcessInstanceId(1);
         taskInstance.setState(ExecutionStatus.KILL);
-        taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-55201\",\"maxRetryTimes\":0,\"name\":\"测试任务\",\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SHELL\",\"workerGroup\":\"default\"}");
+        taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\","
+                + "\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\","
+                + "\"forbidden\":false,\"id\":\"tasks-55201\","
+                + "\"maxRetryTimes\":0,\"name\":\"测试任务\","
+                + "\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\","
+                + "\"retryInterval\":1,\"runFlag\":\"NORMAL\","
+                + "\"taskInstancePriority\":\"MEDIUM\","
+                + "\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\","
+                + "\"type\":\"SHELL\","
+                + "\"workerGroup\":\"default\"}");
         taskInstance.setProcessInstancePriority(Priority.MEDIUM);
         taskInstance.setWorkerGroup("default");
         taskInstance.setExecutorId(2);
 
+        Mockito.doReturn(taskInstance).when(processService).findTaskInstanceById(1);
 
-        Mockito.when( processService.findTaskInstanceById(1)).thenReturn(taskInstance);
-
-        taskPriorityQueueConsumer.taskInstanceIsFinalState(1);
+        Boolean state = taskPriorityQueueConsumer.taskInstanceIsFinalState(1);
+        Assert.assertNotNull(state);
     }
 
+    @Test
+    public void testNotFoundWorkerGroup() throws Exception {
+        TaskInstance taskInstance = new TaskInstance();
+        taskInstance.setId(1);
+        taskInstance.setTaskType("SHELL");
+        taskInstance.setProcessDefinitionId(1);
+        taskInstance.setProcessInstanceId(1);
+        taskInstance.setState(ExecutionStatus.KILL);
+        taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\","
+                + "\"conditionsTask\":false,"
+                + "\"depList\":[],"
+                + "\"dependence\":\"{}\","
+                + "\"forbidden\":false,"
+                + "\"id\":\"tasks-55201\","
+                + "\"maxRetryTimes\":0,"
+                + "\"name\":\"测试任务\","
+                + "\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\","
+                + "\"preTasks\":\"[]\","
+                + "\"retryInterval\":1,"
+                + "\"runFlag\":\"NORMAL\","
+                + "\"taskInstancePriority\":\"MEDIUM\","
+                + "\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},"
+                + "\"timeout\":\"{\\\"enable\\\":false,"
+                + "\\\"strategy\\\":\\\"\\\"}\","
+                + "\"type\":\"SHELL\","
+                + "\"workerGroup\":\"NoWorkGroup\"}");
+        taskInstance.setProcessInstancePriority(Priority.MEDIUM);
+        taskInstance.setWorkerGroup("NoWorkGroup");
+        taskInstance.setExecutorId(2);
+
+        ProcessInstance processInstance = new ProcessInstance();
+        processInstance.setId(1);
+        processInstance.setTenantId(1);
+        processInstance.setCommandType(CommandType.START_PROCESS);
+        taskInstance.setProcessInstance(processInstance);
+        taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS);
+
+        ProcessDefinition processDefinition = new ProcessDefinition();
+        processDefinition.setUserId(2);
+        processDefinition.setProjectId(1);
+        taskInstance.setProcessDefine(processDefinition);
+
+        Mockito.doReturn(taskInstance).when(processService).getTaskInstanceDetailByTaskId(1);
+        Mockito.doReturn(taskInstance).when(processService).findTaskInstanceById(1);
+
+        taskPriorityQueue.put("2_1_2_1_NoWorkGroup");
+
+        TimeUnit.SECONDS.sleep(10);
+
+        Assert.assertNotNull(taskInstance);
+
+    }
+
+    @Test
+    public void testDispatch() throws Exception {
+        TaskInstance taskInstance = new TaskInstance();
+        taskInstance.setId(1);
+        taskInstance.setTaskType("SHELL");
+        taskInstance.setProcessDefinitionId(1);
+        taskInstance.setProcessInstanceId(1);
+        taskInstance.setState(ExecutionStatus.KILL);
+        taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\","
+                + "\"conditionsTask\":false,"
+                + "\"depList\":[],"
+                + "\"dependence\":\"{}\","
+                + "\"forbidden\":false,"
+                + "\"id\":\"tasks-55201\","
+                + "\"maxRetryTimes\":0,"
+                + "\"name\":\"测试任务\","
+                + "\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\","
+                + "\"preTasks\":\"[]\","
+                + "\"retryInterval\":1,"
+                + "\"runFlag\":\"NORMAL\","
+                + "\"taskInstancePriority\":\"MEDIUM\","
+                + "\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},"
+                + "\"timeout\":\"{\\\"enable\\\":false,"
+                + "\\\"strategy\\\":\\\"\\\"}\","
+                + "\"type\":\"SHELL\","
+                + "\"workerGroup\":\"NoWorkGroup\"}");
+        taskInstance.setProcessInstancePriority(Priority.MEDIUM);
+        taskInstance.setWorkerGroup("NoWorkGroup");
+        taskInstance.setExecutorId(2);
+
+        ProcessInstance processInstance = new ProcessInstance();
+        processInstance.setId(1);
+        processInstance.setTenantId(1);
+        processInstance.setCommandType(CommandType.START_PROCESS);
+        taskInstance.setProcessInstance(processInstance);
+        taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS);
+
+        ProcessDefinition processDefinition = new ProcessDefinition();
+        processDefinition.setUserId(2);
+        processDefinition.setProjectId(1);
+        taskInstance.setProcessDefine(processDefinition);
+
+        Mockito.doReturn(taskInstance).when(processService).getTaskInstanceDetailByTaskId(1);
+        Mockito.doReturn(taskInstance).when(processService).findTaskInstanceById(1);
+
+        boolean res  = taskPriorityQueueConsumer.dispatch(1);
+
+        Assert.assertFalse(res);
+    }
+
+    @Test
+    public void testGetTaskExecutionContext() throws Exception {
+
+        TaskInstance taskInstance = new TaskInstance();
+        taskInstance.setId(1);
+        taskInstance.setTaskType("SHELL");
+        taskInstance.setProcessDefinitionId(1);
+        taskInstance.setProcessInstanceId(1);
+        taskInstance.setState(ExecutionStatus.KILL);
+        taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\","
+                + "\"conditionsTask\":false,"
+                + "\"depList\":[],"
+                + "\"dependence\":\"{}\","
+                + "\"forbidden\":false,"
+                + "\"id\":\"tasks-55201\","
+                + "\"maxRetryTimes\":0,"
+                + "\"name\":\"测试任务\","
+                + "\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\","
+                + "\"preTasks\":\"[]\","
+                + "\"retryInterval\":1,"
+                + "\"runFlag\":\"NORMAL\","
+                + "\"taskInstancePriority\":\"MEDIUM\","
+                + "\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},"
+                + "\"timeout\":\"{\\\"enable\\\":false,"
+                + "\\\"strategy\\\":\\\"\\\"}\","
+                + "\"type\":\"SHELL\","
+                + "\"workerGroup\":\"NoWorkGroup\"}");
+        taskInstance.setProcessInstancePriority(Priority.MEDIUM);
+        taskInstance.setWorkerGroup("NoWorkGroup");
+        taskInstance.setExecutorId(2);
+
+        ProcessInstance processInstance = new ProcessInstance();
+        processInstance.setId(1);
+        processInstance.setTenantId(1);
+        processInstance.setCommandType(CommandType.START_PROCESS);
+        taskInstance.setProcessInstance(processInstance);
+        taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS);
+
+        ProcessDefinition processDefinition = new ProcessDefinition();
+        processDefinition.setUserId(2);
+        processDefinition.setProjectId(1);
+        taskInstance.setProcessDefine(processDefinition);
+
+        Mockito.doReturn(taskInstance).when(processService).getTaskInstanceDetailByTaskId(1);
+        Mockito.doReturn(taskInstance).when(processService).findTaskInstanceById(1);
+
+        TaskExecutionContext taskExecutionContext  = taskPriorityQueueConsumer.getTaskExecutionContext(1);
+
+
+
+        Assert.assertNotNull(taskExecutionContext);
+    }
+
+    @Test
+    public void testGetResourceFullNames() throws Exception {
+        TaskInstance taskInstance = new TaskInstance();
+        taskInstance.setId(1);
+        taskInstance.setTaskType("SHELL");
+        taskInstance.setProcessDefinitionId(1);
+        taskInstance.setProcessInstanceId(1);
+        taskInstance.setState(ExecutionStatus.KILL);
+        taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\","
+                + "\"conditionsTask\":false,"
+                + "\"depList\":[],"
+                + "\"dependence\":\"{}\","
+                + "\"forbidden\":false,"
+                + "\"id\":\"tasks-55201\","
+                + "\"maxRetryTimes\":0,"
+                + "\"name\":\"测试任务\","
+                + "\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[{\\\"id\\\":123},{\\\"res\\\":\\\"/data/file\\\"}]}\","
+                + "\"preTasks\":\"[]\","
+                + "\"retryInterval\":1,"
+                + "\"runFlag\":\"NORMAL\","
+                + "\"taskInstancePriority\":\"MEDIUM\","
+                + "\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},"
+                + "\"timeout\":\"{\\\"enable\\\":false,"
+                + "\\\"strategy\\\":\\\"\\\"}\","
+                + "\"type\":\"SHELL\","
+                + "\"workerGroup\":\"NoWorkGroup\"}");
+
+        taskInstance.setProcessInstancePriority(Priority.MEDIUM);
+        taskInstance.setWorkerGroup("NoWorkGroup");
+        taskInstance.setExecutorId(2);
+        // task node
+        TaskNode taskNode = JSONUtils.parseObject(taskInstance.getTaskJson(), TaskNode.class);
+
+        Map<String, String> map = taskPriorityQueueConsumer.getResourceFullNames(taskNode);
+
+        List<Resource> resourcesList = new ArrayList<Resource>();
+        Resource resource = new Resource();
+        resource.setFileName("fileName");
+        resourcesList.add(resource);
+
+        Mockito.doReturn(resourcesList).when(processService).listResourceByIds(new Integer[]{123});
+        Mockito.doReturn("tenantCode").when(processService).queryTenantCodeByResName(resource.getFullName(), ResourceType.FILE);
+        Assert.assertNotNull(map);
+
+    }
+
+    @Test
+    public void testSetDataxTaskRelation() throws Exception {
+
+        DataxTaskExecutionContext dataxTaskExecutionContext = new DataxTaskExecutionContext();
+        TaskNode taskNode = new TaskNode();
+        taskNode.setParams("{\"dataSource\":1,\"dataTarget\":1}");
+        DataSource dataSource = new DataSource();
+        dataSource.setId(1);
+        dataSource.setConnectionParams("");
+        dataSource.setType(DbType.MYSQL);
+        Mockito.doReturn(dataSource).when(processService).findDataSourceById(1);
+
+        taskPriorityQueueConsumer.setDataxTaskRelation(dataxTaskExecutionContext,taskNode);
+
+        Assert.assertEquals(1,dataxTaskExecutionContext.getDataSourceId());
+        Assert.assertEquals(1,dataxTaskExecutionContext.getDataTargetId());
+    }
+
+    @Test
+    public void testRun() throws Exception {
+        TaskInstance taskInstance = new TaskInstance();
+        taskInstance.setId(1);
+        taskInstance.setTaskType("SHELL");
+        taskInstance.setProcessDefinitionId(1);
+        taskInstance.setProcessInstanceId(1);
+        taskInstance.setState(ExecutionStatus.KILL);
+        taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\","
+                + "\"conditionsTask\":false,"
+                + "\"depList\":[],"
+                + "\"dependence\":\"{}\","
+                + "\"forbidden\":false,"
+                + "\"id\":\"tasks-55201\","
+                + "\"maxRetryTimes\":0,"
+                + "\"name\":\"测试任务\","
+                + "\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\","
+                + "\"preTasks\":\"[]\","
+                + "\"retryInterval\":1,"
+                + "\"runFlag\":\"NORMAL\","
+                + "\"taskInstancePriority\":\"MEDIUM\","
+                + "\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},"
+                + "\"timeout\":\"{\\\"enable\\\":false,"
+                + "\\\"strategy\\\":\\\"\\\"}\","
+                + "\"type\":\"SHELL\","
+                + "\"workerGroup\":\"NoWorkGroup\"}");
+        taskInstance.setProcessInstancePriority(Priority.MEDIUM);
+        taskInstance.setWorkerGroup("NoWorkGroup");
+        taskInstance.setExecutorId(2);
+
+        ProcessInstance processInstance = new ProcessInstance();
+        processInstance.setId(1);
+        processInstance.setTenantId(1);
+        processInstance.setCommandType(CommandType.START_PROCESS);
+        taskInstance.setProcessInstance(processInstance);
+        taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS);
+
+        ProcessDefinition processDefinition = new ProcessDefinition();
+        processDefinition.setUserId(2);
+        processDefinition.setProjectId(1);
+        taskInstance.setProcessDefine(processDefinition);
+
+        Mockito.doReturn(taskInstance).when(processService).getTaskInstanceDetailByTaskId(1);
+        Mockito.doReturn(taskInstance).when(processService).findTaskInstanceById(1);
+
+        taskPriorityQueue.put("2_1_2_1_NoWorkGroup");
+
+        taskPriorityQueueConsumer.run();
+
+        TimeUnit.SECONDS.sleep(10);
+        Assert.assertNotEquals(-1,taskPriorityQueue.size());
+
+    }
+
+    @After
+    public void close() {
+        Stopper.stop();
+    }
 
 }

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java

@@ -16,35 +16,124 @@
  */
 package org.apache.dolphinscheduler.server.utils;
 
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.common.utils.HadoopUtils;
+import org.apache.dolphinscheduler.common.utils.OSUtils;
+import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.MockitoAnnotations;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({System.class, OSUtils.class, HadoopUtils.class})
 public class ProcessUtilsTest {
 
-    private static final Logger logger = LoggerFactory.getLogger(ProcessUtilsTest.class);
+    private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class);
+
+    @Before
+    public void setUp() {
+        MockitoAnnotations.initMocks(this);
+    }
 
     @Test
     public void getPidsStr() throws Exception {
-        String pidList = ProcessUtils.getPidsStr(1);
+        int processId = 1;
+        String pidList = ProcessUtils.getPidsStr(processId);
         Assert.assertNotEquals("The child process of process 1 should not be empty", pidList, "");
-        logger.info("Sub process list : {}", pidList);
+
+        PowerMockito.mockStatic(OSUtils.class);
+        when(OSUtils.isMacOS()).thenReturn(true);
+        when(OSUtils.exeCmd(String.format("%s -p %d", Constants.PSTREE, processId))).thenReturn(null);
+        String pidListMac = ProcessUtils.getPidsStr(processId);
+        Assert.assertEquals("", pidListMac);
     }
 
     @Test
-    public void testBuildCommandStr() {
+    public void testBuildCommandStr() throws IOException {
         List<String> commands = new ArrayList<>();
         commands.add("sudo");
-        try {
-            Assert.assertEquals(ProcessUtils.buildCommandStr(commands), "sudo");
-        } catch (IOException e) {
-            Assert.fail(e.getMessage());
-        }
+        commands.add("-u");
+        commands.add("tenantCode");
+        //allowAmbiguousCommands false
+        Assert.assertEquals("sudo -u tenantCode", ProcessUtils.buildCommandStr(commands));
+
+        //quota
+        commands.clear();
+        commands.add("\"sudo\"");
+        Assert.assertEquals("\"sudo\"", ProcessUtils.buildCommandStr(commands));
+
+        //allowAmbiguousCommands true
+        commands.clear();
+        commands.add("sudo");
+        System.setProperty("jdk.lang.Process.allowAmbiguousCommands", "false");
+        Assert.assertEquals("\"sudo\"", ProcessUtils.buildCommandStr(commands));
     }
 
+    @Test
+    public void testKill() {
+        //get taskExecutionContext
+        TaskExecutionContext taskExecutionContext = new TaskExecutionContext();
+
+        //process id eq 0
+        taskExecutionContext.setProcessId(0);
+        ProcessUtils.kill(taskExecutionContext);
+
+        //process id not eq 0
+        taskExecutionContext.setProcessId(1);
+        PowerMockito.mockStatic(OSUtils.class);
+        try {
+            when(OSUtils.exeCmd(String.format("%s -sp %d", Constants.PSTREE, 1))).thenReturn("1111");
+            when(OSUtils.exeCmd(String.format("%s -p %d", Constants.PSTREE, 1))).thenReturn("1111");
+            when(OSUtils.exeCmd("sudo kill -9")).thenReturn("1111");
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        taskExecutionContext.setHost("127.0.0.1:8888");
+        taskExecutionContext.setLogPath("/log/1.log");
+        ProcessUtils.kill(taskExecutionContext);
+        Assert.assertEquals(1, taskExecutionContext.getProcessId());
+    }
+
+    @Test
+    public void testCancelApplication() {
+        List<String> appIds = new ArrayList<>();
+        appIds.add("application_1585532379175_228491");
+        appIds.add("application_1598885606600_3677");
+        String tenantCode = "dev";
+        String executePath = "/ds-exec/1/1/1";
+        ExecutionStatus running = ExecutionStatus.RUNNING_EXEUTION;
+
+        PowerMockito.mockStatic(HadoopUtils.class);
+        HadoopUtils hadoop = HadoopUtils.getInstance();
+
+        try {
+            PowerMockito.whenNew(HadoopUtils.class).withAnyArguments().thenReturn(hadoop);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        try {
+            when(hadoop.getApplicationStatus("application_1585532379175_228491")).thenReturn(running);
+            when(hadoop.getApplicationStatus("application_1598885606600_3677")).thenReturn(running);
+        } catch (Exception e) {
+            e.printStackTrace();
+            ProcessUtils.cancelApplication(appIds, logger, tenantCode, executePath);
+        }
+
+        Assert.assertNotNull(appIds);
+    }
 }