seg_yinzy
/
DolphinScheduler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[Improvement][Datax] Re-assign DATAX_HOME to /opt/soft/datax instead of /opt/soft/datax/bin/datax.py (#5019)

xingchun-chen-patch-1
Shiwen Cheng 2021-03-17 14:38:04 +08:00 committed by GitHub
parent 0ba120e8ab
commit 1cd62b4a5e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 21 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml
View File

@ -112,7 +112,8 @@ export SPARK_HOME2=/opt/soft/spark2
export PYTHON_HOME=/opt/soft/python
export JAVA_HOME=/opt/soft/java
export HIVE_HOME=/opt/soft/hive
export FLINK_HOME=/opt/soft/flink</value>
export FLINK_HOME=/opt/soft/flink
export DATAX_HOME=/opt/soft/datax</value>
<value-attributes>
<type>content</type>
<empty-value-valid>false</empty-value-valid>

3
ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml
View File

@ -112,7 +112,8 @@ export SPARK_HOME2=/opt/soft/spark2
export PYTHON_HOME=/opt/soft/python
export JAVA_HOME=/opt/soft/java
export HIVE_HOME=/opt/soft/hive
export FLINK_HOME=/opt/soft/flink</value>
export FLINK_HOME=/opt/soft/flink
export DATAX_HOME=/opt/soft/datax</value>
<value-attributes>
<type>content</type>
<empty-value-valid>false</empty-value-valid>

2
docker/build/README.md
View File

@ -209,7 +209,7 @@ This environment variable sets `FLINK_HOME`. The default value is `/opt/soft/fli
**`DATAX_HOME`**
This environment variable sets `DATAX_HOME`. The default value is `/opt/soft/datax/bin/datax.py`.
This environment variable sets `DATAX_HOME`. The default value is `/opt/soft/datax`.
**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**

2
docker/build/README_zh_CN.md
View File

@ -209,7 +209,7 @@ DolphinScheduler Docker 容器通过环境变量进行配置，缺省时将会
**`DATAX_HOME`**
配置`dolphinscheduler`的`DATAX_HOME`，默认值 `/opt/soft/datax/bin/datax。py`。
配置`dolphinscheduler`的`DATAX_HOME`，默认值 `/opt/soft/datax`。
**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**

2
docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl vendored
View File

@ -25,4 +25,4 @@ export HIVE_HOME=$HIVE_HOME
export FLINK_HOME=$FLINK_HOME
export DATAX_HOME=$DATAX_HOME
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH

2
docker/build/startup-init-conf.sh
View File

@ -46,7 +46,7 @@ export PYTHON_HOME=${PYTHON_HOME:-"/usr"}
export JAVA_HOME=${JAVA_HOME:-"/usr/lib/jvm/java-1.8-openjdk"}
export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"}
export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"}
export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax/bin/datax.py"}
export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax"}
# common env
export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""}

2
docker/docker-swarm/docker-compose.yml
View File

@ -181,7 +181,7 @@ services:
JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk"
HIVE_HOME: "/opt/soft/hive"
FLINK_HOME: "/opt/soft/flink"
DATAX_HOME: "/opt/soft/datax/bin/datax.py"
DATAX_HOME: "/opt/soft/datax"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
DATABASE_TYPE: postgresql

2
docker/docker-swarm/docker-stack.yml
View File

@ -175,7 +175,7 @@ services:
JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk"
HIVE_HOME: "/opt/soft/hive"
FLINK_HOME: "/opt/soft/flink"
DATAX_HOME: "/opt/soft/datax/bin/datax.py"
DATAX_HOME: "/opt/soft/datax"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
DATABASE_TYPE: postgresql

4
docker/kubernetes/dolphinscheduler/values.yaml
View File

@ -80,8 +80,8 @@ common:
- "export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk"
- "export HIVE_HOME=/opt/soft/hive"
- "export FLINK_HOME=/opt/soft/flink"
- "export DATAX_HOME=/opt/soft/datax/bin/datax.py"
- "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH"
- "export DATAX_HOME=/opt/soft/datax"
- "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
RESOURCE_STORAGE_TYPE: "HDFS"
RESOURCE_UPLOAD_PATH: "/dolphinscheduler"

4
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java
View File

@ -27,7 +27,7 @@ public class DataxParametersTest {
/**
* jvm parameters
*/
public static final String JVM_EVN = " --jvm=\"-Xms%sG -Xmx%sG\" ";
public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" ";
@Test
public void testLoadJvmEnv() {
@ -88,6 +88,6 @@ public class DataxParametersTest {
public String loadJvmEnvTest(DataxParameters dataXParameters) {
int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms();
int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx();
return String.format(JVM_EVN, xms, xmx);
return String.format(JVM_PARAM, xms, xmx);
}
}

10
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
View File

@ -81,15 +81,15 @@ public class DataxTask extends AbstractTask {
/**
* jvm parameters
*/
public static final String JVM_EVN = " --jvm=\"-Xms%sG -Xmx%sG\" ";
public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" ";
/**
* python process(datax only supports version 2.7 by default)
*/
private static final String DATAX_PYTHON = "python2.7";
/**
* datax home path
* datax path
*/
private static final String DATAX_HOME_EVN = "${DATAX_HOME}";
private static final String DATAX_PATH = "${DATAX_HOME}/bin/datax.py";
/**
* datax channel count
*/
@ -396,7 +396,7 @@ public class DataxTask extends AbstractTask {
StringBuilder sbr = new StringBuilder();
sbr.append(DATAX_PYTHON);
sbr.append(" ");
sbr.append(DATAX_HOME_EVN);
sbr.append(DATAX_PATH);
sbr.append(" ");
sbr.append(loadJvmEnv(dataXParameters));
sbr.append(jobConfigFilePath);
@ -424,7 +424,7 @@ public class DataxTask extends AbstractTask {
public String loadJvmEnv(DataxParameters dataXParameters) {
int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms();
int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx();
return String.format(JVM_EVN, xms, xmx);
return String.format(JVM_PARAM, xms, xmx);
}
/**

4
script/env/dolphinscheduler_env.sh vendored
View File

@ -23,6 +23,6 @@ export PYTHON_HOME=/opt/soft/python
export JAVA_HOME=/opt/soft/java
export HIVE_HOME=/opt/soft/hive
export FLINK_HOME=/opt/soft/flink
export DATAX_HOME=/opt/soft/datax/bin/datax.py
export DATAX_HOME=/opt/soft/datax
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH