[python] Fix python api can't connect to remote gateway server (#8248)

For now, python API could only communicate python gateway server
in the same hosts, this patch makes it could work with different hosts,
and export java gateway setting to configure file

Co-authored-by: kezhenxu94 <kezhenxu94@apache.org>
Co-authored-by: ruanwenjun <861923274@qq.com>

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py

@@ -99,6 +99,10 @@ class JavaGatewayDefault(str):
 
     RESULT_DATA = "data"
 
+    SERVER_ADDRESS = "127.0.0.1"
+    SERVER_PORT = 25333
+    AUTO_CONVERT = True
+
 
 class Delimiter(str):
     """Constants for delimiter."""

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py

@@ -26,14 +26,23 @@ from pydolphinscheduler.constants import JavaGatewayDefault
 from pydolphinscheduler.exceptions import PyDSJavaGatewayException
 
 
-def launch_gateway() -> JavaGateway:
+def launch_gateway(
+    address: Optional[str] = None,
+    port: Optional[int] = None,
+    auto_convert: Optional[bool] = True,
+) -> JavaGateway:
     """Launch java gateway to pydolphinscheduler.
 
     TODO Note that automatic conversion makes calling Java methods slightly less efficient because
     in the worst case, Py4J needs to go through all registered converters for all parameters.
     This is why automatic conversion is disabled by default.
     """
-    gateway = JavaGateway(gateway_parameters=GatewayParameters(auto_convert=True))
+    gateway_parameters = GatewayParameters(
+        address=address or JavaGatewayDefault.SERVER_ADDRESS,
+        port=port or JavaGatewayDefault.SERVER_PORT,
+        auto_convert=auto_convert or JavaGatewayDefault.AUTO_CONVERT,
+    )
+    gateway = JavaGateway(gateway_parameters=gateway_parameters)
     return gateway
 
 

dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java

@@ -53,8 +53,11 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
 import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
 import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
 import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
+import org.apache.dolphinscheduler.server.config.PythonGatewayConfig;
 import org.apache.dolphinscheduler.spi.enums.ResourceType;
 
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -132,6 +135,9 @@ public class PythonGatewayServer extends SpringBootServletInitializer {
     @Autowired
     private DataSourceMapper dataSourceMapper;
 
+    @Autowired
+    private PythonGatewayConfig pythonGatewayConfig;
+
     // TODO replace this user to build in admin user if we make sure build in one could not be change
     private final User dummyAdminUser = new User() {
         {
@@ -501,10 +507,26 @@ public class PythonGatewayServer extends SpringBootServletInitializer {
 
     @PostConstruct
     public void run() {
-        GatewayServer server = new GatewayServer(this);
+        GatewayServer server;
-        GatewayServer.turnLoggingOn();
+        try {
-        // Start server to accept python client socket
+            InetAddress gatewayHost = InetAddress.getByName(pythonGatewayConfig.getGatewayServerAddress());
-        server.start();
+            InetAddress pythonHost = InetAddress.getByName(pythonGatewayConfig.getPythonAddress());
+            server = new GatewayServer(
+                this,
+                pythonGatewayConfig.getGatewayServerPort(),
+                pythonGatewayConfig.getPythonPort(),
+                gatewayHost,
+                pythonHost,
+                pythonGatewayConfig.getConnectTimeout(),
+                pythonGatewayConfig.getReadTimeout(),
+                null
+            );
+            GatewayServer.turnLoggingOn();
+            logger.info("PythonGatewayServer started on: " + gatewayHost.toString());
+            server.start();
+        } catch (UnknownHostException e) {
+            logger.error("exception occurred while constructing PythonGatewayServer().", e);
+        }
     }
 
     public static void main(String[] args) {

dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/config/PythonGatewayConfig.java

@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.config;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.stereotype.Component;
+
+@Component
+@EnableConfigurationProperties
+@ConfigurationProperties("python-gateway")
+public class PythonGatewayConfig {
+    private String gatewayServerAddress;
+    private int gatewayServerPort;
+    private String pythonAddress;
+    private int pythonPort;
+    private int connectTimeout;
+    private int readTimeout;
+
+    public String getGatewayServerAddress() {
+        return gatewayServerAddress;
+    }
+
+    public void setGatewayServerAddress(String gatewayServerAddress) {
+        this.gatewayServerAddress = gatewayServerAddress;
+    }
+
+    public int getGatewayServerPort() {
+        return gatewayServerPort;
+    }
+
+    public void setGatewayServerPort(int gatewayServerPort) {
+        this.gatewayServerPort = gatewayServerPort;
+    }
+
+    public String getPythonAddress() {
+        return pythonAddress;
+    }
+
+    public void setPythonAddress(String pythonAddress) {
+        this.pythonAddress = pythonAddress;
+    }
+
+    public int getPythonPort() {
+        return pythonPort;
+    }
+
+    public void setPythonPort(int pythonPort) {
+        this.pythonPort = pythonPort;
+    }
+
+    public int getConnectTimeout() {
+        return connectTimeout;
+    }
+
+    public void setConnectTimeout(int connectTimeout) {
+        this.connectTimeout = connectTimeout;
+    }
+
+    public int getReadTimeout() {
+        return readTimeout;
+    }
+
+    public void setReadTimeout(int readTimeout) {
+        this.readTimeout = readTimeout;
+    }
+}

dolphinscheduler-python/src/main/resources/application.yaml

@@ -37,6 +37,24 @@ spring:
     hibernate:
       ddl-auto: none
 
+python-gateway:
+  # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
+  # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
+  gateway-server-address: 0.0.0.0
+  # The port of Python gateway server start. Define which port you could connect to Python gateway server from
+  # Python API side.
+  gateway-server-port: 25333
+  # The address of Python callback client.
+  python-address: 127.0.0.1
+  # The port of Python callback client.
+  python-port: 25334
+  # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite),
+  # and socket server would never close even though no requests accept
+  connect-timeout: 0
+  # Close each active connection of socket server if python program not active after x milliseconds. Define value is
+  # (0 = infinite), and socket server would never close even though no requests accept
+  read-timeout: 0
+
 server:
   port: 54321
 

dolphinscheduler-standalone-server/src/main/resources/application.yaml

@@ -138,6 +138,24 @@ worker:
 alert:
   port: 50052
 
+python-gateway:
+  # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
+  # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
+  gateway-server-address: 0.0.0.0
+  # The port of Python gateway server start. Define which port you could connect to Python gateway server from
+  # Python API side.
+  gateway-server-port: 25333
+  # The address of Python callback client.
+  python-address: 127.0.0.1
+  # The port of Python callback client.
+  python-port: 25334
+  # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite),
+  # and socket server would never close even though no requests accept
+  connect-timeout: 0
+  # Close each active connection of socket server if python program not active after x milliseconds. Define value is
+  # (0 = infinite), and socket server would never close even though no requests accept
+  read-timeout: 0
+
 server:
   port: 12345
   servlet: