From 01936a660e14bfc6ea061bec065e47f4541b7519 Mon Sep 17 00:00:00 2001 From: Jiajie Zhong Date: Sat, 29 Jan 2022 16:48:18 +0800 Subject: [PATCH] [python] Fix python api can't connect to remote gateway server (#8248) For now, python API could only communicate python gateway server in the same hosts, this patch makes it could work with different hosts, and export java gateway setting to configure file Co-authored-by: kezhenxu94 Co-authored-by: ruanwenjun <861923274@qq.com> --- .../src/pydolphinscheduler/constants.py | 4 + .../src/pydolphinscheduler/java_gateway.py | 13 ++- .../server/PythonGatewayServer.java | 30 ++++++- .../server/config/PythonGatewayConfig.java | 83 +++++++++++++++++++ .../src/main/resources/application.yaml | 18 ++++ .../src/main/resources/application.yaml | 18 ++++ 6 files changed, 160 insertions(+), 6 deletions(-) create mode 100644 dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/config/PythonGatewayConfig.java diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py index 7bd71b985..65bf6c514 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py @@ -99,6 +99,10 @@ class JavaGatewayDefault(str): RESULT_DATA = "data" + SERVER_ADDRESS = "127.0.0.1" + SERVER_PORT = 25333 + AUTO_CONVERT = True + class Delimiter(str): """Constants for delimiter.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py index d0b4c054d..2876ed521 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py @@ -26,14 +26,23 @@ from pydolphinscheduler.constants import JavaGatewayDefault from pydolphinscheduler.exceptions import PyDSJavaGatewayException -def launch_gateway() -> JavaGateway: +def launch_gateway( + address: Optional[str] = None, + port: Optional[int] = None, + auto_convert: Optional[bool] = True, +) -> JavaGateway: """Launch java gateway to pydolphinscheduler. TODO Note that automatic conversion makes calling Java methods slightly less efficient because in the worst case, Py4J needs to go through all registered converters for all parameters. This is why automatic conversion is disabled by default. """ - gateway = JavaGateway(gateway_parameters=GatewayParameters(auto_convert=True)) + gateway_parameters = GatewayParameters( + address=address or JavaGatewayDefault.SERVER_ADDRESS, + port=port or JavaGatewayDefault.SERVER_PORT, + auto_convert=auto_convert or JavaGatewayDefault.AUTO_CONVERT, + ) + gateway = JavaGateway(gateway_parameters=gateway_parameters) return gateway diff --git a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java index 2b062866e..ae97a9d16 100644 --- a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java +++ b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java @@ -53,8 +53,11 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; +import org.apache.dolphinscheduler.server.config.PythonGatewayConfig; import org.apache.dolphinscheduler.spi.enums.ResourceType; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -132,6 +135,9 @@ public class PythonGatewayServer extends SpringBootServletInitializer { @Autowired private DataSourceMapper dataSourceMapper; + @Autowired + private PythonGatewayConfig pythonGatewayConfig; + // TODO replace this user to build in admin user if we make sure build in one could not be change private final User dummyAdminUser = new User() { { @@ -501,10 +507,26 @@ public class PythonGatewayServer extends SpringBootServletInitializer { @PostConstruct public void run() { - GatewayServer server = new GatewayServer(this); - GatewayServer.turnLoggingOn(); - // Start server to accept python client socket - server.start(); + GatewayServer server; + try { + InetAddress gatewayHost = InetAddress.getByName(pythonGatewayConfig.getGatewayServerAddress()); + InetAddress pythonHost = InetAddress.getByName(pythonGatewayConfig.getPythonAddress()); + server = new GatewayServer( + this, + pythonGatewayConfig.getGatewayServerPort(), + pythonGatewayConfig.getPythonPort(), + gatewayHost, + pythonHost, + pythonGatewayConfig.getConnectTimeout(), + pythonGatewayConfig.getReadTimeout(), + null + ); + GatewayServer.turnLoggingOn(); + logger.info("PythonGatewayServer started on: " + gatewayHost.toString()); + server.start(); + } catch (UnknownHostException e) { + logger.error("exception occurred while constructing PythonGatewayServer().", e); + } } public static void main(String[] args) { diff --git a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/config/PythonGatewayConfig.java b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/config/PythonGatewayConfig.java new file mode 100644 index 000000000..f06ea765b --- /dev/null +++ b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/config/PythonGatewayConfig.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; +import org.springframework.stereotype.Component; + +@Component +@EnableConfigurationProperties +@ConfigurationProperties("python-gateway") +public class PythonGatewayConfig { + private String gatewayServerAddress; + private int gatewayServerPort; + private String pythonAddress; + private int pythonPort; + private int connectTimeout; + private int readTimeout; + + public String getGatewayServerAddress() { + return gatewayServerAddress; + } + + public void setGatewayServerAddress(String gatewayServerAddress) { + this.gatewayServerAddress = gatewayServerAddress; + } + + public int getGatewayServerPort() { + return gatewayServerPort; + } + + public void setGatewayServerPort(int gatewayServerPort) { + this.gatewayServerPort = gatewayServerPort; + } + + public String getPythonAddress() { + return pythonAddress; + } + + public void setPythonAddress(String pythonAddress) { + this.pythonAddress = pythonAddress; + } + + public int getPythonPort() { + return pythonPort; + } + + public void setPythonPort(int pythonPort) { + this.pythonPort = pythonPort; + } + + public int getConnectTimeout() { + return connectTimeout; + } + + public void setConnectTimeout(int connectTimeout) { + this.connectTimeout = connectTimeout; + } + + public int getReadTimeout() { + return readTimeout; + } + + public void setReadTimeout(int readTimeout) { + this.readTimeout = readTimeout; + } +} diff --git a/dolphinscheduler-python/src/main/resources/application.yaml b/dolphinscheduler-python/src/main/resources/application.yaml index af315b1f7..b5dc00cd8 100644 --- a/dolphinscheduler-python/src/main/resources/application.yaml +++ b/dolphinscheduler-python/src/main/resources/application.yaml @@ -37,6 +37,24 @@ spring: hibernate: ddl-auto: none +python-gateway: + # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different + # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost` + gateway-server-address: 0.0.0.0 + # The port of Python gateway server start. Define which port you could connect to Python gateway server from + # Python API side. + gateway-server-port: 25333 + # The address of Python callback client. + python-address: 127.0.0.1 + # The port of Python callback client. + python-port: 25334 + # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite), + # and socket server would never close even though no requests accept + connect-timeout: 0 + # Close each active connection of socket server if python program not active after x milliseconds. Define value is + # (0 = infinite), and socket server would never close even though no requests accept + read-timeout: 0 + server: port: 54321 diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 5499e6ff6..53cfcbc9e 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -138,6 +138,24 @@ worker: alert: port: 50052 +python-gateway: + # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different + # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost` + gateway-server-address: 0.0.0.0 + # The port of Python gateway server start. Define which port you could connect to Python gateway server from + # Python API side. + gateway-server-port: 25333 + # The address of Python callback client. + python-address: 127.0.0.1 + # The port of Python callback client. + python-port: 25334 + # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite), + # and socket server would never close even though no requests accept + connect-timeout: 0 + # Close each active connection of socket server if python program not active after x milliseconds. Define value is + # (0 = infinite), and socket server would never close even though no requests accept + read-timeout: 0 + server: port: 12345 servlet: