diff --git a/.github/workflows/py-tests.yml b/.github/workflows/py-tests.yml new file mode 100644 index 000000000..e37cab3d8 --- /dev/null +++ b/.github/workflows/py-tests.yml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Python API Tests + +on: + push: + branches: + - dev + paths: + - 'dolphinscheduler-python/**' + pull_request: + paths: + - 'dolphinscheduler-python/**' + +defaults: + run: + working-directory: dolphinscheduler-python/pydolphinscheduler + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + os: [ubuntu-18.04, macOS-latest, windows-latest] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies & pydolphinscheduler + run: | + pip install -r requirements.txt -r requirements_dev.txt + pip install -e . + - name: Run tests + run: | + pytest diff --git a/dolphinscheduler-python/pydolphinscheduler/README.md b/dolphinscheduler-python/pydolphinscheduler/README.md index a98748336..a6609844e 100644 --- a/dolphinscheduler-python/pydolphinscheduler/README.md +++ b/dolphinscheduler-python/pydolphinscheduler/README.md @@ -19,6 +19,8 @@ # pydolphinscheduler +[![GitHub Build][ga-py-test]][ga] + pydolphinscheduler is python API for Apache DolphinScheduler, which allow you definition your workflow by python code, aka workflow-as-codes. @@ -47,7 +49,7 @@ section "DolphinScheduler Standalone Quick Start" to set up developer environmen and frontend server in this step, which mean that you could view DolphinScheduler UI in your browser with URL http://localhost:12345/dolphinscheduler -After backend server is being start, all requests from `pydolphinscheduler` would be sends to backend server. +After backend server is being start, all requests from `pydolphinscheduler` would be sent to backend server. And for now we could run a simple example by: ```shell @@ -78,7 +80,7 @@ just open directory `dolphinscheduler-python/pydolphinscheduler` instead of `dol ### Brief Concept Apache DolphinScheduler is design to define workflow by UI, and pydolphinscheduler try to define it by code. When -define by code, user usually do not care user, tanant, or queue exists or not. All user care about is create +define by code, user usually do not care user, tenant, or queue exists or not. All user care about is created a new workflow by the code his/her definition. So we have some **side object** in `pydolphinscheduler/side` directory, their only check object exists or not, and create them if not exists. @@ -91,9 +93,20 @@ other word for more simple). pydolphinscheduler tasks object, we use tasks to define exact job we want DolphinScheduler do for us. For now, we only support `shell` task to execute shell task. [This link][all-task] list all tasks support in DolphinScheduler -and would be implement in the further. +and would be implemented in the further. +### Testing +pydolphinscheduler using [pytest][pytest] to test our codebase. GitHub Action will run our test when you create +pull request or commit to dev branch, with python version `3.6|3.7|3.8|3.9` and operating system `linux|macOS|windows`. + +To test locally, you could directly run pytest after set `PYTHONPATH` + +```shell +PYTHONPATH=src/ pytest +``` + + [pypi]: https://pypi.org/ [dev-setup]: https://dolphinscheduler.apache.org/en-us/development/development-environment-setup.html [ui-project]: http://8.142.34.29:12345/dolphinscheduler/ui/#/projects/list @@ -101,3 +114,7 @@ and would be implement in the further. [pycharm]: https://www.jetbrains.com/pycharm [idea]: https://www.jetbrains.com/idea/ [all-task]: https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/guide/task/shell.html +[pytest]: https://docs.pytest.org/en/latest/ + +[ga-py-test]: https://github.com/apache/dolphinscheduler/actions/workflows/py-tests.yml/badge.svg?branch=dev +[ga]: https://github.com/apache/dolphinscheduler/actions diff --git a/dolphinscheduler-python/pydolphinscheduler/pytest.ini b/dolphinscheduler-python/pydolphinscheduler/pytest.ini new file mode 100644 index 000000000..f2c7ae648 --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/pytest.ini @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[pytest] +# Do not test test_java_gateway.py due to we can not mock java gateway for now +addopts = --ignore=tests/test_java_gateway.py + +# add path here to skip pytest scan it +norecursedirs = + tests/testing diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index fa97c76b6..51ad74bed 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -151,10 +151,7 @@ class Task(Base): @property def process_definition(self) -> Optional[ProcessDefinition]: - if self._process_definition: - return self._process_definition - else: - raise ValueError(f'Task {self} has not been assigned to a ProcessDefinition yet') + return self._process_definition @process_definition.setter def process_definition(self, process_definition: Optional[ProcessDefinition]): diff --git a/dolphinscheduler-python/pydolphinscheduler/src/__init__.py b/dolphinscheduler-python/pydolphinscheduler/tests/__init__.py similarity index 100% rename from dolphinscheduler-python/pydolphinscheduler/src/__init__.py rename to dolphinscheduler-python/pydolphinscheduler/tests/__init__.py diff --git a/dolphinscheduler-python/pydolphinscheduler/test/__init__.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/__init__.py similarity index 100% rename from dolphinscheduler-python/pydolphinscheduler/test/__init__.py rename to dolphinscheduler-python/pydolphinscheduler/tests/core/__init__.py diff --git a/dolphinscheduler-python/pydolphinscheduler/test/core/test_process_definition.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py similarity index 98% rename from dolphinscheduler-python/pydolphinscheduler/test/core/test_process_definition.py rename to dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py index 7155447d6..83a4c2b96 100644 --- a/dolphinscheduler-python/pydolphinscheduler/test/core/test_process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py @@ -19,8 +19,9 @@ import pytest from pydolphinscheduler.constants import ProcessDefinitionDefault, ProcessDefinitionReleaseState from pydolphinscheduler.core.process_definition import ProcessDefinition -from pydolphinscheduler.core.task import Task, TaskParams +from pydolphinscheduler.core.task import TaskParams from pydolphinscheduler.side import Tenant, Project, User +from tests.testing.task import Task TEST_PROCESS_DEFINITION_NAME = "simple-test-process-definition" diff --git a/dolphinscheduler-python/pydolphinscheduler/test/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py similarity index 93% rename from dolphinscheduler-python/pydolphinscheduler/test/core/test_task.py rename to dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index 1d89b0778..6e0342806 100644 --- a/dolphinscheduler-python/pydolphinscheduler/test/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -52,14 +52,15 @@ def test_task_relation_to_dict(): def test_task_to_dict(): - code = "123" + code = 123 + version = 1 name = "test_task_to_dict" task_type = "test_task_to_dict_type" raw_script = "test_task_params_to_dict" expect = { "code": code, "name": name, - "version": 1, + "version": version, "description": None, "delayTime": 0, "taskType": task_type, @@ -80,14 +81,14 @@ def test_task_to_dict(): }, "flag": "YES", "taskPriority": "MEDIUM", - "workerGroup": "worker-group-pydolphin", + "workerGroup": "default", "failRetryTimes": 0, "failRetryInterval": 1, "timeoutFlag": "CLOSE", "timeoutNotifyStrategy": None, "timeout": 0 } - with patch('pydolphinscheduler.core.task.Task.gen_code', return_value=code): + with patch('pydolphinscheduler.core.task.Task.gen_code_and_version', return_value=(code, version)): task = Task( name=name, task_type=task_type, diff --git a/dolphinscheduler-python/pydolphinscheduler/test/core/__init__.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/__init__.py similarity index 100% rename from dolphinscheduler-python/pydolphinscheduler/test/core/__init__.py rename to dolphinscheduler-python/pydolphinscheduler/tests/tasks/__init__.py diff --git a/dolphinscheduler-python/pydolphinscheduler/test/tasks/test_shell.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py similarity index 91% rename from dolphinscheduler-python/pydolphinscheduler/test/tasks/test_shell.py rename to dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py index ff1b7b546..91cc431ff 100644 --- a/dolphinscheduler-python/pydolphinscheduler/test/tasks/test_shell.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py @@ -22,7 +22,8 @@ from pydolphinscheduler.tasks.shell import Shell def test_shell_to_dict(): - code = "123" + code = 123 + version = 1 name = "test_shell_to_dict" command = "echo test shell" expect = { @@ -49,13 +50,13 @@ def test_shell_to_dict(): }, "flag": "YES", "taskPriority": "MEDIUM", - "workerGroup": "worker-group-pydolphin", + "workerGroup": "default", "failRetryTimes": 0, "failRetryInterval": 1, "timeoutFlag": "CLOSE", "timeoutNotifyStrategy": None, "timeout": 0 } - with patch('pydolphinscheduler.core.task.Task.gen_code', return_value=code): + with patch('pydolphinscheduler.core.task.Task.gen_code_and_version', return_value=(code, version)): shell = Shell(name, command) assert shell.to_dict() == expect diff --git a/dolphinscheduler-python/pydolphinscheduler/test/test_java_gateway.py b/dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py similarity index 100% rename from dolphinscheduler-python/pydolphinscheduler/test/test_java_gateway.py rename to dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py diff --git a/dolphinscheduler-python/pydolphinscheduler/test/example/__init__.py b/dolphinscheduler-python/pydolphinscheduler/tests/testing/__init__.py similarity index 100% rename from dolphinscheduler-python/pydolphinscheduler/test/example/__init__.py rename to dolphinscheduler-python/pydolphinscheduler/tests/testing/__init__.py diff --git a/dolphinscheduler-python/pydolphinscheduler/test/tasks/__init__.py b/dolphinscheduler-python/pydolphinscheduler/tests/testing/task.py similarity index 78% rename from dolphinscheduler-python/pydolphinscheduler/test/tasks/__init__.py rename to dolphinscheduler-python/pydolphinscheduler/tests/testing/task.py index 13a83393a..32d3ffaf7 100644 --- a/dolphinscheduler-python/pydolphinscheduler/test/tasks/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/testing/task.py @@ -14,3 +14,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +import uuid + +from pydolphinscheduler.core.task import Task as SourceTask + + +class Task(SourceTask): + DEFAULT_VERSION = 1 + + def gen_code_and_version(self): + return uuid.uuid1().time, self.DEFAULT_VERSION