Compare commits
2 Commits
dev
...
subscribe-
| Author | SHA1 | Date |
|---|---|---|
lidongdai
|
a850b9b1e3 | |
|
lidongdai
|
21af43a599 |
39
.dlc.json
39
.dlc.json
|
|
@ -1,39 +0,0 @@
|
|||
{
|
||||
"ignorePatterns": [
|
||||
{
|
||||
"pattern": "^http://localhost"
|
||||
},
|
||||
{
|
||||
"pattern": "^https://hive.apache.org"
|
||||
},
|
||||
{
|
||||
"pattern": "^http://192"
|
||||
},
|
||||
{
|
||||
"pattern": "^https://img.shields.io/badge"
|
||||
}
|
||||
],
|
||||
"replacementPatterns": [
|
||||
{
|
||||
"pattern": "^/en-us/download/download.html$",
|
||||
"replacement": "https://dolphinscheduler.apache.org/en-us/download/download.html"
|
||||
},
|
||||
{
|
||||
"pattern": "^/zh-cn/download/download.html$",
|
||||
"replacement": "https://dolphinscheduler.apache.org/zh-cn/download/download.html"
|
||||
},
|
||||
{
|
||||
"pattern": "^/img",
|
||||
"replacement": "{{BASEURL}}/docs/img"
|
||||
}
|
||||
],
|
||||
"timeout": "10s",
|
||||
"retryOn429": true,
|
||||
"retryCount": 10,
|
||||
"fallbackRetryDelay": "1000s",
|
||||
"aliveStatusCodes": [
|
||||
200,
|
||||
401,
|
||||
0
|
||||
]
|
||||
}
|
||||
|
|
@ -20,4 +20,3 @@ dolphinscheduler/dolphinscheduler-e2e @kezhenxu94
|
|||
dolphinscheduler/dolphinscheduler-registry @kezhenxu94
|
||||
dolphinscheduler/dolphinscheduler-standalone-server @kezhenxu94
|
||||
dolphinscheduler/dolphinscheduler-python @zhongjiajie
|
||||
dolphinscheduler/docs @zhongjiajie @Tianqi-Dotes
|
||||
|
|
|
|||
|
|
@ -22,12 +22,11 @@ labels: [ "bug", "Waiting for reply" ]
|
|||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
value: |
|
||||
Please make sure what you are reporting is indeed a bug with reproducible steps, if you want to ask questions
|
||||
or share ideas, you can head to our
|
||||
[Discussions](https://github.com/apache/dolphinscheduler/discussions) tab, you can also
|
||||
[join our slack](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw)
|
||||
and send your question to channel `#troubleshooting`
|
||||
[Discussions](https://github.com/apache/dolphinscheduler/discussions) tab, you can also [subscribe to our mailing list](mailto:dev-subscribe@dolphinscheduler.apache.org) and send
|
||||
emails to [our mailing list](mailto:dev@dolphinscheduler.apache.org)
|
||||
|
||||
For better global communication, Please write in English.
|
||||
|
||||
|
|
|
|||
|
|
@ -38,23 +38,8 @@ concurrency:
|
|||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
paths-filter:
|
||||
name: Backend-Path-Filter
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
not-ignore:
|
||||
- '!(docs/**)'
|
||||
build:
|
||||
name: Backend-Build
|
||||
needs: paths-filter
|
||||
if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
|
|
@ -78,20 +63,3 @@ jobs:
|
|||
-Dmaven.wagon.httpconnectionManager.ttlSeconds=120
|
||||
- name: Check dependency license
|
||||
run: tools/dependencies/check-LICENSE.sh
|
||||
result:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ build, paths-filter ]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Status
|
||||
run: |
|
||||
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
|
||||
echo "Skip Build!"
|
||||
exit 0
|
||||
fi
|
||||
if [[ ${{ needs.build.result }} != 'success' ]]; then
|
||||
echo "Build Failed!"
|
||||
exit -1
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -1,54 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Docs
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'docs/**'
|
||||
|
||||
concurrency:
|
||||
group: doc-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
img-check:
|
||||
name: Image Check
|
||||
timeout-minutes: 15
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: docs
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python 3.9
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Run Image Check
|
||||
run: python img_utils.py -v check
|
||||
dead-link:
|
||||
name: Dead Link
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: sudo npm install -g markdown-link-check@3.10.0
|
||||
- run: |
|
||||
for file in $(find ./docs -name "*.md"); do
|
||||
markdown-link-check -c .dlc.json -q "$file"
|
||||
done
|
||||
|
|
@ -29,23 +29,8 @@ concurrency:
|
|||
|
||||
|
||||
jobs:
|
||||
paths-filter:
|
||||
name: E2E-Path-Filter
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
not-ignore:
|
||||
- '!(docs/**)'
|
||||
build:
|
||||
name: E2E-Build
|
||||
needs: paths-filter
|
||||
if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
|
|
@ -152,20 +137,18 @@ jobs:
|
|||
name: recording-${{ matrix.case.name }}
|
||||
path: ${{ env.RECORDING_PATH }}
|
||||
retention-days: 1
|
||||
|
||||
result:
|
||||
name: E2E
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ e2e, paths-filter ]
|
||||
needs: [ e2e ]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Status
|
||||
run: |
|
||||
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
|
||||
echo "Skip E2E!"
|
||||
exit 0
|
||||
fi
|
||||
if [[ ${{ needs.e2e.result }} != 'success' ]]; then
|
||||
echo "E2E Failed!"
|
||||
if [[ ${{ needs.e2e.result }} == 'success' ]]; then
|
||||
echo "Passed!"
|
||||
else
|
||||
exit -1
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -58,7 +58,6 @@ jobs:
|
|||
node-version: 16
|
||||
- name: Compile and Build
|
||||
run: |
|
||||
npm install pnpm -g
|
||||
pnpm install
|
||||
pnpm run lint
|
||||
pnpm run build:prod
|
||||
npm install
|
||||
npm run lint
|
||||
npm run build:prod
|
||||
|
|
|
|||
|
|
@ -40,8 +40,5 @@ jobs:
|
|||
- name: "Comment in issue"
|
||||
uses: ./.github/actions/comment-on-issue
|
||||
with:
|
||||
message: |
|
||||
Thank you for your feedback, we have received your issue, Please wait patiently for a reply.
|
||||
* In order for us to understand your request as soon as possible, please provide detailed information、version or pictures.
|
||||
* If you haven't received a reply for a long time, you can [join our slack](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw) and send your question to channel `#troubleshooting`
|
||||
message: "Hi:\n* Thank you for your feedback, we have received your issue, Please wait patiently for a reply.\n* In order for us to understand your request as soon as possible, please provide detailed information、version or pictures.\n* If you haven't received a reply for a long time, you can subscribe to the developer's email，Mail subscription steps reference https://dolphinscheduler.apache.org/en-us/community/development/subscribe.html ,Then write the issue URL in the email content and send question to dev@dolphinscheduler.apache.org."
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
|
|
|||
|
|
@ -36,23 +36,8 @@ concurrency:
|
|||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
paths-filter:
|
||||
name: Unit-Test-Path-Filter
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
not-ignore:
|
||||
- '!(docs/**)'
|
||||
unit-test:
|
||||
name: Unit-Test
|
||||
needs: paths-filter
|
||||
if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
|
||||
name: Unit Test
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
|
|
@ -111,20 +96,3 @@ jobs:
|
|||
with:
|
||||
name: unit-test-logs
|
||||
path: ${LOG_DIR}
|
||||
result:
|
||||
name: Unit Test
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ unit-test, paths-filter ]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Status
|
||||
run: |
|
||||
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
|
||||
echo "Skip Unit Test!"
|
||||
exit 0
|
||||
fi
|
||||
if [[ ${{ needs.unit-test.result }} != 'success' ]]; then
|
||||
echo "Unit Test Failed!"
|
||||
exit -1
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ header:
|
|||
- .gitattributes
|
||||
- '**/licenses/**/LICENSE-*'
|
||||
- '**/*.md'
|
||||
- '**/*.svg'
|
||||
- '**/*.json'
|
||||
- '**/*.iml'
|
||||
- '**/*.ini'
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ We would like to express our deep gratitude to all the open-source projects used
|
|||
## Get Help
|
||||
|
||||
1. Submit an [issue](https://github.com/apache/dolphinscheduler/issues/new/choose)
|
||||
2. [Join our slack](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw) and send your question to channel `#troubleshooting`
|
||||
1. Subscribe to this mailing list: https://dolphinscheduler.apache.org/en-us/community/development/subscribe.html, then email dev@dolphinscheduler.apache.org
|
||||
|
||||
## Community
|
||||
|
||||
|
|
|
|||
|
|
@ -87,8 +87,8 @@ Dolphin Scheduler使用了很多优秀的开源项目，比如google的guava、g
|
|||
|
||||
## 获得帮助
|
||||
|
||||
1. 提交 [issue](https://github.com/apache/dolphinscheduler/issues/new/choose)
|
||||
2. [加入slack群](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw) 并在频道 `#troubleshooting` 中提问
|
||||
1. 提交issue
|
||||
2. 先订阅邮件开发列表:[订阅邮件列表](https://dolphinscheduler.apache.org/zh-cn/community/development/subscribe.html), 订阅成功后发送邮件到dev@dolphinscheduler.apache.org.
|
||||
|
||||
## 社区
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,11 @@
|
|||
# DolphinScheduler for Docker and Kubernetes
|
||||
|
||||
* [Start Up DolphinScheduler with Docker](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/guide/installation/docker.html)
|
||||
* [Start Up DolphinScheduler with Kubernetes](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/guide/installation/kubernetes.html)
|
||||
### QuickStart in Docker
|
||||
|
||||
[](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/docker-deployment.html)
|
||||
[](https://dolphinscheduler.apache.org/zh-cn/docs/latest/user_doc/docker-deployment.html)
|
||||
|
||||
### QuickStart in Kubernetes
|
||||
|
||||
[](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/kubernetes-deployment.html)
|
||||
[](https://dolphinscheduler.apache.org/zh-cn/docs/latest/user_doc/kubernetes-deployment.html)
|
||||
|
|
|
|||
|
|
@ -140,6 +140,29 @@ services:
|
|||
networks:
|
||||
- dolphinscheduler
|
||||
|
||||
dolphinscheduler-python-gateway:
|
||||
image: ${HUB}/dolphinscheduler-python:${TAG}
|
||||
ports:
|
||||
- "54321:54321"
|
||||
- "25333:25333"
|
||||
env_file: .env
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "http://localhost:54321/actuator/health" ]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
depends_on:
|
||||
dolphinscheduler-schema-initializer:
|
||||
condition: service_completed_successfully
|
||||
dolphinscheduler-zookeeper:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
|
||||
- dolphinscheduler-shared-local:/opt/soft
|
||||
- dolphinscheduler-resource-local:/dolphinscheduler
|
||||
networks:
|
||||
- dolphinscheduler
|
||||
|
||||
networks:
|
||||
dolphinscheduler:
|
||||
driver: bridge
|
||||
|
|
|
|||
|
|
@ -118,6 +118,27 @@ services:
|
|||
mode: replicated
|
||||
replicas: 1
|
||||
|
||||
dolphinscheduler-python-gateway:
|
||||
image: apache/dolphinscheduler-python-gateway
|
||||
ports:
|
||||
- 54321:54321
|
||||
- 25333:25333
|
||||
env_file: .env
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "http://localhost:54321/actuator/health" ]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
volumes:
|
||||
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
|
||||
- dolphinscheduler-shared-local:/opt/soft
|
||||
- dolphinscheduler-resource-local:/dolphinscheduler
|
||||
networks:
|
||||
- dolphinscheduler
|
||||
deploy:
|
||||
mode: replicated
|
||||
replicas: 1
|
||||
|
||||
networks:
|
||||
dolphinscheduler:
|
||||
driver: overlay
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ version: 2.0.0
|
|||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application.
|
||||
appVersion: dev-SNAPSHOT
|
||||
appVersion: 2.0.4-SNAPSHOT
|
||||
|
||||
dependencies:
|
||||
- name: postgresql
|
||||
|
|
|
|||
|
|
@ -44,6 +44,9 @@ Create default docker images' fullname.
|
|||
{{- define "dolphinscheduler.image.fullname.tools" -}}
|
||||
{{- .Values.image.registry }}/dolphinscheduler-tools:{{ .Values.image.tag | default .Chart.AppVersion -}}
|
||||
{{- end -}}
|
||||
{{- define "dolphinscheduler.image.fullname.python-gateway" -}}
|
||||
{{- .Values.image.registry }}/dolphinscheduler-python-gateway:{{ .Values.image.tag | default .Chart.AppVersion -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a default common labels.
|
||||
|
|
|
|||
|
|
@ -28,10 +28,6 @@ metadata:
|
|||
labels:
|
||||
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}
|
||||
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
rules:
|
||||
- host: {{ .Values.ingress.host }}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ timezone: "Asia/Shanghai"
|
|||
|
||||
image:
|
||||
registry: "dolphinscheduler.docker.scarf.sh/apache"
|
||||
tag: "dev-SNAPSHOT"
|
||||
tag: "2.0.4-SNAPSHOT"
|
||||
pullPolicy: "IfNotPresent"
|
||||
pullSecret: ""
|
||||
|
||||
|
|
@ -403,7 +403,6 @@ ingress:
|
|||
enabled: false
|
||||
host: "dolphinscheduler.org"
|
||||
path: "/dolphinscheduler"
|
||||
annotations: {}
|
||||
tls:
|
||||
enabled: false
|
||||
secretName: "dolphinscheduler-tls"
|
||||
|
|
|
|||
|
|
@ -1,657 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
*/
|
||||
export default {
|
||||
'en-us': {
|
||||
sidemenu: [
|
||||
{
|
||||
title: 'About',
|
||||
children: [
|
||||
{
|
||||
title: 'Introduction',
|
||||
link: '/en-us/docs/dev/user_doc/about/introduction.html',
|
||||
},
|
||||
{
|
||||
title: 'Hardware Environment',
|
||||
link: '/en-us/docs/dev/user_doc/about/hardware.html',
|
||||
},
|
||||
{
|
||||
title: 'Glossary',
|
||||
link: '/en-us/docs/dev/user_doc/about/glossary.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'Quick Start',
|
||||
children: [
|
||||
{
|
||||
title: 'Quick Start',
|
||||
link: '/en-us/docs/dev/user_doc/guide/start/quick-start.html',
|
||||
},
|
||||
{
|
||||
title: 'Docker Deployment',
|
||||
link: '/en-us/docs/dev/user_doc/guide/start/docker.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'Installation',
|
||||
children: [
|
||||
{
|
||||
title: 'Standalone Deployment',
|
||||
link: '/en-us/docs/dev/user_doc/guide/installation/standalone.html',
|
||||
},
|
||||
{
|
||||
title: 'Pseudo Cluster Deployment',
|
||||
link: '/en-us/docs/dev/user_doc/guide/installation/pseudo-cluster.html',
|
||||
},
|
||||
{
|
||||
title: 'Cluster Deployment',
|
||||
link: '/en-us/docs/dev/user_doc/guide/installation/cluster.html',
|
||||
},
|
||||
{
|
||||
title: 'Kubernetes Deployment',
|
||||
link: '/en-us/docs/dev/user_doc/guide/installation/kubernetes.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'Introduction to Functions',
|
||||
children: [
|
||||
{
|
||||
title: 'Workflow Overview',
|
||||
link: '/en-us/docs/dev/user_doc/guide/homepage.html',
|
||||
},
|
||||
{
|
||||
title: 'Project',
|
||||
children: [
|
||||
{
|
||||
title: 'Project List',
|
||||
link: '/en-us/docs/dev/user_doc/guide/project/project-list.html',
|
||||
},
|
||||
{
|
||||
title: 'Workflow Definition',
|
||||
link: '/en-us/docs/dev/user_doc/guide/project/workflow-definition.html',
|
||||
},
|
||||
{
|
||||
title: 'Workflow Instance',
|
||||
link: '/en-us/docs/dev/user_doc/guide/project/workflow-instance.html',
|
||||
},
|
||||
{
|
||||
title: 'Task Instance',
|
||||
link: '/en-us/docs/dev/user_doc/guide/project/task-instance.html',
|
||||
},
|
||||
{
|
||||
title: 'Task Definition',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/project/task-definition.html',
|
||||
},
|
||||
]
|
||||
},
|
||||
{
|
||||
title: 'Task',
|
||||
children: [
|
||||
{
|
||||
title: 'Shell',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/shell.html',
|
||||
},
|
||||
{
|
||||
title: 'SubProcess',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/sub-process.html',
|
||||
},
|
||||
{
|
||||
title: 'Dependent',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/dependent.html',
|
||||
},
|
||||
{
|
||||
title: 'Stored Procedure',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/stored-procedure.html',
|
||||
},
|
||||
{
|
||||
title: 'SQL',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/sql.html',
|
||||
},
|
||||
{
|
||||
title: 'Spark',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/spark.html',
|
||||
},
|
||||
{
|
||||
title: 'MapReduce',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/map-reduce.html',
|
||||
},
|
||||
{
|
||||
title: 'Python',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/python.html',
|
||||
},
|
||||
{
|
||||
title: 'Flink',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/flink.html',
|
||||
},
|
||||
{
|
||||
title: 'HTTP',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/http.html',
|
||||
},
|
||||
{
|
||||
title: 'DataX',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/datax.html',
|
||||
},
|
||||
{
|
||||
title: 'Pigeon',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/pigeon.html',
|
||||
},
|
||||
{
|
||||
title: 'Conditions',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/conditions.html',
|
||||
},
|
||||
{
|
||||
title: 'Switch',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/switch.html',
|
||||
},
|
||||
{
|
||||
title: 'Amazon EMR',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/emr.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'Parameter',
|
||||
children: [
|
||||
{
|
||||
title: 'Built-in Parameter',
|
||||
link: '/en-us/docs/dev/user_doc/guide/parameter/built-in.html',
|
||||
},
|
||||
{
|
||||
title: 'Global Parameter',
|
||||
link: '/en-us/docs/dev/user_doc/guide/parameter/global.html',
|
||||
},
|
||||
{
|
||||
title: 'Local Parameter',
|
||||
link: '/en-us/docs/dev/user_doc/guide/parameter/local.html',
|
||||
},
|
||||
{
|
||||
title: 'Parameter Context',
|
||||
link: '/en-us/docs/dev/user_doc/guide/parameter/context.html',
|
||||
},
|
||||
{
|
||||
title: 'Parameter Priority',
|
||||
link: '/en-us/docs/dev/user_doc/guide/parameter/priority.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'Data Source',
|
||||
children: [
|
||||
{
|
||||
title: 'Introduction',
|
||||
link: '/en-us/docs/dev/user_doc/guide/datasource/introduction.html',
|
||||
},
|
||||
{
|
||||
title: 'MySQL',
|
||||
link: '/en-us/docs/dev/user_doc/guide/datasource/mysql.html',
|
||||
},
|
||||
{
|
||||
title: 'PostgreSQL',
|
||||
link: '/en-us/docs/dev/user_doc/guide/datasource/postgresql.html',
|
||||
},
|
||||
{
|
||||
title: 'HIVE',
|
||||
link: '/en-us/docs/dev/user_doc/guide/datasource/hive.html',
|
||||
},
|
||||
{
|
||||
title: 'Spark',
|
||||
link: '/en-us/docs/dev/user_doc/guide/datasource/spark.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'Alert',
|
||||
children: [
|
||||
{
|
||||
title: 'Alert Component User Guide ',
|
||||
link: '/en-us/docs/dev/user_doc/guide/alert/alert_plugin_user_guide.html',
|
||||
},
|
||||
{
|
||||
title: 'Telegram',
|
||||
link: '/en-us/docs/dev/user_doc/guide/alert/telegram.html',
|
||||
},
|
||||
{
|
||||
title: 'Ding Talk',
|
||||
link: '/en-us/docs/dev/user_doc/guide/alert/dingtalk.html',
|
||||
},
|
||||
{
|
||||
title: 'Enterprise Wechat',
|
||||
link: '/en-us/docs/dev/user_doc/guide/alert/enterprise-wechat.html',
|
||||
},
|
||||
{
|
||||
title: 'Enterprise Webexteams',
|
||||
link: '/en-us/docs/dev/user_doc/guide/alert/enterprise-webexteams.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'Data Quality',
|
||||
link: '/en-us/docs/dev/user_doc/guide/data-quality.html',
|
||||
},
|
||||
{
|
||||
title: 'Resource',
|
||||
link: '/en-us/docs/dev/user_doc/guide/resource.html',
|
||||
},
|
||||
{
|
||||
title: 'Monitor',
|
||||
link: '/en-us/docs/dev/user_doc/guide/monitor.html',
|
||||
},
|
||||
{
|
||||
title: 'Security',
|
||||
link: '/en-us/docs/dev/user_doc/guide/security.html',
|
||||
},
|
||||
{
|
||||
title: 'How-To',
|
||||
children: [
|
||||
{
|
||||
title: 'General Setting',
|
||||
link: '/en-us/docs/dev/user_doc/guide/howto/general-setting.html',
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'Open API',
|
||||
link: '/en-us/docs/dev/user_doc/guide/open-api.html',
|
||||
},
|
||||
{
|
||||
title: 'Flink',
|
||||
link: '/en-us/docs/dev/user_doc/guide/flink-call.html',
|
||||
},
|
||||
{
|
||||
title: 'Upgrade',
|
||||
link: '/en-us/docs/dev/user_doc/guide/upgrade.html',
|
||||
},
|
||||
{
|
||||
title: 'Expansion and Reduction',
|
||||
link: '/en-us/docs/dev/user_doc/guide/expansion-reduction.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'Architecture Guide',
|
||||
children: [
|
||||
{
|
||||
title: 'Architecture Design',
|
||||
link: '/en-us/docs/dev/user_doc/architecture/design.html',
|
||||
},
|
||||
{
|
||||
title: 'Metadata',
|
||||
link: '/en-us/docs/dev/user_doc/architecture/metadata.html',
|
||||
},
|
||||
{
|
||||
title: 'Configuration File',
|
||||
link: '/en-us/docs/dev/user_doc/architecture/configuration.html',
|
||||
},
|
||||
{
|
||||
title: 'Task Structure',
|
||||
link: '/en-us/docs/dev/user_doc/architecture/task-structure.html',
|
||||
},
|
||||
{
|
||||
title: 'Load Balance',
|
||||
link: '/en-us/docs/dev/user_doc/architecture/load-balance.html',
|
||||
},
|
||||
{
|
||||
title: 'Cache',
|
||||
link: '/en-us/docs/dev/user_doc/architecture/cache.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'FAQ',
|
||||
children: [
|
||||
{
|
||||
title: 'FAQ',
|
||||
link: '/en-us/docs/release/faq.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'Older Versions',
|
||||
children: [
|
||||
{
|
||||
title: 'Older Versions',
|
||||
link: '/en-us/docs/release/history-versions.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
barText: 'Documentation',
|
||||
},
|
||||
'zh-cn': {
|
||||
sidemenu: [
|
||||
{
|
||||
title: '关于Apache DolphinScheduler',
|
||||
children: [
|
||||
{
|
||||
title: '简介',
|
||||
link: '/zh-cn/docs/dev/user_doc/about/introduction.html',
|
||||
},
|
||||
{
|
||||
title: '建议配置',
|
||||
link: '/zh-cn/docs/dev/user_doc/about/hardware.html',
|
||||
},
|
||||
{
|
||||
title: '名词解释',
|
||||
link: '/zh-cn/docs/dev/user_doc/about/glossary.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: '快速上手',
|
||||
children: [
|
||||
{
|
||||
title: '快速上手',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/start/quick-start.html',
|
||||
},
|
||||
{
|
||||
title: 'Docker部署(Docker)',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/start/docker.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: '部署指南',
|
||||
children: [
|
||||
{
|
||||
title: '单机部署(Standalone)',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/installation/standalone.html',
|
||||
},
|
||||
{
|
||||
title: '伪集群部署(Pseudo-Cluster)',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/installation/pseudo-cluster.html',
|
||||
},
|
||||
{
|
||||
title: '集群部署(Cluster)',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/installation/cluster.html',
|
||||
},
|
||||
{
|
||||
title: 'Kubernetes部署(Kubernetes)',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/installation/kubernetes.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: '功能介绍',
|
||||
children: [
|
||||
{
|
||||
title: '指标总览',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/homepage.html',
|
||||
},
|
||||
{
|
||||
title: '项目管理',
|
||||
children: [
|
||||
{
|
||||
title: '项目列表',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/project/project-list.html',
|
||||
},
|
||||
{
|
||||
title: '工作流定义',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/project/workflow-definition.html',
|
||||
},
|
||||
{
|
||||
title: '工作流实例',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/project/workflow-instance.html',
|
||||
},
|
||||
{
|
||||
title: '任务实例',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/project/task-instance.html',
|
||||
},
|
||||
{
|
||||
title: '任务定义',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/project/task-definition.html',
|
||||
},
|
||||
]
|
||||
},
|
||||
{
|
||||
title: '任务类型',
|
||||
children: [
|
||||
{
|
||||
title: 'Shell',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/shell.html',
|
||||
},
|
||||
{
|
||||
title: 'SubProcess',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/sub-process.html',
|
||||
},
|
||||
{
|
||||
title: 'Dependent',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/dependent.html',
|
||||
},
|
||||
{
|
||||
title: 'Stored Procedure',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/stored-procedure.html',
|
||||
},
|
||||
{
|
||||
title: 'SQL',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/sql.html',
|
||||
},
|
||||
{
|
||||
title: 'Spark',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/spark.html',
|
||||
},
|
||||
{
|
||||
title: 'MapReduce',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/map-reduce.html',
|
||||
},
|
||||
{
|
||||
title: 'Python',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/python.html',
|
||||
},
|
||||
{
|
||||
title: 'Flink',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/flink.html',
|
||||
},
|
||||
{
|
||||
title: 'HTTP',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/http.html',
|
||||
},
|
||||
{
|
||||
title: 'DataX',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/datax.html',
|
||||
},
|
||||
{
|
||||
title: 'Pigeon',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/pigeon.html',
|
||||
},
|
||||
{
|
||||
title: 'Conditions',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/conditions.html',
|
||||
},
|
||||
{
|
||||
title: 'Switch',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/switch.html',
|
||||
},
|
||||
{
|
||||
title: 'Amazon EMR',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/emr.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: '参数',
|
||||
children: [
|
||||
{
|
||||
title: '内置参数',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/parameter/built-in.html',
|
||||
},
|
||||
{
|
||||
title: '全局参数',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/parameter/global.html',
|
||||
},
|
||||
{
|
||||
title: '本地参数',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/parameter/local.html',
|
||||
},
|
||||
{
|
||||
title: '参数传递',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/parameter/context.html',
|
||||
},
|
||||
{
|
||||
title: '参数优先级',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/parameter/priority.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: '数据源中心',
|
||||
children: [
|
||||
{
|
||||
title: '简介',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/datasource/introduction.html',
|
||||
},
|
||||
{
|
||||
title: 'MySQL',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/datasource/mysql.html',
|
||||
},
|
||||
{
|
||||
title: 'PostgreSQL',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/datasource/postgresql.html',
|
||||
},
|
||||
{
|
||||
title: 'HIVE',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/datasource/hive.html',
|
||||
},
|
||||
{
|
||||
title: 'Spark',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/datasource/spark.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: '告警',
|
||||
children: [
|
||||
{
|
||||
title: '告警组件向导',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/alert/alert_plugin_user_guide.html',
|
||||
},
|
||||
{
|
||||
title: 'Telegram',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/alert/telegram.html',
|
||||
},
|
||||
{
|
||||
title: '钉钉告警',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/alert/dingtalk.html',
|
||||
},
|
||||
{
|
||||
title: '企业微信',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/alert/enterprise-wechat.html',
|
||||
},
|
||||
{
|
||||
title: 'Webexteams',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/alert/enterprise-webexteams.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: '数据质量',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide /data-quality.html',
|
||||
},
|
||||
{
|
||||
title: '资源中心',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/resource.html',
|
||||
},
|
||||
{
|
||||
title: '监控中心',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/monitor.html',
|
||||
},
|
||||
{
|
||||
title: '安全中心',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/security.html',
|
||||
},
|
||||
{
|
||||
title: '常见操作',
|
||||
children: [
|
||||
{
|
||||
title: '通用设置',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/howto/general-setting.html',
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'API调用',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/open-api.html',
|
||||
},
|
||||
{
|
||||
title: 'Flink调用',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/flink-call.html',
|
||||
},
|
||||
{
|
||||
title: '升级',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/upgrade.html',
|
||||
},
|
||||
{
|
||||
title: '扩/缩容',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/expansion-reduction.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: '架构设计',
|
||||
children: [
|
||||
{
|
||||
title: '元数据文档',
|
||||
link: '/zh-cn/docs/dev/user_doc/architecture/metadata.html',
|
||||
},
|
||||
{
|
||||
title: '架构设计',
|
||||
link: '/zh-cn/docs/dev/user_doc/architecture/design.html',
|
||||
},
|
||||
{
|
||||
title: '配置文件',
|
||||
link: '/zh-cn/docs/dev/user_doc/architecture/configuration.html',
|
||||
},
|
||||
{
|
||||
title: '任务结构',
|
||||
link: '/zh-cn/docs/dev/user_doc/architecture/task-structure.html',
|
||||
},
|
||||
{
|
||||
title: '负载均衡',
|
||||
link: '/zh-cn/docs/dev/user_doc/architecture/load-balance.html',
|
||||
},
|
||||
{
|
||||
title: '缓存',
|
||||
link: '/zh-cn/docs/dev/user_doc/architecture/cache.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'FAQ',
|
||||
children: [
|
||||
{
|
||||
title: 'FAQ',
|
||||
link: '/zh-cn/docs/release/faq.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: '历史版本',
|
||||
children: [
|
||||
{
|
||||
title: '历史版本',
|
||||
link: '/zh-cn/docs/release/history-versions.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
barText: '文档',
|
||||
},
|
||||
};
|
||||
|
|
@ -1,129 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import ReactDOM from 'react-dom';
|
||||
import cookie from 'js-cookie';
|
||||
import Language from '../../components/language';
|
||||
import Header from '../../components/header';
|
||||
import Footer from '../../components/footer';
|
||||
import Md2Html from '../../components/md2html';
|
||||
import Sidemenu from '../../components/sidemenu';
|
||||
import siteConfig from '../../../site_config/site';
|
||||
import docs120Config from '../../../site_config/docs1-2-0';
|
||||
import docs121Config from '../../../site_config/docs1-2-1';
|
||||
import docs131Config from '../../../site_config/docs1-3-1';
|
||||
import docs132Config from '../../../site_config/docs1-3-2';
|
||||
import docs133Config from '../../../site_config/docs1-3-3';
|
||||
import docs134Config from '../../../site_config/docs1-3-4';
|
||||
import docs135Config from '../../../site_config/docs1-3-5';
|
||||
import docs136Config from '../../../site_config/docs1-3-6';
|
||||
import docs138Config from '../../../site_config/docs1-3-8';
|
||||
import docs139Config from '../../../site_config/docs1-3-9';
|
||||
import docs200Config from '../../../site_config/docs2-0-0';
|
||||
import docs201Config from '../../../site_config/docs2-0-1';
|
||||
import docs202Config from '../../../site_config/docs2-0-2';
|
||||
import docs203Config from '../../../site_config/docs2-0-3';
|
||||
import docs205Config from '../../../site_config/docs2-0-5';
|
||||
import docs300Config from '../../../site_config/docs3-0-0';
|
||||
import docsDevConfig from '../../../site_config/docsdev';
|
||||
|
||||
const docsSource = {
|
||||
'1.2.0': docs120Config,
|
||||
'1.2.1': docs121Config,
|
||||
'1.3.1': docs131Config,
|
||||
'1.3.2': docs132Config,
|
||||
'1.3.3': docs133Config,
|
||||
'1.3.4': docs134Config,
|
||||
'1.3.5': docs135Config,
|
||||
'1.3.6': docs136Config,
|
||||
'1.3.8': docs138Config,
|
||||
'1.3.9': docs139Config,
|
||||
'2.0.0': docs200Config,
|
||||
'2.0.1': docs201Config,
|
||||
'2.0.2': docs202Config,
|
||||
'2.0.3': docs203Config,
|
||||
'2.0.5': docs205Config,
|
||||
'3.0.0': docs300Config,
|
||||
dev: docsDevConfig,
|
||||
};
|
||||
|
||||
const isValidVersion = version => version && docsSource.hasOwnProperty(version);
|
||||
|
||||
class Docs extends Md2Html(Language) {
|
||||
render() {
|
||||
const language = this.getLanguage();
|
||||
let dataSource = {};
|
||||
// from location path
|
||||
let version = window.location.pathname.split('/')[3];
|
||||
if (isValidVersion(version) || version === 'latest') {
|
||||
cookie.set('docs_version', version);
|
||||
}
|
||||
// from rendering html
|
||||
if (!version && this.props.subdir) {
|
||||
version = this.props.subdir.split('/')[0];
|
||||
}
|
||||
if (isValidVersion(version)) {
|
||||
dataSource = docsSource[version][language];
|
||||
} else if (isValidVersion(cookie.get('docs_version'))) {
|
||||
dataSource = docsSource[cookie.get('docs_version')][language];
|
||||
} else if (isValidVersion(siteConfig.docsLatest)) {
|
||||
dataSource = docsSource[siteConfig.docsLatest][language];
|
||||
dataSource.sidemenu.forEach((menu) => {
|
||||
menu.children.forEach((submenu) => {
|
||||
if (!submenu.children) {
|
||||
submenu.link = submenu.link.replace(`docs/${siteConfig.docsLatest}`, 'docs/latest');
|
||||
} else {
|
||||
submenu.children.forEach((menuLevel3) => {
|
||||
menuLevel3.link = menuLevel3.link.replace(`docs/${siteConfig.docsLatest}`, 'docs/latest');
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
const __html = this.props.__html || this.state.__html;
|
||||
return (
|
||||
<div className="md2html docs-page">
|
||||
<Header
|
||||
currentKey="docs"
|
||||
type="dark"
|
||||
logo="/img/hlogo_white.svg"
|
||||
language={language}
|
||||
onLanguageChange={this.onLanguageChange}
|
||||
/>
|
||||
<section className="content-section">
|
||||
<Sidemenu dataSource={dataSource.sidemenu} />
|
||||
<div
|
||||
className="doc-content markdown-body"
|
||||
ref={(node) => { this.markdownContainer = node; }}
|
||||
dangerouslySetInnerHTML={{ __html }}
|
||||
/>
|
||||
</section>
|
||||
<Footer logo="/img/ds_gray.svg" language={language} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById('root') && ReactDOM.render(<Docs />, document.getElementById('root'));
|
||||
|
||||
export default Docs;
|
||||
|
|
@ -1,404 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
// 全局的一些配置
|
||||
export default {
|
||||
rootPath: '',
|
||||
port: 8080,
|
||||
domain: 'dolphinscheduler.apache.org',
|
||||
copyToDist: ['asset', 'img', 'file', '.asf.yaml', 'sitemap.xml', '.nojekyll', '.htaccess', 'googled0df7b96f277a143.html'],
|
||||
docsLatest: '3.0.0',
|
||||
defaultSearch: 'google', // default search engine
|
||||
defaultLanguage: 'en-us',
|
||||
'en-us': {
|
||||
banner: {
|
||||
text: '🤔 Have queries regarding Apache DolphinScheduler, Join Slack channel to disscuss them ',
|
||||
link: 'https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw'
|
||||
},
|
||||
pageMenu: [
|
||||
{
|
||||
key: 'home',
|
||||
text: 'HOME',
|
||||
link: '/en-us/index.html',
|
||||
},
|
||||
{
|
||||
key: 'docs',
|
||||
text: 'DOCS',
|
||||
link: '/en-us/docs/latest/user_doc/about/introduction.html',
|
||||
children: [
|
||||
{
|
||||
key: 'docs0',
|
||||
text: 'latest(3.0.0-alpha)',
|
||||
link: '/en-us/docs/latest/user_doc/about/introduction.html',
|
||||
},
|
||||
{
|
||||
key: 'docs1',
|
||||
text: '1.3.9',
|
||||
link: '/en-us/docs/1.3.9/user_doc/quick-start.html',
|
||||
},
|
||||
{
|
||||
key: 'docsHistory',
|
||||
text: 'Older Versions',
|
||||
link: '/en-us/docs/release/history-versions.html',
|
||||
},
|
||||
{
|
||||
key: 'pythonAPI',
|
||||
text: 'PyDolphinScheduler',
|
||||
link: '/python/index.html',
|
||||
},
|
||||
{
|
||||
key: 'docsdev',
|
||||
text: 'dev',
|
||||
link: '/en-us/docs/dev/user_doc/about/introduction.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'download',
|
||||
text: 'DOWNLOAD',
|
||||
link: '/en-us/download/download.html',
|
||||
},
|
||||
{ key: 'blog',
|
||||
text: 'BLOG',
|
||||
link: '/en-us/blog/index.html',
|
||||
},
|
||||
{
|
||||
key: 'development',
|
||||
text: 'DEVELOPMENT',
|
||||
link: '/en-us/development/development-environment-setup.html',
|
||||
},
|
||||
{
|
||||
key: 'community',
|
||||
text: 'COMMUNITY',
|
||||
link: '/en-us/community/team.html',
|
||||
},
|
||||
{
|
||||
key: 'ASF',
|
||||
text: 'ASF',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/',
|
||||
children: [
|
||||
{
|
||||
key: 'Foundation',
|
||||
text: 'Foundation',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/',
|
||||
},
|
||||
{
|
||||
key: 'License',
|
||||
text: 'License',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/licenses/',
|
||||
},
|
||||
{
|
||||
key: 'Events',
|
||||
text: 'Events',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/events/current-event',
|
||||
},
|
||||
{
|
||||
key: 'Security',
|
||||
text: 'Security',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/security/',
|
||||
},
|
||||
{
|
||||
key: 'Sponsorship',
|
||||
text: 'Sponsorship',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/foundation/sponsorship.html',
|
||||
},
|
||||
{
|
||||
key: 'Thanks',
|
||||
text: 'Thanks',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/foundation/thanks.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'user',
|
||||
text: 'USER',
|
||||
link: '/en-us/user/index.html',
|
||||
},
|
||||
],
|
||||
documentation: {
|
||||
title: 'Documentation',
|
||||
list: [
|
||||
{
|
||||
text: 'Overview',
|
||||
link: '/en-us/development/architecture-design.html',
|
||||
},
|
||||
{
|
||||
text: 'Quick start',
|
||||
link: '/en-us/docs/latest/user_doc/guide/quick-start.html',
|
||||
},
|
||||
{
|
||||
text: 'Developer guide',
|
||||
link: '/en-us/development/development-environment-setup.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
asf: {
|
||||
title: 'ASF',
|
||||
list: [
|
||||
{
|
||||
text: 'Foundation',
|
||||
link: 'http://www.apache.org',
|
||||
},
|
||||
{
|
||||
text: 'License',
|
||||
link: 'http://www.apache.org/licenses/',
|
||||
},
|
||||
{
|
||||
text: 'Events',
|
||||
link: 'http://www.apache.org/events/current-event',
|
||||
},
|
||||
{
|
||||
text: 'Sponsorship',
|
||||
link: 'http://www.apache.org/foundation/sponsorship.html',
|
||||
},
|
||||
{
|
||||
text: 'Thanks',
|
||||
link: 'http://www.apache.org/foundation/thanks.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
contact: {
|
||||
title: 'About us',
|
||||
content: 'Do you need feedback? Please contact us through the following ways.',
|
||||
list: [
|
||||
{
|
||||
name: 'Email List',
|
||||
img1: '/img/emailgray.png',
|
||||
img2: '/img/emailblue.png',
|
||||
link: '/en-us/community/development/subscribe.html',
|
||||
},
|
||||
{
|
||||
name: 'Twitter',
|
||||
img1: '/img/twittergray.png',
|
||||
img2: '/img/twitterblue.png',
|
||||
link: 'https://twitter.com/dolphinschedule',
|
||||
},
|
||||
{
|
||||
name: 'Stack Overflow',
|
||||
img1: '/img/stackoverflow.png',
|
||||
img2: '/img/stackoverflow-selected.png',
|
||||
link: 'https://stackoverflow.com/questions/tagged/apache-dolphinscheduler',
|
||||
},
|
||||
{
|
||||
name: 'Slack',
|
||||
img1: '/img/slack.png',
|
||||
img2: '/img/slack-selected.png',
|
||||
link: 'https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw',
|
||||
},
|
||||
],
|
||||
},
|
||||
copyright: 'Copyright © 2019-2021 The Apache Software Foundation. Apache DolphinScheduler, DolphinScheduler, and its feather logo are trademarks of The Apache Software Foundation.',
|
||||
},
|
||||
'zh-cn': {
|
||||
banner: {
|
||||
text: '🤔 有关于 Apache DolphinScheduler 的疑问，加入 Slack 频道来讨论他们 ',
|
||||
link: 'https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw'
|
||||
},
|
||||
pageMenu: [
|
||||
{
|
||||
key: 'home',
|
||||
text: '首页',
|
||||
link: '/zh-cn/index.html',
|
||||
},
|
||||
{
|
||||
key: 'docs',
|
||||
text: '文档',
|
||||
link: '/zh-cn/docs/latest/user_doc/about/introduction.html',
|
||||
children: [
|
||||
{
|
||||
key: 'docs0',
|
||||
text: '最新版本latest(3.0.0-alpha)',
|
||||
link: '/zh-cn/docs/latest/user_doc/about/introduction.html',
|
||||
},
|
||||
{
|
||||
key: 'docs1',
|
||||
text: '1.3.9',
|
||||
link: '/zh-cn/docs/1.3.9/user_doc/quick-start.html',
|
||||
},
|
||||
{
|
||||
key: 'docsHistory',
|
||||
text: '历史版本',
|
||||
link: '/zh-cn/docs/release/history-versions.html',
|
||||
},
|
||||
{
|
||||
key: 'pythonAPI',
|
||||
text: 'PyDolphinScheduler',
|
||||
link: '/python/index.html',
|
||||
},
|
||||
{
|
||||
key: 'docsdev',
|
||||
text: 'dev',
|
||||
link: '/zh-cn/docs/dev/user_doc/about/introduction.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'download',
|
||||
text: '下载',
|
||||
link: '/zh-cn/download/download.html',
|
||||
},
|
||||
{
|
||||
key: 'blog',
|
||||
text: '博客',
|
||||
link: '/zh-cn/blog/index.html',
|
||||
},
|
||||
{
|
||||
key: 'development',
|
||||
text: '开发者',
|
||||
link: '/zh-cn/development/development-environment-setup.html',
|
||||
},
|
||||
{
|
||||
key: 'community',
|
||||
text: '社区',
|
||||
link: '/zh-cn/community/team.html',
|
||||
},
|
||||
{
|
||||
key: 'ASF',
|
||||
text: 'ASF',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/',
|
||||
children: [
|
||||
{
|
||||
key: 'Foundation',
|
||||
text: 'Foundation',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/',
|
||||
},
|
||||
{
|
||||
key: 'License',
|
||||
text: 'License',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/licenses/',
|
||||
},
|
||||
{
|
||||
key: 'Events',
|
||||
text: 'Events',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/events/current-event',
|
||||
},
|
||||
{
|
||||
key: 'Security',
|
||||
text: 'Security',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/security/',
|
||||
},
|
||||
{
|
||||
key: 'Sponsorship',
|
||||
text: 'Sponsorship',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/foundation/sponsorship.html',
|
||||
},
|
||||
{
|
||||
key: 'Thanks',
|
||||
text: 'Thanks',
|
||||
target: '_blank',
|
||||
link: 'https://www.apache.org/foundation/thanks.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'user',
|
||||
text: '用户',
|
||||
// link: '',
|
||||
link: '/zh-cn/user/index.html',
|
||||
},
|
||||
],
|
||||
documentation: {
|
||||
title: '文档',
|
||||
list: [
|
||||
{
|
||||
text: '概览',
|
||||
link: '/zh-cn/development/architecture-design.html',
|
||||
},
|
||||
{
|
||||
text: '快速开始',
|
||||
link: '/zh-cn/docs/latest/user_doc/guide/quick-start.html',
|
||||
},
|
||||
{
|
||||
text: '开发者指南',
|
||||
link: '/zh-cn/development/development-environment-setup.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
asf: {
|
||||
title: 'ASF',
|
||||
list: [
|
||||
{
|
||||
text: '基金会',
|
||||
link: 'http://www.apache.org',
|
||||
},
|
||||
{
|
||||
text: '证书',
|
||||
link: 'http://www.apache.org/licenses/',
|
||||
},
|
||||
{
|
||||
text: '事件',
|
||||
link: 'http://www.apache.org/events/current-event',
|
||||
},
|
||||
{
|
||||
text: '赞助',
|
||||
link: 'http://www.apache.org/foundation/sponsorship.html',
|
||||
},
|
||||
{
|
||||
text: '致谢',
|
||||
link: 'http://www.apache.org/foundation/thanks.html',
|
||||
},
|
||||
],
|
||||
},
|
||||
contact: {
|
||||
title: '联系我们',
|
||||
content: '有问题需要反馈？请通过以下方式联系我们。',
|
||||
list: [
|
||||
{
|
||||
name: '邮件列表',
|
||||
img1: '/img/emailgray.png',
|
||||
img2: '/img/emailblue.png',
|
||||
link: '/zh-cn/community/development/subscribe.html',
|
||||
},
|
||||
{
|
||||
name: 'Twitter',
|
||||
img1: '/img/twittergray.png',
|
||||
img2: '/img/twitterblue.png',
|
||||
link: 'https://twitter.com/dolphinschedule',
|
||||
},
|
||||
{
|
||||
name: 'Stack Overflow',
|
||||
img1: '/img/stackoverflow.png',
|
||||
img2: '/img/stackoverflow-selected.png',
|
||||
link: 'https://stackoverflow.com/questions/tagged/apache-dolphinscheduler',
|
||||
},
|
||||
{
|
||||
name: 'Slack',
|
||||
img1: '/img/slack.png',
|
||||
img2: '/img/slack-selected.png',
|
||||
link: 'https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw',
|
||||
},
|
||||
],
|
||||
},
|
||||
copyright: 'Copyright © 2019-2021 The Apache Software Foundation. Apache DolphinScheduler, DolphinScheduler, and its feather logo are trademarks of The Apache Software Foundation.',
|
||||
},
|
||||
};
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
## System Architecture Design
|
||||
|
||||
Before explaining the architecture of the scheduling system, let's first understand the commonly used terms of the
|
||||
scheduling system
|
||||
|
||||
### 1.Glossary
|
||||
|
||||
**DAG：** The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the
|
||||
form of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until
|
||||
there are no subsequent nodes. Examples are as follows:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/dag_examples_cn.jpg" alt="dag example" width="60%" />
|
||||
<p align="center">
|
||||
<em>dag example</em>
|
||||
</p>
|
||||
</p>
|
||||
|
||||
**Process definition**: Visualization formed by dragging task nodes and establishing task node associations**DAG**
|
||||
|
||||
**Process instance**: The process instance is the instantiation of the process definition, which can be generated by
|
||||
manual start or scheduled scheduling. Each time the process definition runs, a process instance is generated
|
||||
|
||||
**Task instance**: The task instance is the instantiation of the task node in the process definition, which identifies
|
||||
the specific task execution status
|
||||
|
||||
**Task type**: Currently supports SHELL, SQL, SUB_PROCESS (sub-process), PROCEDURE, MR, SPARK, PYTHON, DEPENDENT (
|
||||
depends), and plans to support dynamic plug-in expansion, note: **SUB_PROCESS** It is also a separate process
|
||||
definition that can be started and executed separately
|
||||
|
||||
**Scheduling method**: The system supports scheduled scheduling and manual scheduling based on cron expressions. Command
|
||||
type support: start workflow, start execution from current node, resume fault-tolerant workflow, resume pause process,
|
||||
start execution from failed node, complement, timing, rerun, pause, stop, resume waiting thread. Among them **Resume
|
||||
fault-tolerant workflow** and **Resume waiting thread** The two command types are used by the internal control of
|
||||
scheduling, and cannot be called from the outside
|
||||
|
||||
**Scheduled**: System adopts **quartz** distributed scheduler, and supports the visual generation of cron expressions
|
||||
|
||||
**Rely**: The system not only supports **DAG** simple dependencies between the predecessor and successor nodes, but also
|
||||
provides **task dependent** nodes, supporting **between processes**
|
||||
|
||||
**Priority**: Support the priority of process instances and task instances, if the priority of process instances and
|
||||
task instances is not set, the default is first-in-first-out
|
||||
|
||||
**Email alert**: Support **SQL task** Query result email sending, process instance running result email alert and fault
|
||||
tolerance alert notification
|
||||
|
||||
**Failure strategy**: For tasks running in parallel, if a task fails, two failure strategy processing methods are
|
||||
provided. **Continue** refers to regardless of the status of the task running in parallel until the end of the process
|
||||
failure. **End** means that once a failed task is found, Kill will also run the parallel task at the same time, and the
|
||||
process fails and ends
|
||||
|
||||
**Complement**: Supplement historical data，Supports **interval parallel and serial** two complement methods
|
||||
|
||||
### 2.Module introduction
|
||||
|
||||
- dolphinscheduler-alert alarm module, providing AlertServer service.
|
||||
|
||||
- dolphinscheduler-api web application module, providing ApiServer service.
|
||||
|
||||
- dolphinscheduler-common General constant enumeration, utility class, data structure or base class
|
||||
|
||||
- dolphinscheduler-dao provides operations such as database access.
|
||||
|
||||
- dolphinscheduler-remote client and server based on netty
|
||||
|
||||
- dolphinscheduler-server MasterServer and WorkerServer services
|
||||
|
||||
- dolphinscheduler-service service module, including Quartz, Zookeeper, log client access service, easy to call server
|
||||
module and api module
|
||||
|
||||
- dolphinscheduler-ui front-end module
|
||||
|
||||
### Sum up
|
||||
|
||||
From the perspective of scheduling, this article preliminarily introduces the architecture principles and implementation
|
||||
ideas of the big data distributed workflow scheduling system-DolphinScheduler. To be continued
|
||||
|
||||
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
# Hardware Environment
|
||||
|
||||
DolphinScheduler, as an open-source distributed workflow task scheduling system, can deploy and run smoothly in Intel architecture server environments and mainstream virtualization environments and supports mainstream Linux operating system environments.
|
||||
|
||||
## Linux Operating System Version Requirements
|
||||
|
||||
| OS | Version |
|
||||
| :----------------------- | :----------: |
|
||||
| Red Hat Enterprise Linux | 7.0 and above |
|
||||
| CentOS | 7.0 and above |
|
||||
| Oracle Enterprise Linux | 7.0 and above |
|
||||
| Ubuntu LTS | 16.04 and above |
|
||||
|
||||
> **Attention:**
|
||||
>The above Linux operating systems can run on physical servers and mainstream virtualization environments such as VMware, KVM, and XEN.
|
||||
|
||||
## Recommended Server Configuration
|
||||
|
||||
DolphinScheduler supports 64-bit hardware platforms with Intel x86-64 architecture. The following shows the recommended server requirements in a production environment:
|
||||
|
||||
### Production Environment
|
||||
|
||||
| **CPU** | **MEM** | **HD** | **NIC** | **Num** |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 4 core+ | 8 GB+ | SAS | GbE | 1+ |
|
||||
|
||||
> **Attention:**
|
||||
> - The above recommended configuration is the minimum configuration for deploying DolphinScheduler. Higher configuration is strongly recommended for production environments.
|
||||
> - The recommended hard disk size is more than 50GB and separate the system disk and data disk.
|
||||
|
||||
|
||||
## Network Requirements
|
||||
|
||||
DolphinScheduler provides the following network port configurations for normal operation:
|
||||
|
||||
| Server | Port | Desc |
|
||||
| --- | --- | --- |
|
||||
| MasterServer | 5678 | not the communication port, require the native ports do not conflict |
|
||||
| WorkerServer | 1234 | not the communication port, require the native ports do not conflict |
|
||||
| ApiApplicationServer | 12345 | backend communication port |
|
||||
|
||||
> **Attention:**
|
||||
> - MasterServer and WorkerServer do not need to enable communication between the networks. As long as the local ports do not conflict.
|
||||
> - Administrators can adjust relevant ports on the network side and host-side according to the deployment plan of DolphinScheduler components in the actual environment.
|
||||
|
||||
## Browser Requirements
|
||||
|
||||
DolphinScheduler recommends Chrome and the latest browsers which use Chrome Kernel to access the front-end UI page.
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
# About DolphinScheduler
|
||||
|
||||
Apache DolphinScheduler is a distributed, easy to extend visual DAG workflow task scheduling open-source system. Solves the intricate dependencies of data R&D ETL and the inability to monitor the health status of tasks. DolphinScheduler assembles tasks in the DAG streaming way, which can monitor the execution status of tasks in time, and supports operations like retry, recovery failure from specified nodes, pause, resume and kill tasks, etc.
|
||||
|
||||
## Simple to Use
|
||||
|
||||
- DolphinScheduler has DAG monitoring user interfaces, users can customize DAG by dragging and dropping. All process definitions are visualized, supports rich third-party systems APIs and one-click deployment.
|
||||
|
||||
## High Reliability
|
||||
|
||||
- Decentralized multi-masters and multi-workers, support HA, select queues to avoid overload.
|
||||
|
||||
## Rich Scenarios
|
||||
|
||||
- Support features like multi-tenants, suspend and resume operations to cope with big data scenarios. Support many task types like Spark, Flink, Hive, MR, shell, python, sub_process.
|
||||
|
||||
## High Scalability
|
||||
|
||||
- Supports customized task types, distributed scheduling, and the overall scheduling capability increases linearly with the scale of the cluster.
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
# Cache
|
||||
|
||||
## Purpose
|
||||
|
||||
Due to the large database read operations during the master-server scheduling process. Such as read tables like `tenant`, `user`, `processDefinition`, etc. Operations stress read pressure to the DB, and slow down the entire core scheduling process.
|
||||
|
||||
By considering this part of the business data is a high-read and low-write scenario, a cache module is introduced to reduce the DB read pressure and speed up the core scheduling process.
|
||||
|
||||
## Cache Settings
|
||||
|
||||
```yaml
|
||||
spring:
|
||||
cache:
|
||||
# default disable cache, you can enable by `type: caffeine`
|
||||
type: none
|
||||
cache-names:
|
||||
- tenant
|
||||
- user
|
||||
- processDefinition
|
||||
- processTaskRelation
|
||||
- taskDefinition
|
||||
caffeine:
|
||||
spec: maximumSize=100,expireAfterWrite=300s,recordStats
|
||||
```
|
||||
|
||||
The cache module uses [spring-cache](https://spring.io/guides/gs/caching/), so you can set cache config like whether to enable cache (`none` to disable by default), cache types in the spring `application.yaml` directly.
|
||||
|
||||
Currently, implements the config of [caffeine](https://github.com/ben-manes/caffeine), you can assign cache configs like cache size, expire time, etc.
|
||||
|
||||
## Cache Read
|
||||
|
||||
The cache module adopts the `@Cacheable` annotation from spring-cache and you can annotate the annotation in the related mapper layer. Refer to the `TenantMapper`.
|
||||
|
||||
## Cache Evict
|
||||
|
||||
The business data updates come from the api-server, and the cache side is in the master-server. Then it is necessary to monitor the data updates from the api-server (use aspect point cut interceptor `@CacheEvict`), and notify the master-server of `cacheEvictCommand` when processing a cache eviction.
|
||||
|
||||
Note: the final strategy for cache update comes from the expiration strategy configuration in caffeine, therefore configure it under the business scenarios;
|
||||
|
||||
The sequence diagram shows below:
|
||||
|
||||
<img src="/img/cache-evict.png" alt="cache-evict" style="zoom: 67%;" />
|
||||
|
|
@ -1,444 +0,0 @@
|
|||
<!-- markdown-link-check-disable -->
|
||||
|
||||
# Configuration
|
||||
|
||||
## Preface
|
||||
|
||||
This document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.
|
||||
|
||||
## Directory Structure
|
||||
|
||||
Currently, all the configuration files are under [conf ] directory.
|
||||
Check the following simplified DolphinScheduler installation directories to have a direct view about the position of [conf] directory and configuration files it has.
|
||||
This document only describes DolphinScheduler configurations and other topics are not going into.
|
||||
|
||||
[Note: the DolphinScheduler (hereinafter called the ‘DS’) .]
|
||||
|
||||
```
|
||||
├─bin DS application commands directory
|
||||
│ ├─dolphinscheduler-daemon.sh startup or shutdown DS application
|
||||
│ ├─start-all.sh startup all DS services with configurations
|
||||
│ ├─stop-all.sh shutdown all DS services with configurations
|
||||
├─conf configurations directory
|
||||
│ ├─application-api.properties API-service config properties
|
||||
│ ├─datasource.properties datasource config properties
|
||||
│ ├─zookeeper.properties ZooKeeper config properties
|
||||
│ ├─master.properties master-service config properties
|
||||
│ ├─worker.properties worker-service config properties
|
||||
│ ├─quartz.properties quartz config properties
|
||||
│ ├─common.properties common-service [storage] config properties
|
||||
│ ├─alert.properties alert-service config properties
|
||||
│ ├─config environment variables config directory
|
||||
│ ├─install_config.conf DS environment variables configuration script [install or start DS]
|
||||
│ ├─env load environment variables configs script directory
|
||||
│ ├─dolphinscheduler_env.sh load environment variables configs [eg: JAVA_HOME,HADOOP_HOME, HIVE_HOME ...]
|
||||
│ ├─org mybatis mapper files directory
|
||||
│ ├─i18n i18n configs directory
|
||||
│ ├─logback-api.xml API-service log config
|
||||
│ ├─logback-master.xml master-service log config
|
||||
│ ├─logback-worker.xml worker-service log config
|
||||
│ ├─logback-alert.xml alert-service log config
|
||||
├─sql .sql files to create or upgrade DS metadata
|
||||
│ ├─create create SQL scripts directory
|
||||
│ ├─upgrade upgrade SQL scripts directory
|
||||
│ ├─dolphinscheduler_postgre.sql PostgreSQL database init script
|
||||
│ ├─dolphinscheduler_mysql.sql MySQL database init script
|
||||
│ ├─soft_version current DS version-id file
|
||||
├─script DS services deployment, database create or upgrade scripts directory
|
||||
│ ├─create-dolphinscheduler.sh DS database init script
|
||||
│ ├─upgrade-dolphinscheduler.sh DS database upgrade script
|
||||
│ ├─monitor-server.sh DS monitor-server start script
|
||||
│ ├─scp-hosts.sh transfer installation files script
|
||||
│ ├─remove-zk-node.sh cleanup ZooKeeper caches script
|
||||
├─ui front-end web resources directory
|
||||
├─lib DS .jar dependencies directory
|
||||
├─install.sh auto-setup DS services script
|
||||
```
|
||||
|
||||
## Configurations in Details
|
||||
|
||||
serial number| service classification| config file|
|
||||
|--|--|--|
|
||||
1|startup or shutdown DS application|dolphinscheduler-daemon.sh
|
||||
2|datasource config properties|datasource.properties
|
||||
3|ZooKeeper config properties|zookeeper.properties
|
||||
4|common-service[storage] config properties|common.properties
|
||||
5|API-service config properties|application-api.properties
|
||||
6|master-service config properties|master.properties
|
||||
7|worker-service config properties|worker.properties
|
||||
8|alert-service config properties|alert.properties
|
||||
9|quartz config properties|quartz.properties
|
||||
10|DS environment variables configuration script[install/start DS]|install_config.conf
|
||||
11|load environment variables configs <br /> [eg: JAVA_HOME,HADOOP_HOME, HIVE_HOME ...]|dolphinscheduler_env.sh
|
||||
12|services log config files|API-service log config : logback-api.xml <br /> master-service log config : logback-master.xml <br /> worker-service log config : logback-worker.xml <br /> alert-service log config : logback-alert.xml
|
||||
|
||||
|
||||
### dolphinscheduler-daemon.sh [startup or shutdown DS application]
|
||||
|
||||
dolphinscheduler-daemon.sh is responsible for DS startup and shutdown.
|
||||
Essentially, start-all.sh or stop-all.sh startup and shutdown the cluster via dolphinscheduler-daemon.sh.
|
||||
Currently, DS just makes a basic config, remember to config further JVM options based on your practical situation of resources.
|
||||
|
||||
Default simplified parameters are:
|
||||
```bash
|
||||
export DOLPHINSCHEDULER_OPTS="
|
||||
-server
|
||||
-Xmx16g
|
||||
-Xms1g
|
||||
-Xss512k
|
||||
-XX:+UseConcMarkSweepGC
|
||||
-XX:+CMSParallelRemarkEnabled
|
||||
-XX:+UseFastAccessorMethods
|
||||
-XX:+UseCMSInitiatingOccupancyOnly
|
||||
-XX:CMSInitiatingOccupancyFraction=70
|
||||
"
|
||||
```
|
||||
|
||||
> "-XX:DisableExplicitGC" is not recommended due to may lead to memory link (DS dependent on Netty to communicate).
|
||||
|
||||
### datasource.properties [datasource config properties]
|
||||
|
||||
DS uses Druid to manage database connections and default simplified configs are:
|
||||
|Parameters | Default value| Description|
|
||||
|--|--|--|
|
||||
spring.datasource.driver-class-name||datasource driver
|
||||
spring.datasource.url||datasource connection url
|
||||
spring.datasource.username||datasource username
|
||||
spring.datasource.password||datasource password
|
||||
spring.datasource.initialSize|5| initial connection pool size number
|
||||
spring.datasource.minIdle|5| minimum connection pool size number
|
||||
spring.datasource.maxActive|5| maximum connection pool size number
|
||||
spring.datasource.maxWait|60000| max wait milliseconds
|
||||
spring.datasource.timeBetweenEvictionRunsMillis|60000| idle connection check interval
|
||||
spring.datasource.timeBetweenConnectErrorMillis|60000| retry interval
|
||||
spring.datasource.minEvictableIdleTimeMillis|300000| connections over minEvictableIdleTimeMillis will be collect when idle check
|
||||
spring.datasource.validationQuery|SELECT 1| validate connection by running the SQL
|
||||
spring.datasource.validationQueryTimeout|3| validate connection timeout[seconds]
|
||||
spring.datasource.testWhileIdle|true| set whether the pool validates the allocated connection when a new connection request comes
|
||||
spring.datasource.testOnBorrow|true| validity check when the program requests a new connection
|
||||
spring.datasource.testOnReturn|false| validity check when the program recalls a connection
|
||||
spring.datasource.defaultAutoCommit|true| whether auto commit
|
||||
spring.datasource.keepAlive|true| runs validationQuery SQL to avoid the connection closed by pool when the connection idles over minEvictableIdleTimeMillis
|
||||
spring.datasource.poolPreparedStatements|true| open PSCache
|
||||
spring.datasource.maxPoolPreparedStatementPerConnectionSize|20| specify the size of PSCache on each connection
|
||||
|
||||
|
||||
### zookeeper.properties [zookeeper config properties]
|
||||
|
||||
|Parameters | Default value| Description|
|
||||
|--|--|--|
|
||||
zookeeper.quorum|localhost:2181| ZooKeeper cluster connection info
|
||||
zookeeper.dolphinscheduler.root|/dolphinscheduler| DS is stored under ZooKeeper root directory
|
||||
zookeeper.session.timeout|60000| session timeout
|
||||
zookeeper.connection.timeout|30000| connection timeout
|
||||
zookeeper.retry.base.sleep|100| time to wait between subsequent retries
|
||||
zookeeper.retry.max.sleep|30000| maximum time to wait between subsequent retries
|
||||
zookeeper.retry.maxtime|10| maximum retry times
|
||||
|
||||
|
||||
### common.properties [hadoop、s3、yarn config properties]
|
||||
|
||||
Currently, common.properties mainly configures Hadoop,s3a related configurations.
|
||||
|Parameters | Default value| Description|
|
||||
|--|--|--|
|
||||
data.basedir.path|/tmp/dolphinscheduler| local directory used to store temp files
|
||||
resource.storage.type|NONE| type of resource files: HDFS, S3, NONE
|
||||
resource.upload.path|/dolphinscheduler| storage path of resource files
|
||||
hadoop.security.authentication.startup.state|false| whether hadoop grant kerberos permission
|
||||
java.security.krb5.conf.path|/opt/krb5.conf|kerberos config directory
|
||||
login.user.keytab.username|hdfs-mycluster@ESZ.COM|kerberos username
|
||||
login.user.keytab.path|/opt/hdfs.headless.keytab|kerberos user keytab
|
||||
kerberos.expire.time|2|kerberos expire time,integer,the unit is hour
|
||||
resource.view.suffixs| txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties| file types supported by resource center
|
||||
hdfs.root.user|hdfs| configure users with corresponding permissions if storage type is HDFS
|
||||
fs.defaultFS|hdfs://mycluster:8020|If resource.storage.type=S3, then the request url would be similar to 's3a://dolphinscheduler'. Otherwise if resource.storage.type=HDFS and hadoop supports HA, copy core-site.xml and hdfs-site.xml into 'conf' directory
|
||||
fs.s3a.endpoint||s3 endpoint url
|
||||
fs.s3a.access.key||s3 access key
|
||||
fs.s3a.secret.key||s3 secret key
|
||||
yarn.resourcemanager.ha.rm.ids||specify the yarn resourcemanager url. if resourcemanager supports HA, input HA IP addresses (separated by comma), or input null for standalone
|
||||
yarn.application.status.address|http://ds1:8088/ws/v1/cluster/apps/%s|keep default if ResourceManager supports HA or not use ResourceManager, or replace ds1 with corresponding hostname if ResourceManager in standalone mode
|
||||
dolphinscheduler.env.path|env/dolphinscheduler_env.sh|load environment variables configs [eg: JAVA_HOME,HADOOP_HOME, HIVE_HOME ...]
|
||||
development.state|false| specify whether in development state
|
||||
|
||||
|
||||
### application-api.properties [API-service log config]
|
||||
|
||||
|Parameters | Default value| Description|
|
||||
|--|--|--|
|
||||
server.port|12345|api service communication port
|
||||
server.servlet.session.timeout|7200|session timeout
|
||||
server.servlet.context-path|/dolphinscheduler | request path
|
||||
spring.servlet.multipart.max-file-size|1024MB| maximum file size
|
||||
spring.servlet.multipart.max-request-size|1024MB| maximum request size
|
||||
server.jetty.max-http-post-size|5000000| jetty maximum post size
|
||||
spring.messages.encoding|UTF-8| message encoding
|
||||
spring.jackson.time-zone|GMT+8| time zone
|
||||
spring.messages.basename|i18n/messages| i18n config
|
||||
security.authentication.type|PASSWORD| authentication type
|
||||
|
||||
|
||||
### master.properties [master-service log config]
|
||||
|
||||
|Parameters | Default value| Description|
|
||||
|--|--|--|
|
||||
master.listen.port|5678|master listen port
|
||||
master.exec.threads|100|master-service execute thread number, used to limit the number of process instances in parallel
|
||||
master.exec.task.num|20|defines the number of parallel tasks for each process instance of the master-service
|
||||
master.dispatch.task.num|3|defines the number of dispatch tasks for each batch of the master-service
|
||||
master.host.selector|LowerWeight|master host selector, to select a suitable worker to run the task, optional value: random, round-robin, lower weight
|
||||
master.heartbeat.interval|10|master heartbeat interval, the unit is second
|
||||
master.task.commit.retryTimes|5|master commit task retry times
|
||||
master.task.commit.interval|1000|master commit task interval, the unit is millisecond
|
||||
master.max.cpuload.avg|-1|master max CPU load avg, only higher than the system CPU load average, master server can schedule. default value -1: the number of CPU cores * 2
|
||||
master.reserved.memory|0.3|master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G
|
||||
|
||||
|
||||
### worker.properties [worker-service log config]
|
||||
|
||||
|Parameters | Default value| Description|
|
||||
|--|--|--|
|
||||
worker.listen.port|1234|worker-service listen port
|
||||
worker.exec.threads|100|worker-service execute thread number, used to limit the number of task instances in parallel
|
||||
worker.heartbeat.interval|10|worker-service heartbeat interval, the unit is second
|
||||
worker.max.cpuload.avg|-1|worker max CPU load avg, only higher than the system CPU load average, worker server can be dispatched tasks. default value -1: the number of CPU cores * 2
|
||||
worker.reserved.memory|0.3|worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G
|
||||
worker.groups|default|worker groups separated by comma, e.g., 'worker.groups=default,test' <br> worker will join corresponding group according to this config when startup
|
||||
|
||||
|
||||
### alert.properties [alert-service log config]
|
||||
|
||||
|Parameters | Default value| Description|
|
||||
|--|--|--|
|
||||
alert.type|EMAIL|alter type|
|
||||
mail.protocol|SMTP|mail server protocol
|
||||
mail.server.host|xxx.xxx.com|mail server host
|
||||
mail.server.port|25|mail server port
|
||||
mail.sender|xxx@xxx.com|mail sender email
|
||||
mail.user|xxx@xxx.com|mail sender email name
|
||||
mail.passwd|111111|mail sender email password
|
||||
mail.smtp.starttls.enable|true|specify mail whether open tls
|
||||
mail.smtp.ssl.enable|false|specify mail whether open ssl
|
||||
mail.smtp.ssl.trust|xxx.xxx.com|specify mail ssl trust list
|
||||
xls.file.path|/tmp/xls|mail attachment temp storage directory
|
||||
||following configure WeCom[optional]|
|
||||
enterprise.wechat.enable|false|specify whether enable WeCom
|
||||
enterprise.wechat.corp.id|xxxxxxx|WeCom corp id
|
||||
enterprise.wechat.secret|xxxxxxx|WeCom secret
|
||||
enterprise.wechat.agent.id|xxxxxxx|WeCom agent id
|
||||
enterprise.wechat.users|xxxxxxx|WeCom users
|
||||
enterprise.wechat.token.url|https://qyapi.weixin.qq.com/cgi-bin/gettoken? <br /> corpid=$corpId&corpsecret=$secret|WeCom token url
|
||||
enterprise.wechat.push.url|https://qyapi.weixin.qq.com/cgi-bin/message/send? <br /> access_token=$token|WeCom push url
|
||||
enterprise.wechat.user.send.msg||send message format
|
||||
enterprise.wechat.team.send.msg||group message format
|
||||
plugin.dir|/Users/xx/your/path/to/plugin/dir|plugin directory
|
||||
|
||||
|
||||
### quartz.properties [quartz config properties]
|
||||
|
||||
This part describes quartz configs and configure them based on your practical situation and resources.
|
||||
|Parameters | Default value| Description|
|
||||
|--|--|--|
|
||||
org.quartz.jobStore.driverDelegateClass | org.quartz.impl.jdbcjobstore.StdJDBCDelegate |
|
||||
org.quartz.jobStore.driverDelegateClass | org.quartz.impl.jdbcjobstore.PostgreSQLDelegate |
|
||||
org.quartz.scheduler.instanceName | DolphinScheduler |
|
||||
org.quartz.scheduler.instanceId | AUTO |
|
||||
org.quartz.scheduler.makeSchedulerThreadDaemon | true |
|
||||
org.quartz.jobStore.useProperties | false |
|
||||
org.quartz.threadPool.class | org.quartz.simpl.SimpleThreadPool |
|
||||
org.quartz.threadPool.makeThreadsDaemons | true |
|
||||
org.quartz.threadPool.threadCount | 25 |
|
||||
org.quartz.threadPool.threadPriority | 5 |
|
||||
org.quartz.jobStore.class | org.quartz.impl.jdbcjobstore.JobStoreTX |
|
||||
org.quartz.jobStore.tablePrefix | QRTZ_ |
|
||||
org.quartz.jobStore.isClustered | true |
|
||||
org.quartz.jobStore.misfireThreshold | 60000 |
|
||||
org.quartz.jobStore.clusterCheckinInterval | 5000 |
|
||||
org.quartz.jobStore.acquireTriggersWithinLock|true |
|
||||
org.quartz.jobStore.dataSource | myDs |
|
||||
org.quartz.dataSource.myDs.connectionProvider.class | org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider |
|
||||
|
||||
|
||||
### install_config.conf [DS environment variables configuration script[install or start DS]]
|
||||
|
||||
install_config.conf is a bit complicated and is mainly used in the following two places.
|
||||
* DS Cluster Auto Installation.
|
||||
|
||||
> System will load configs in the install_config.conf and auto-configure files below, based on the file content when executing 'install.sh'.
|
||||
> Files such as dolphinscheduler-daemon.sh, datasource.properties, zookeeper.properties, common.properties, application-api.properties, master.properties, worker.properties, alert.properties, quartz.properties, etc.
|
||||
|
||||
* Startup and Shutdown DS Cluster.
|
||||
|
||||
> The system will load masters, workers, alert-server, API-servers and other parameters inside the file to startup or shutdown DS cluster.
|
||||
|
||||
#### File Content
|
||||
|
||||
```bash
|
||||
|
||||
# Note: please escape the character if the file contains special characters such as `.*[]^${}\+?|()@#&`.
|
||||
# eg: `[` escape to `\[`
|
||||
|
||||
# Database type (DS currently only supports PostgreSQL and MySQL)
|
||||
dbtype="mysql"
|
||||
|
||||
# Database url and port
|
||||
dbhost="192.168.xx.xx:3306"
|
||||
|
||||
# Database name
|
||||
dbname="dolphinscheduler"
|
||||
|
||||
|
||||
# Database username
|
||||
username="xx"
|
||||
|
||||
# Database password
|
||||
password="xx"
|
||||
|
||||
# ZooKeeper url
|
||||
zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
|
||||
|
||||
# DS installation path, such as '/data1_1T/dolphinscheduler'
|
||||
installPath="/data1_1T/dolphinscheduler"
|
||||
|
||||
# Deployment user
|
||||
# Note: Deployment user needs 'sudo' privilege and has rights to operate HDFS.
|
||||
# Root directory must be created by the same user if using HDFS, otherwise permission related issues will be raised.
|
||||
deployUser="dolphinscheduler"
|
||||
|
||||
|
||||
# Followings are alert-service configs
|
||||
# Mail server host
|
||||
mailServerHost="smtp.exmail.qq.com"
|
||||
|
||||
# Mail server port
|
||||
mailServerPort="25"
|
||||
|
||||
# Mail sender
|
||||
mailSender="xxxxxxxxxx"
|
||||
|
||||
# Mail user
|
||||
mailUser="xxxxxxxxxx"
|
||||
|
||||
# Mail password
|
||||
mailPassword="xxxxxxxxxx"
|
||||
|
||||
# Whether mail supports TLS
|
||||
starttlsEnable="true"
|
||||
|
||||
# Whether mail supports SSL. Note: starttlsEnable and sslEnable cannot both set true.
|
||||
sslEnable="false"
|
||||
|
||||
# Mail server host, same as mailServerHost
|
||||
sslTrust="smtp.exmail.qq.com"
|
||||
|
||||
# Specify which resource upload function to use for resources storage, such as sql files. And supported options are HDFS, S3 and NONE. HDFS for upload to HDFS and NONE for not using this function.
|
||||
resourceStorageType="NONE"
|
||||
|
||||
# if S3, write S3 address. HA, for example: s3a://dolphinscheduler，
|
||||
# Note: s3 make sure to create the root directory /dolphinscheduler
|
||||
defaultFS="hdfs://mycluster:8020"
|
||||
|
||||
# If parameter 'resourceStorageType' is S3, following configs are needed:
|
||||
s3Endpoint="http://192.168.xx.xx:9010"
|
||||
s3AccessKey="xxxxxxxxxx"
|
||||
s3SecretKey="xxxxxxxxxx"
|
||||
|
||||
# If ResourceManager supports HA, then input master and standby node IP or hostname, eg: '192.168.xx.xx,192.168.xx.xx'. Or else ResourceManager run in standalone mode, please set yarnHaIps="" and "" for not using yarn.
|
||||
yarnHaIps="192.168.xx.xx,192.168.xx.xx"
|
||||
|
||||
|
||||
# If ResourceManager runs in standalone, then set ResourceManager node ip or hostname, or else remain default.
|
||||
singleYarnIp="yarnIp1"
|
||||
|
||||
# Storage path when using HDFS/S3
|
||||
resourceUploadPath="/dolphinscheduler"
|
||||
|
||||
|
||||
# HDFS/S3 root user
|
||||
hdfsRootUser="hdfs"
|
||||
|
||||
# Followings are Kerberos configs
|
||||
|
||||
# Specify Kerberos enable or not
|
||||
kerberosStartUp="false"
|
||||
|
||||
# Kdc krb5 config file path
|
||||
krb5ConfPath="$installPath/conf/krb5.conf"
|
||||
|
||||
# Keytab username
|
||||
keytabUserName="hdfs-mycluster@ESZ.COM"
|
||||
|
||||
# Username keytab path
|
||||
keytabPath="$installPath/conf/hdfs.headless.keytab"
|
||||
|
||||
|
||||
# API-service port
|
||||
apiServerPort="12345"
|
||||
|
||||
|
||||
# All hosts deploy DS
|
||||
ips="ds1,ds2,ds3,ds4,ds5"
|
||||
|
||||
# Ssh port, default 22
|
||||
sshPort="22"
|
||||
|
||||
# Master service hosts
|
||||
masters="ds1,ds2"
|
||||
|
||||
# All hosts deploy worker service
|
||||
# Note: Each worker needs to set a worker group name and default name is "default"
|
||||
workers="ds1:default,ds2:default,ds3:default,ds4:default,ds5:default"
|
||||
|
||||
# Host deploy alert-service
|
||||
alertServer="ds3"
|
||||
|
||||
# Host deploy API-service
|
||||
apiServers="ds1"
|
||||
```
|
||||
|
||||
### dolphinscheduler_env.sh [load environment variables configs]
|
||||
|
||||
When using shell to commit tasks, DolphinScheduler will export environment variables from `bin/env/dolphinscheduler_env.sh`. The
|
||||
mainly configuration including `JAVA_HOME`, mata database, registry center, and task configuration.
|
||||
|
||||
```bash
|
||||
# JAVA_HOME, will use it to start DolphinScheduler server
|
||||
export JAVA_HOME=${JAVA_HOME:-/opt/soft/java}
|
||||
|
||||
# Database related configuration, set database type, username and password
|
||||
export DATABASE=${DATABASE:-postgresql}
|
||||
export SPRING_PROFILES_ACTIVE=${DATABASE}
|
||||
export SPRING_DATASOURCE_DRIVER_CLASS_NAME
|
||||
export SPRING_DATASOURCE_URL
|
||||
export SPRING_DATASOURCE_USERNAME
|
||||
export SPRING_DATASOURCE_PASSWORD
|
||||
|
||||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
|
||||
export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-localhost:2181}
|
||||
|
||||
# Tasks related configurations, need to change the configuration if you use the related tasks.
|
||||
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
|
||||
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
|
||||
export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
|
||||
export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
|
||||
export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
|
||||
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
|
||||
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
|
||||
export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
|
||||
|
||||
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
|
||||
```
|
||||
|
||||
### Services logback configs
|
||||
|
||||
Services name| logback config name |
|
||||
--|--|
|
||||
API-service logback config |logback-api.xml|
|
||||
master-service logback config|logback-master.xml |
|
||||
worker-service logback config|logback-worker.xml |
|
||||
alert-service logback config|logback-alert.xml |
|
||||
|
|
@ -1,282 +0,0 @@
|
|||
# System Architecture Design
|
||||
|
||||
## System Structure
|
||||
|
||||
### System Architecture Diagram
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/architecture-1.3.0.jpg" alt="System architecture diagram" width="70%" />
|
||||
<p align="center">
|
||||
<em>System architecture diagram</em>
|
||||
</p>
|
||||
</p>
|
||||
|
||||
### Start Process Activity Diagram
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/process-start-flow-1.3.0.png" alt="Start process activity diagram" width="70%" />
|
||||
<p align="center">
|
||||
<em>Start process activity diagram</em>
|
||||
</p>
|
||||
</p>
|
||||
|
||||
### Architecture Description
|
||||
|
||||
* **MasterServer**
|
||||
|
||||
MasterServer adopts a distributed and decentralized design concept. MasterServer is mainly responsible for DAG task segmentation, task submission monitoring, and monitoring the health status of other MasterServer and WorkerServer at the same time.
|
||||
When the MasterServer service starts, register a temporary node with ZooKeeper, and perform fault tolerance by monitoring changes in the temporary node of ZooKeeper.
|
||||
MasterServer provides monitoring services based on netty.
|
||||
|
||||
#### The Service Mainly Includes:
|
||||
|
||||
- **Distributed Quartz** distributed scheduling component, which is mainly responsible for the start and stop operations of schedule tasks. When Quartz starts the task, there will be a thread pool inside the Master responsible for the follow-up operation of the processing task.
|
||||
|
||||
- **MasterSchedulerThread** is a scanning thread that regularly scans the **command** table in the database and runs different business operations according to different **command types**.
|
||||
|
||||
- **MasterExecThread** is mainly responsible for DAG task segmentation, task submission monitoring, and logical processing to different command types.
|
||||
|
||||
- **MasterTaskExecThread** is mainly responsible for the persistence to tasks.
|
||||
|
||||
* **WorkerServer**
|
||||
|
||||
WorkerServer also adopts a distributed and decentralized design concept. WorkerServer is mainly responsible for task execution and providing log services.
|
||||
|
||||
When the WorkerServer service starts, register a temporary node with ZooKeeper and maintain a heartbeat.
|
||||
Server provides monitoring services based on netty.
|
||||
|
||||
#### The Service Mainly Includes:
|
||||
|
||||
- **Fetch TaskThread** is mainly responsible for continuously getting tasks from the **Task Queue**, and calling **TaskScheduleThread** corresponding executor according to different task types.
|
||||
|
||||
* **ZooKeeper**
|
||||
|
||||
ZooKeeper service, MasterServer and WorkerServer nodes in the system all use ZooKeeper for cluster management and fault tolerance. In addition, the system implements event monitoring and distributed locks based on ZooKeeper.
|
||||
|
||||
We have also implemented queues based on Redis, but we hope DolphinScheduler depends on as few components as possible, so we finally removed the Redis implementation.
|
||||
|
||||
* **Task Queue**
|
||||
|
||||
Provide task queue operation, the current queue is also implement base on ZooKeeper. Due to little information stored in the queue, there is no need to worry about excessive data in the queue. In fact, we have tested the millions of data storage in queues, which has no impact on system stability and performance.
|
||||
|
||||
* **Alert**
|
||||
|
||||
Provide alarm related interface, the interface mainly includes **alarm** two types of alarm data storage, query and notification functions. Among them, there are **email notification** and **SNMP (not yet implemented)**.
|
||||
|
||||
* **API**
|
||||
|
||||
The API interface layer is mainly responsible for processing requests from the front-end UI layer. The service uniformly provides RESTful APIs to provide request services to external.
|
||||
Interfaces include workflow creation, definition, query, modification, release, logoff, manual start, stop, pause, resume, start execution from specific node, etc.
|
||||
|
||||
* **UI**
|
||||
|
||||
The front-end page of the system provides various visual operation interfaces of the system, see more at [Introduction to Functions](../guide/homepage.md) section.
|
||||
|
||||
### Architecture Design Ideas
|
||||
|
||||
#### Decentralization VS Centralization
|
||||
|
||||
##### Centralized Thinking
|
||||
|
||||
The centralized design concept is relatively simple. The nodes in the distributed cluster are roughly divided into two roles according to responsibilities:
|
||||
<p align="center">
|
||||
<img src="https://analysys.github.io/easyscheduler_docs_cn/images/master_slave.png" alt="master-slave character" width="50%" />
|
||||
</p>
|
||||
|
||||
- The role of the master is mainly responsible for task distribution and monitoring the health status of the slave, and can dynamically balance the task to the slave, so that the slave node won't be in a "busy dead" or "idle dead" state.
|
||||
- The role of Worker is mainly responsible for task execution and heartbeat maintenance to the Master, so that Master can assign tasks to Slave.
|
||||
|
||||
Problems in centralized thought design:
|
||||
|
||||
- Once there is a problem with the Master, the team grow aimless without commander and the entire cluster collapse. In order to solve this problem, most of the Master and Slave architecture models adopt the design scheme of active and standby Master, which can be hot standby or cold standby, or automatic switching or manual switching. More and more new systems are beginning to have ability to automatically elect and switch Master to improve the availability of the system.
|
||||
- Another problem is that if the Scheduler is on the Master, although it can support different tasks in a DAG running on different machines, it will cause the Master to be overloaded. If the Scheduler is on the slave, all tasks in a DAG can only submit jobs on a certain machine. When there are more parallel tasks, the pressure on the slave may be greater.
|
||||
|
||||
##### Decentralized
|
||||
|
||||
<p align="center">
|
||||
<img src="https://analysys.github.io/easyscheduler_docs_cn/images/decentralization.png" alt="Decentralization" width="50%" />
|
||||
</p>
|
||||
|
||||
- In the decentralized design, there is usually no concept of Master or Slave. All roles are the same, the status is equal, the global Internet is a typical decentralized distributed system. Any node connected to the network goes down, will only affect a small range of functions.
|
||||
- The core design of decentralized design is that there is no distinct "manager" different from other nodes in the entire distributed system, so there is no single point failure. However, because there is no "manager" node, each node needs to communicate with other nodes to obtain the necessary machine information, and the unreliability of distributed system communication greatly increases the difficulty to implement the above functions.
|
||||
- In fact, truly decentralized distributed systems are rare. Instead, dynamic centralized distributed systems are constantly pouring out. Under this architecture, the managers in the cluster are dynamically selected, rather than preset, and when the cluster fails, the nodes of the cluster will automatically hold "meetings" to elect new "managers" To preside over the work. The most typical case is Etcd implemented by ZooKeeper and Go language.
|
||||
- The decentralization of DolphinScheduler is that the Master and Worker register in ZooKeeper, for implement the centerless feature to Master cluster and Worker cluster. Use the ZooKeeper distributed lock to elect one of the Master or Worker as the "manager" to perform the task.
|
||||
|
||||
#### Distributed Lock Practice
|
||||
|
||||
DolphinScheduler uses ZooKeeper distributed lock to implement only one Master executes Scheduler at the same time, or only one Worker executes the submission of tasks.
|
||||
1. The following shows the core process algorithm for acquiring distributed locks:
|
||||
<p align="center">
|
||||
<img src="https://analysys.github.io/easyscheduler_docs_cn/images/distributed_lock.png" alt="Obtain distributed lock process" width="50%" />
|
||||
</p>
|
||||
|
||||
2. Flow diagram of implementation of Scheduler thread distributed lock in DolphinScheduler:
|
||||
<p align="center">
|
||||
<img src="/img/distributed_lock_procss.png" alt="Obtain distributed lock process" width="50%" />
|
||||
</p>
|
||||
|
||||
|
||||
#### Insufficient Thread Loop Waiting Problem
|
||||
|
||||
- If there is no sub-process in a DAG, when the number of data in the Command is greater than the threshold set by the thread pool, the process directly waits or fails.
|
||||
- If a large DAG nests many sub-processes, there will produce a "dead" state as the following figure:
|
||||
|
||||
<p align="center">
|
||||
<img src="https://analysys.github.io/easyscheduler_docs_cn/images/lack_thread.png" alt="Insufficient threads waiting loop problem" width="50%" />
|
||||
</p>
|
||||
In the above figure, MainFlowThread waits for the end of SubFlowThread1, SubFlowThread1 waits for the end of SubFlowThread2, SubFlowThread2 waits for the end of SubFlowThread3, and SubFlowThread3 waits for a new thread in the thread pool, then the entire DAG process cannot finish, and the threads cannot be released. In this situation, the state of the child-parent process loop waiting is formed. At this moment, unless a new Master is started and add threads to break such a "stalemate", the scheduling cluster will no longer use.
|
||||
|
||||
It seems a bit unsatisfactory to start a new Master to break the deadlock, so we proposed the following three solutions to reduce this risk:
|
||||
|
||||
1. Calculate the sum of all Master threads, and then calculate the number of threads required for each DAG, that is, pre-calculate before the DAG process executes. Because it is a multi-master thread pool, it is unlikely to obtain the total number of threads in real time.
|
||||
2. Judge whether the single-master thread pool is full, let the thread fail directly when fulfilled.
|
||||
3. Add a Command type with insufficient resources. If the thread pool is insufficient, suspend the main process. In this way, there are new threads in the thread pool, which can make the process suspended by insufficient resources wake up to execute again.
|
||||
|
||||
Note: The Master Scheduler thread executes by FIFO when acquiring the Command.
|
||||
|
||||
So we choose the third way to solve the problem of insufficient threads.
|
||||
|
||||
|
||||
#### Fault-Tolerant Design
|
||||
|
||||
Fault tolerance divides into service downtime fault tolerance and task retry, and service downtime fault tolerance divides into master fault tolerance and worker fault tolerance.
|
||||
|
||||
##### Downtime Fault Tolerance
|
||||
|
||||
The service fault-tolerance design relies on ZooKeeper's Watcher mechanism, and the implementation principle shows in the figure:
|
||||
|
||||
<p align="center">
|
||||
<img src="https://analysys.github.io/easyscheduler_docs_cn/images/fault-tolerant.png" alt="DolphinScheduler fault-tolerant design" width="40%" />
|
||||
</p>
|
||||
Among them, the Master monitors the directories of other Masters and Workers. If the remove event is triggered, perform fault tolerance of the process instance or task instance according to the specific business logic.
|
||||
|
||||
|
||||
|
||||
- Master fault tolerance：
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/failover-master.jpg" alt="failover-master" width="50%" />
|
||||
</p>
|
||||
|
||||
Fault tolerance range: From the perspective of host, the fault tolerance range of Master includes: own host and node host that does not exist in the registry, and the entire process of fault tolerance will be locked;
|
||||
|
||||
Fault-tolerant content: Master's fault-tolerant content includes: fault-tolerant process instances and task instances. Before fault-tolerant, compares the start time of the instance with the server start-up time, and skips fault-tolerance if after the server start time;
|
||||
|
||||
Fault-tolerant post-processing: After the fault tolerance of ZooKeeper Master completed, then re-schedule by the Scheduler thread in DolphinScheduler, traverses the DAG to find the "running" and "submit successful" tasks. Monitor the status of its task instances for the "running" tasks, and for the "commits successful" tasks, it is necessary to find out whether the task queue already exists. If exists, monitor the status of the task instance. Otherwise, resubmit the task instance.
|
||||
|
||||
- Worker fault tolerance：
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/failover-worker.jpg" alt="failover-worker" width="50%" />
|
||||
</p>
|
||||
|
||||
Fault tolerance range: From the perspective of process instance, each Master is only responsible for fault tolerance of its own process instance; it will lock only when `handleDeadServer`;
|
||||
|
||||
Fault-tolerant content: When sending the remove event of the Worker node, the Master only fault-tolerant task instances. Before fault-tolerant, compares the start time of the instance with the server start-up time, and skips fault-tolerance if after the server start time;
|
||||
|
||||
Fault-tolerant post-processing: Once the Master Scheduler thread finds that the task instance is in the "fault-tolerant" state, it takes over the task and resubmits it.
|
||||
|
||||
Note: Due to "network jitter", the node may lose heartbeat with ZooKeeper in a short period of time, and the node's remove event may occur. For this situation, we use the simplest way, that is, once the node and ZooKeeper timeout connection occurs, then directly stop the Master or Worker service.
|
||||
|
||||
##### Task Failed and Try Again
|
||||
|
||||
Here we must first distinguish the concepts of task failure retry, process failure recovery, and process failure re-run:
|
||||
|
||||
- Task failure retry is at the task level and is automatically performed by the schedule system. For example, if a Shell task sets to retry for 3 times, it will try to run it again up to 3 times after the Shell task fails.
|
||||
- Process failure recovery is at the process level and is performed manually. Recovery can only perform **from the failed node** or **from the current node**.
|
||||
- Process failure re-run is also at the process level and is performed manually, re-run perform from the beginning node.
|
||||
|
||||
Next to the main point, we divide the task nodes in the workflow into two types.
|
||||
|
||||
- One is a business node, which corresponds to an actual script or process command, such as shell node, MR node, Spark node, and dependent node.
|
||||
|
||||
- Another is a logical node, which does not operate actual script or process command, but only logical processing to the entire process flow, such as sub-process sections.
|
||||
|
||||
Each **business node** can configure the number of failed retries. When the task node fails, it will automatically retry until it succeeds or exceeds the retry times. **Logical node** failure retry is not supported, but the tasks in the logical node support.
|
||||
|
||||
If there is a task failure in the workflow that reaches the maximum retry times, the workflow will fail and stop, and the failed workflow can be manually re-run or process recovery operations.
|
||||
|
||||
#### Task Priority Design
|
||||
|
||||
In the early schedule design, if there is no priority design and use the fair scheduling, the task submitted first may complete at the same time with the task submitted later, thus invalid the priority of process or task. So we have re-designed this, and the following is our current design:
|
||||
|
||||
- According to **the priority of different process instances** prior over **priority of the same process instance** prior over **priority of tasks within the same process** prior over **tasks within the same process**, process task submission order from highest to Lowest.
|
||||
- The specific implementation is to parse the priority according to the JSON of the task instance, and then save the **process instance priority_process instance id_task priority_task id** information to the ZooKeeper task queue. When obtain from the task queue, we can get the highest priority task by comparing string.
|
||||
|
||||
- The priority of the process definition is to consider that some processes need to process before other processes. Configure the priority when the process starts or schedules. There are 5 levels in total, which are HIGHEST, HIGH, MEDIUM, LOW, and LOWEST. As shown below
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/10797147/146744784-eb351b14-c94a-4ed6-8ba4-5132c2a3d116.png" alt="Process priority configuration" width="40%" />
|
||||
</p>
|
||||
|
||||
- The priority of the task is also divides into 5 levels, ordered by HIGHEST, HIGH, MEDIUM, LOW, LOWEST. As shown below:
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/10797147/146744830-5eac611f-5933-4f53-a0c6-31613c283708.png" alt="Task priority configuration" width="35%" />
|
||||
</p>
|
||||
|
||||
#### Logback and Netty Implement Log Access
|
||||
|
||||
- Since Web (UI) and Worker are not always on the same machine, to view the log cannot be like querying a local file. There are two options:
|
||||
- Put logs on the ES search engine.
|
||||
- Obtain remote log information through netty communication.
|
||||
|
||||
- In consideration of the lightness of DolphinScheduler as much as possible, so choose gRPC to achieve remote access to log information.
|
||||
|
||||
<p align="center">
|
||||
<img src="https://analysys.github.io/easyscheduler_docs_cn/images/grpc.png" alt="grpc remote access" width="50%" />
|
||||
</p>
|
||||
|
||||
- We use the customized FileAppender and Filter functions from Logback to implement each task instance generates one log file.
|
||||
- The following is the FileAppender implementation：
|
||||
|
||||
```java
|
||||
/**
|
||||
* task log appender
|
||||
*/
|
||||
public class TaskLogAppender extends FileAppender<ILoggingEvent> {
|
||||
|
||||
...
|
||||
|
||||
@Override
|
||||
protected void append(ILoggingEvent event) {
|
||||
|
||||
if (currentlyActiveFile == null){
|
||||
currentlyActiveFile = getFile();
|
||||
}
|
||||
String activeFile = currentlyActiveFile;
|
||||
// thread name： taskThreadName-processDefineId_processInstanceId_taskInstanceId
|
||||
String threadName = event.getThreadName();
|
||||
String[] threadNameArr = threadName.split("-");
|
||||
// logId = processDefineId_processInstanceId_taskInstanceId
|
||||
String logId = threadNameArr[1];
|
||||
...
|
||||
super.subAppend(event);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Generate logs in the form of /process definition id /process instance id /task instance id.log
|
||||
|
||||
- Filter to match the thread name starting with TaskLogInfo:
|
||||
|
||||
- The following shows the TaskLogFilter implementation:
|
||||
|
||||
```java
|
||||
/**
|
||||
* task log filter
|
||||
*/
|
||||
public class TaskLogFilter extends Filter<ILoggingEvent> {
|
||||
|
||||
@Override
|
||||
public FilterReply decide(ILoggingEvent event) {
|
||||
if (event.getThreadName().startsWith("TaskLogInfo-")){
|
||||
return FilterReply.ACCEPT;
|
||||
}
|
||||
return FilterReply.DENY;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Sum Up
|
||||
|
||||
From the perspective of scheduling, this article preliminarily introduces the architecture principles and implementation ideas of the big data distributed workflow scheduling system: DolphinScheduler. To be continued.
|
||||
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
# Load Balance
|
||||
|
||||
Load balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.
|
||||
|
||||
## DolphinScheduler-Worker Load Balancing Algorithms
|
||||
|
||||
DolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:
|
||||
|
||||
- Weighted random (random)
|
||||
|
||||
- Smoothing polling (round-robin)
|
||||
|
||||
- Linear load (lower weight)
|
||||
|
||||
The default configuration is the linear load.
|
||||
|
||||
As the routing sets on the client side, the master service, you can change master.host.selector in master.properties to configure the algorithm.
|
||||
|
||||
e.g. master.host.selector=random (case-insensitive)
|
||||
|
||||
## Worker Load Balancing Configuration
|
||||
|
||||
The configuration file is worker.properties
|
||||
|
||||
### Weight
|
||||
|
||||
All the load algorithms above are weighted based on weights, which affect the routing outcome. You can set different weights for different machines by modifying the `worker.weight` value.
|
||||
|
||||
### Preheating
|
||||
|
||||
Consider JIT optimization, worker runs at low power for a period of time after startup, so that it can gradually reach its optimal state, a process we call preheating. If you are interested, you can read some articles about JIT.
|
||||
|
||||
So the worker gradually reaches its maximum weight with time after starts up ( by default ten minutes, there is no configuration about the pre-heating duration, it's recommend to submit a PR if have needs to change the duration).
|
||||
|
||||
## Load Balancing Algorithm in Details
|
||||
|
||||
### Random (Weighted)
|
||||
|
||||
This algorithm is relatively simple, select a worker by random (the weight affects its weighting).
|
||||
|
||||
### Smoothed Polling (Weighted)
|
||||
|
||||
An obvious drawback of the weighted polling algorithm, which is under special weights circumstance, weighted polling scheduling generates an imbalanced sequence of instances, and this unsmooth load may cause some instances to experience transient high loads, leading to a risk of system crash. To address this scheduling flaw, we provide a smooth weighted polling algorithm.
|
||||
|
||||
Each worker has two weights parameters, weight (which remains constant after warm-up is complete) and current_weight (which changes dynamically). For every route, calculate the current_weight + weight and is iterated over all the workers, the weight of all the workers sum up and count as total_weight, then the worker with the largest current_weight is selected as the worker for this task. By meantime, set worker's current_weight-total_weight.
|
||||
|
||||
### Linear Weighting (Default Algorithm)
|
||||
|
||||
This algorithm reports its own load information to the registry at regular intervals. Make decision on two main pieces of information:
|
||||
|
||||
- load average (default is the number of CPU cores * 2)
|
||||
- available physical memory (default is 0.3, in G)
|
||||
|
||||
If either of these is lower than the configured item, then this worker will not participate in the load. (no traffic will be allocated)
|
||||
|
||||
You can customise the configuration by changing the following properties in worker.properties
|
||||
|
||||
- worker.max.cpuload.avg=-1 (worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2)
|
||||
- worker.reserved.memory=0.3 (worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G)
|
||||
|
|
@ -1,193 +0,0 @@
|
|||
# MetaData
|
||||
|
||||
## DolphinScheduler DB Table Overview
|
||||
|
||||
| Table Name | Comment |
|
||||
| :---: | :---: |
|
||||
| t_ds_access_token | token for access DolphinScheduler backend |
|
||||
| t_ds_alert | alert detail |
|
||||
| t_ds_alertgroup | alert group |
|
||||
| t_ds_command | command detail |
|
||||
| t_ds_datasource | data source |
|
||||
| t_ds_error_command | error command detail |
|
||||
| t_ds_process_definition | process definition |
|
||||
| t_ds_process_instance | process instance |
|
||||
| t_ds_project | project |
|
||||
| t_ds_queue | queue |
|
||||
| t_ds_relation_datasource_user | datasource related to user |
|
||||
| t_ds_relation_process_instance | sub process |
|
||||
| t_ds_relation_project_user | project related to user |
|
||||
| t_ds_relation_resources_user | resource related to user |
|
||||
| t_ds_relation_udfs_user | UDF functions related to user |
|
||||
| t_ds_relation_user_alertgroup | alert group related to user |
|
||||
| t_ds_resources | resoruce center file |
|
||||
| t_ds_schedules | process definition schedule |
|
||||
| t_ds_session | user login session |
|
||||
| t_ds_task_instance | task instance |
|
||||
| t_ds_tenant | tenant |
|
||||
| t_ds_udfs | UDF resource |
|
||||
| t_ds_user | user detail |
|
||||
| t_ds_version | DolphinScheduler version |
|
||||
|
||||
|
||||
---
|
||||
|
||||
## E-R Diagram
|
||||
|
||||
### User Queue DataSource
|
||||
|
||||
|
||||
|
||||
- One tenant can own Multiple users.
|
||||
- The queue field in the t_ds_user table stores the queue_name information in the t_ds_queue table, t_ds_tenant stores queue information using queue_id column. During the execution of the process definition, the user queue has the highest priority. If the user queue is null, use the tenant queue.
|
||||
- The user_id field in the t_ds_datasource table shows the user who create the data source. The user_id in t_ds_relation_datasource_user shows the user who has permission to the data source.
|
||||
|
||||
### Project Resource Alert
|
||||
|
||||
|
||||
|
||||
- User can have multiple projects, user project authorization completes the relationship binding using project_id and user_id in t_ds_relation_project_user table.
|
||||
- The user_id in the t_ds_projcet table represents the user who create the project, and the user_id in the t_ds_relation_project_user table represents users who have permission to the project.
|
||||
- The user_id in the t_ds_resources table represents the user who create the resource, and the user_id in t_ds_relation_resources_user represents the user who has permissions to the resource.
|
||||
- The user_id in the t_ds_udfs table represents the user who create the UDF, and the user_id in the t_ds_relation_udfs_user table represents a user who has permission to the UDF.
|
||||
|
||||
### Command Process Task
|
||||
|
||||
<br />
|
||||
|
||||
- A project has multiple process definitions, a process definition can generate multiple process instances, and a process instance can generate multiple task instances.
|
||||
- The t_ds_schedulers table stores the specified time schedule information for process definition.
|
||||
- The data stored in the t_ds_relation_process_instance table is used to deal with the sub-processes of a process definition, parent_process_instance_id field represents the id of the main process instance who contains child processes, process_instance_id field represents the id of the sub-process instance, parent_task_instance_id field represents the task instance id of the sub-process node.
|
||||
- The process instance table and the task instance table correspond to the t_ds_process_instance table and the t_ds_task_instance table, respectively.
|
||||
|
||||
---
|
||||
|
||||
## Core Table Schema
|
||||
|
||||
### t_ds_process_definition
|
||||
|
||||
| Field | Type | Comment |
|
||||
| --- | --- | --- |
|
||||
| id | int | primary key |
|
||||
| name | varchar | process definition name |
|
||||
| version | int | process definition version |
|
||||
| release_state | tinyint | process definition release state：0:offline,1:online |
|
||||
| project_id | int | project id |
|
||||
| user_id | int | process definition creator id |
|
||||
| process_definition_json | longtext | process definition JSON content |
|
||||
| description | text | process definition description |
|
||||
| global_params | text | global parameters |
|
||||
| flag | tinyint | whether process available: 0 not available, 1 available |
|
||||
| locations | text | Node location information |
|
||||
| connects | text | Node connection information |
|
||||
| receivers | text | receivers |
|
||||
| receivers_cc | text | carbon copy list |
|
||||
| create_time | datetime | create time |
|
||||
| timeout | int | timeout |
|
||||
| tenant_id | int | tenant id |
|
||||
| update_time | datetime | update time |
|
||||
| modify_by | varchar | define user modify the process |
|
||||
| resource_ids | varchar | resource id set |
|
||||
|
||||
### t_ds_process_instance
|
||||
|
||||
| Field | Type | Comment |
|
||||
| --- | --- | --- |
|
||||
| id | int | primary key |
|
||||
| name | varchar | process instance name |
|
||||
| process_definition_id | int | process definition id |
|
||||
| state | tinyint | process instance Status: 0 successful commit, 1 running, 2 prepare to pause, 3 pause, 4 prepare to stop, 5 stop, 6 fail, 7 succeed, 8 need fault tolerance, 9 kill, 10 wait for thread, 11 wait for dependency to complete |
|
||||
| recovery | tinyint | process instance failover flag：0: normal,1: failover instance needs restart |
|
||||
| start_time | datetime | process instance start time |
|
||||
| end_time | datetime | process instance end time |
|
||||
| run_times | int | process instance run times |
|
||||
| host | varchar | process instance host |
|
||||
| command_type | tinyint | command type：0 start ,1 start from the current node,2 resume a fault-tolerant process,3 resume from pause process, 4 execute from the failed node,5 complement, 6 dispatch, 7 re-run, 8 pause, 9 stop, 10 resume waiting thread |
|
||||
| command_param | text | JSON command parameters |
|
||||
| task_depend_type | tinyint | node dependency type: 0 current node, 1 forward, 2 backward |
|
||||
| max_try_times | tinyint | max try times |
|
||||
| failure_strategy | tinyint | failure strategy, 0: end the process when node failed,1: continue run the other nodes when failed |
|
||||
| warning_type | tinyint | warning type 0: no warning, 1: warning if process success, 2: warning if process failed, 3: warning whatever results |
|
||||
| warning_group_id | int | warning group id |
|
||||
| schedule_time | datetime | schedule time |
|
||||
| command_start_time | datetime | command start time |
|
||||
| global_params | text | global parameters |
|
||||
| process_instance_json | longtext | process instance JSON |
|
||||
| flag | tinyint | whether process instance is available: 0 not available, 1 available |
|
||||
| update_time | timestamp | update time |
|
||||
| is_sub_process | int | whether the process is sub process: 1 sub-process, 0 not sub-process |
|
||||
| executor_id | int | executor id |
|
||||
| locations | text | node location information |
|
||||
| connects | text | node connection information |
|
||||
| history_cmd | text | history commands, record all the commands to a instance |
|
||||
| dependence_schedule_times | text | depend schedule estimate time |
|
||||
| process_instance_priority | int | process instance priority. 0 highest,1 high,2 medium,3 low,4 lowest |
|
||||
| worker_group | varchar | worker group who assign the task |
|
||||
| timeout | int | timeout |
|
||||
| tenant_id | int | tenant id |
|
||||
|
||||
### t_ds_task_instance
|
||||
|
||||
| Field | Type | Comment |
|
||||
| --- | --- | --- |
|
||||
| id | int | primary key |
|
||||
| name | varchar | task name |
|
||||
| task_type | varchar | task type |
|
||||
| process_definition_id | int | process definition id |
|
||||
| process_instance_id | int | process instance id |
|
||||
| task_json | longtext | task content JSON |
|
||||
| state | tinyint | Status: 0 commit succeeded, 1 running, 2 prepare to pause, 3 pause, 4 prepare to stop, 5 stop, 6 fail, 7 succeed, 8 need fault tolerance, 9 kill, 10 wait for thread, 11 wait for dependency to complete |
|
||||
| submit_time | datetime | task submit time |
|
||||
| start_time | datetime | task start time |
|
||||
| end_time | datetime | task end time |
|
||||
| host | varchar | host of task running on |
|
||||
| execute_path | varchar | task execute path in the host |
|
||||
| log_path | varchar | task log path |
|
||||
| alert_flag | tinyint | whether alert |
|
||||
| retry_times | int | task retry times |
|
||||
| pid | int | pid of task |
|
||||
| app_link | varchar | Yarn app id |
|
||||
| flag | tinyint | task instance is available : 0 not available, 1 available |
|
||||
| retry_interval | int | retry interval when task failed |
|
||||
| max_retry_times | int | max retry times |
|
||||
| task_instance_priority | int | task instance priority:0 highest,1 high,2 medium,3 low,4 lowest |
|
||||
| worker_group | varchar | worker group who assign the task |
|
||||
|
||||
#### t_ds_schedules
|
||||
|
||||
| Field | Type | Comment |
|
||||
| --- | --- | --- |
|
||||
| id | int | primary key |
|
||||
| process_definition_id | int | process definition id |
|
||||
| start_time | datetime | schedule start time |
|
||||
| end_time | datetime | schedule end time |
|
||||
| crontab | varchar | crontab expression |
|
||||
| failure_strategy | tinyint | failure strategy: 0 end,1 continue |
|
||||
| user_id | int | user id |
|
||||
| release_state | tinyint | release status: 0 not yet released,1 released |
|
||||
| warning_type | tinyint | warning type: 0: no warning, 1: warning if process success, 2: warning if process failed, 3: warning whatever results |
|
||||
| warning_group_id | int | warning group id |
|
||||
| process_instance_priority | int | process instance priority:0 highest,1 high,2 medium,3 low,4 lowest |
|
||||
| worker_group | varchar | worker group who assign the task |
|
||||
| create_time | datetime | create time |
|
||||
| update_time | datetime | update time |
|
||||
|
||||
### t_ds_command
|
||||
|
||||
| Field | Type | Comment |
|
||||
| --- | --- | --- |
|
||||
| id | int | primary key |
|
||||
| command_type | tinyint | command type: 0 start workflow, 1 start execution from current node, 2 resume fault-tolerant workflow, 3 resume pause process, 4 start execution from failed node, 5 complement, 6 schedule, 7 re-run, 8 pause, 9 stop, 10 resume waiting thread |
|
||||
| process_definition_id | int | process definition id |
|
||||
| command_param | text | JSON command parameters |
|
||||
| task_depend_type | tinyint | node dependency type: 0 current node, 1 forward, 2 backward |
|
||||
| failure_strategy | tinyint | failed policy: 0 end, 1 continue |
|
||||
| warning_type | tinyint | alarm type: 0 no alarm, 1 alarm if process success, 2: alarm if process failed, 3: warning whatever results |
|
||||
| warning_group_id | int | warning group id |
|
||||
| schedule_time | datetime | schedule time |
|
||||
| start_time | datetime | start time |
|
||||
| executor_id | int | executor id |
|
||||
| dependence | varchar | dependence column |
|
||||
| update_time | datetime | update time |
|
||||
| process_instance_priority | int | process instance priority: 0 highest,1 high,2 medium,3 low,4 lowest |
|
||||
| worker_group_id | int | worker group who assign the task |
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,100 +0,0 @@
|
|||
# API design standard
|
||||
A standardized and unified API is the cornerstone of project design.The API of DolphinScheduler follows the REST ful standard. REST ful is currently the most popular Internet software architecture. It has a clear structure, conforms to standards, is easy to understand and extend.
|
||||
|
||||
This article uses the DolphinScheduler API as an example to explain how to construct a Restful API.
|
||||
|
||||
## 1. URI design
|
||||
REST is "Representational State Transfer".The design of Restful URI is based on resources.The resource corresponds to an entity on the network, for example: a piece of text, a picture, and a service. And each resource corresponds to a URI.
|
||||
|
||||
+ One Kind of Resource: expressed in the plural, such as `task-instances`、`groups` ;
|
||||
+ A Resource: expressed in the singular, or use the ID to represent the corresponding resource, such as `group`、`groups/{groupId}`;
|
||||
+ Sub Resources: Resources under a certain resource, such as `/instances/{instanceId}/tasks`;
|
||||
+ A Sub Resource：`/instances/{instanceId}/tasks/{taskId}`;
|
||||
|
||||
## 2. Method design
|
||||
We need to locate a certain resource by URI, and then use Method or declare actions in the path suffix to reflect the operation of the resource.
|
||||
|
||||
### ① Query - GET
|
||||
Use URI to locate the resource, and use GET to indicate query.
|
||||
|
||||
+ When the URI is a type of resource, it means to query a type of resource. For example, the following example indicates paging query `alter-groups`.
|
||||
```
|
||||
Method: GET
|
||||
/api/dolphinscheduler/alert-groups
|
||||
```
|
||||
|
||||
+ When the URI is a single resource, it means to query this resource. For example, the following example means to query the specified `alter-group`.
|
||||
```
|
||||
Method: GET
|
||||
/api/dolphinscheduler/alter-groups/{id}
|
||||
```
|
||||
|
||||
+ In addition, we can also express query sub-resources based on URI, as follows:
|
||||
```
|
||||
Method: GET
|
||||
/api/dolphinscheduler/projects/{projectId}/tasks
|
||||
```
|
||||
|
||||
**The above examples all represent paging query. If we need to query all data, we need to add `/list` after the URI to distinguish. Do not mix the same API for both paged query and query.**
|
||||
```
|
||||
Method: GET
|
||||
/api/dolphinscheduler/alert-groups/list
|
||||
```
|
||||
|
||||
### ② Create - POST
|
||||
Use URI to locate the resource, use POST to indicate create, and then return the created id to requester.
|
||||
|
||||
+ create an `alter-group`：
|
||||
|
||||
```
|
||||
Method: POST
|
||||
/api/dolphinscheduler/alter-groups
|
||||
```
|
||||
|
||||
+ create sub-resources is also the same as above.
|
||||
```
|
||||
Method: POST
|
||||
/api/dolphinscheduler/alter-groups/{alterGroupId}/tasks
|
||||
```
|
||||
|
||||
### ③ Modify - PUT
|
||||
Use URI to locate the resource, use PUT to indicate modify.
|
||||
+ modify an `alert-group`
|
||||
```
|
||||
Method: PUT
|
||||
/api/dolphinscheduler/alter-groups/{alterGroupId}
|
||||
```
|
||||
|
||||
### ④ Delete -DELETE
|
||||
Use URI to locate the resource, use DELETE to indicate delete.
|
||||
|
||||
+ delete an `alert-group`
|
||||
```
|
||||
Method: DELETE
|
||||
/api/dolphinscheduler/alter-groups/{alterGroupId}
|
||||
```
|
||||
|
||||
+ batch deletion: batch delete the id array，we should use POST. **（Do not use the DELETE method, because the body of the DELETE request has no semantic meaning, and it is possible that some gateways, proxies, and firewalls will directly strip off the request body after receiving the DELETE request.）**
|
||||
```
|
||||
Method: POST
|
||||
/api/dolphinscheduler/alter-groups/batch-delete
|
||||
```
|
||||
|
||||
### ⑤ Others
|
||||
In addition to creating, deleting, modifying and quering, we also locate the corresponding resource through url, and then append operations to it after the path, such as:
|
||||
```
|
||||
/api/dolphinscheduler/alert-groups/verify-name
|
||||
/api/dolphinscheduler/projects/{projectCode}/process-instances/{code}/view-gantt
|
||||
```
|
||||
|
||||
## 3. Parameter design
|
||||
There are two types of parameters, one is request parameter and the other is path parameter. And the parameter must use small hump.
|
||||
|
||||
In the case of paging, if the parameter entered by the user is less than 1, the front end needs to automatically turn to 1, indicating that the first page is requested; When the backend finds that the parameter entered by the user is greater than the total number of pages, it should directly return to the last page.
|
||||
|
||||
## 4. Others design
|
||||
### base URL
|
||||
The URI of the project needs to use `/api/<project_name>` as the base path, so as to identify that these APIs are under this project.
|
||||
```
|
||||
/api/dolphinscheduler
|
||||
```
|
||||
|
|
@ -1,315 +0,0 @@
|
|||
## Architecture Design
|
||||
Before explaining the architecture of the schedule system, let us first understand the common nouns of the schedule system.
|
||||
|
||||
### 1.Noun Interpretation
|
||||
|
||||
**DAG：** Full name Directed Acyclic Graph，referred to as DAG。Tasks in the workflow are assembled in the form of directed acyclic graphs, which are topologically traversed from nodes with zero indegrees of ingress until there are no successor nodes. For example, the following picture:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/architecture-design/dag_examples.png" alt="dag示例" width="80%" />
|
||||
<p align="center">
|
||||
<em>dag example</em>
|
||||
</p>
|
||||
</p>
|
||||
|
||||
**Process definition**: Visualization **DAG** by dragging task nodes and establishing associations of task nodes
|
||||
|
||||
**Process instance**: A process instance is an instantiation of a process definition, which can be generated by manual startup or scheduling. The process definition runs once, a new process instance is generated
|
||||
|
||||
**Task instance**: A task instance is the instantiation of a specific task node when a process instance runs, which indicates the specific task execution status
|
||||
|
||||
**Task type**: Currently supports SHELL, SQL, SUB_PROCESS (sub-process), PROCEDURE, MR, SPARK, PYTHON, DEPENDENT (dependency), and plans to support dynamic plug-in extension, note: the sub-**SUB_PROCESS** is also A separate process definition that can be launched separately
|
||||
|
||||
**Schedule mode** : The system supports timing schedule and manual schedule based on cron expressions. Command type support: start workflow, start execution from current node, resume fault-tolerant workflow, resume pause process, start execution from failed node, complement, timer, rerun, pause, stop, resume waiting thread. Where **recovers the fault-tolerant workflow** and **restores the waiting thread** The two command types are used by the scheduling internal control and cannot be called externally
|
||||
|
||||
**Timed schedule**: The system uses **quartz** distributed scheduler and supports the generation of cron expression visualization
|
||||
|
||||
**Dependency**: The system does not only support **DAG** Simple dependencies between predecessors and successor nodes, but also provides **task dependencies** nodes, support for **custom task dependencies between processes**
|
||||
|
||||
**Priority**: Supports the priority of process instances and task instances. If the process instance and task instance priority are not set, the default is first in, first out.
|
||||
|
||||
**Mail Alert**: Support **SQL Task** Query Result Email Send, Process Instance Run Result Email Alert and Fault Tolerant Alert Notification
|
||||
|
||||
**Failure policy**: For tasks running in parallel, if there are tasks that fail, two failure policy processing methods are provided. **Continue** means that the status of the task is run in parallel until the end of the process failure. **End** means that once a failed task is found, Kill also drops the running parallel task and the process ends.
|
||||
|
||||
**Complement**: Complement historical data, support **interval parallel and serial** two complement methods
|
||||
|
||||
|
||||
|
||||
### 2.System architecture
|
||||
|
||||
#### 2.1 System Architecture Diagram
|
||||
<p align="center">
|
||||
<img src="/img/architecture.jpg" alt="System Architecture Diagram" />
|
||||
<p align="center">
|
||||
<em>System Architecture Diagram</em>
|
||||
</p>
|
||||
</p>
|
||||
|
||||
|
||||
|
||||
#### 2.2 Architectural description
|
||||
|
||||
* **MasterServer**
|
||||
|
||||
MasterServer adopts the distributed non-central design concept. MasterServer is mainly responsible for DAG task split, task submission monitoring, and monitoring the health status of other MasterServer and WorkerServer.
|
||||
When the MasterServer service starts, it registers a temporary node with Zookeeper, and listens to the Zookeeper temporary node state change for fault tolerance processing.
|
||||
|
||||
|
||||
|
||||
##### The service mainly contains:
|
||||
|
||||
- **Distributed Quartz** distributed scheduling component, mainly responsible for the start and stop operation of the scheduled task. When the quartz picks up the task, the master internally has a thread pool to be responsible for the subsequent operations of the task.
|
||||
|
||||
- **MasterSchedulerThread** is a scan thread that periodically scans the **command** table in the database for different business operations based on different **command types**
|
||||
|
||||
- **MasterExecThread** is mainly responsible for DAG task segmentation, task submission monitoring, logic processing of various command types
|
||||
|
||||
- **MasterTaskExecThread** is mainly responsible for task persistence
|
||||
|
||||
|
||||
|
||||
* **WorkerServer**
|
||||
|
||||
- WorkerServer also adopts a distributed, non-central design concept. WorkerServer is mainly responsible for task execution and providing log services. When the WorkerServer service starts, it registers the temporary node with Zookeeper and maintains the heartbeat.
|
||||
|
||||
##### This service contains:
|
||||
|
||||
- **FetchTaskThread** is mainly responsible for continuously receiving tasks from **Task Queue** and calling **TaskScheduleThread** corresponding executors according to different task types.
|
||||
|
||||
- **ZooKeeper**
|
||||
|
||||
The ZooKeeper service, the MasterServer and the WorkerServer nodes in the system all use the ZooKeeper for cluster management and fault tolerance. In addition, the system also performs event monitoring and distributed locking based on ZooKeeper.
|
||||
We have also implemented queues based on Redis, but we hope that DolphinScheduler relies on as few components as possible, so we finally removed the Redis implementation.
|
||||
|
||||
- **Task Queue**
|
||||
|
||||
The task queue operation is provided. Currently, the queue is also implemented based on Zookeeper. Since there is less information stored in the queue, there is no need to worry about too much data in the queue. In fact, we have over-measured a million-level data storage queue, which has no effect on system stability and performance.
|
||||
|
||||
- **Alert**
|
||||
|
||||
Provides alarm-related interfaces. The interfaces mainly include **Alarms**. The storage, query, and notification functions of the two types of alarm data. The notification function has two types: **mail notification** and **SNMP (not yet implemented)**.
|
||||
|
||||
- **API**
|
||||
|
||||
The API interface layer is mainly responsible for processing requests from the front-end UI layer. The service provides a RESTful api to provide request services externally.
|
||||
Interfaces include workflow creation, definition, query, modification, release, offline, manual start, stop, pause, resume, start execution from this node, and more.
|
||||
|
||||
- **UI**
|
||||
|
||||
The front-end page of the system provides various visual operation interfaces of the system. For details, see the [quick start](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/about/introduction.html) section.
|
||||
|
||||
|
||||
|
||||
#### 2.3 Architectural Design Ideas
|
||||
|
||||
##### I. Decentralized vs centralization
|
||||
|
||||
###### Centralization Thought
|
||||
|
||||
The centralized design concept is relatively simple. The nodes in the distributed cluster are divided into two roles according to their roles:
|
||||
|
||||
<p align="center">
|
||||
<img src="https://analysys.github.io/easyscheduler_docs_cn/images/master_slave.png" alt="master-slave role" width="50%" />
|
||||
</p>
|
||||
|
||||
- The role of Master is mainly responsible for task distribution and supervising the health status of Slave. It can dynamically balance the task to Slave, so that the Slave node will not be "busy" or "free".
|
||||
- The role of the Worker is mainly responsible for the execution of the task and maintains the heartbeat with the Master so that the Master can assign tasks to the Slave.
|
||||
|
||||
Problems in the design of centralized :
|
||||
|
||||
- Once the Master has a problem, the group has no leader and the entire cluster will crash. In order to solve this problem, most Master/Slave architecture modes adopt the design scheme of the master and backup masters, which can be hot standby or cold standby, automatic switching or manual switching, and more and more new systems are available. Automatically elects the ability to switch masters to improve system availability.
|
||||
- Another problem is that if the Scheduler is on the Master, although it can support different tasks in one DAG running on different machines, it will generate overload of the Master. If the Scheduler is on the Slave, all tasks in a DAG can only be submitted on one machine. If there are more parallel tasks, the pressure on the Slave may be larger.
|
||||
|
||||
###### Decentralization
|
||||
|
||||
<p align="center"
|
||||
<img src="https://analysys.github.io/easyscheduler_docs_cn/images/decentralization.png" alt="decentralized" width="50%" />
|
||||
</p>
|
||||
|
||||
- In the decentralized design, there is usually no Master/Slave concept, all roles are the same, the status is equal, the global Internet is a typical decentralized distributed system, networked arbitrary node equipment down machine , all will only affect a small range of features.
|
||||
- The core design of decentralized design is that there is no "manager" that is different from other nodes in the entire distributed system, so there is no single point of failure problem. However, since there is no "manager" node, each node needs to communicate with other nodes to get the necessary machine information, and the unreliable line of distributed system communication greatly increases the difficulty of implementing the above functions.
|
||||
- In fact, truly decentralized distributed systems are rare. Instead, dynamic centralized distributed systems are constantly emerging. Under this architecture, the managers in the cluster are dynamically selected, rather than preset, and when the cluster fails, the nodes of the cluster will spontaneously hold "meetings" to elect new "managers". Go to preside over the work. The most typical case is the Etcd implemented in ZooKeeper and Go.
|
||||
|
||||
- Decentralization of DolphinScheduler is the registration of Master/Worker to ZooKeeper. The Master Cluster and the Worker Cluster are not centered, and the Zookeeper distributed lock is used to elect one Master or Worker as the “manager” to perform the task.
|
||||
|
||||
##### 二、Distributed lock practice
|
||||
|
||||
DolphinScheduler uses ZooKeeper distributed locks to implement only one Master to execute the Scheduler at the same time, or only one Worker to perform task submission.
|
||||
|
||||
1. The core process algorithm for obtaining distributed locks is as follows
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/architecture-design/distributed_lock.png" alt="Get Distributed Lock Process" width="70%" />
|
||||
</p>
|
||||
|
||||
2. Scheduler thread distributed lock implementation flow chart in DolphinScheduler:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/architecture-design/distributed_lock_procss.png" alt="Get Distributed Lock Process" />
|
||||
</p>
|
||||
|
||||
##### Third, the thread is insufficient loop waiting problem
|
||||
|
||||
- If there is no subprocess in a DAG, if the number of data in the Command is greater than the threshold set by the thread pool, the direct process waits or fails.
|
||||
- If a large number of sub-processes are nested in a large DAG, the following figure will result in a "dead" state:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/architecture-design/lack_thread.png" alt="Thread is not enough to wait for loop" width="70%" />
|
||||
</p>
|
||||
|
||||
In the above figure, MainFlowThread waits for SubFlowThread1 to end, SubFlowThread1 waits for SubFlowThread2 to end, SubFlowThread2 waits for SubFlowThread3 to end, and SubFlowThread3 waits for a new thread in the thread pool, then the entire DAG process cannot end, and thus the thread cannot be released. This forms the state of the child parent process loop waiting. At this point, the scheduling cluster will no longer be available unless a new Master is started to add threads to break such a "stuck."
|
||||
|
||||
It seems a bit unsatisfactory to start a new Master to break the deadlock, so we proposed the following three options to reduce this risk:
|
||||
|
||||
1. Calculate the sum of the threads of all Masters, and then calculate the number of threads required for each DAG, that is, pre-calculate before the DAG process is executed. Because it is a multi-master thread pool, the total number of threads is unlikely to be obtained in real time.
|
||||
2. Judge the single master thread pool. If the thread pool is full, let the thread fail directly.
|
||||
3. Add a Command type with insufficient resources. If the thread pool is insufficient, the main process will be suspended. This way, the thread pool has a new thread, which can make the process with insufficient resources hang up and wake up again.
|
||||
|
||||
Note: The Master Scheduler thread is FIFO-enabled when it gets the Command.
|
||||
|
||||
So we chose the third way to solve the problem of insufficient threads.
|
||||
|
||||
##### IV. Fault Tolerant Design
|
||||
|
||||
Fault tolerance is divided into service fault tolerance and task retry. Service fault tolerance is divided into two types: Master Fault Tolerance and Worker Fault Tolerance.
|
||||
|
||||
###### 1. Downtime fault tolerance
|
||||
|
||||
Service fault tolerance design relies on ZooKeeper's Watcher mechanism. The implementation principle is as follows:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/architecture-design/fault-tolerant.png" alt="DolphinScheduler Fault Tolerant Design" width="70%" />
|
||||
</p>
|
||||
|
||||
The Master monitors the directories of other Masters and Workers. If the remove event is detected, the process instance is fault-tolerant or the task instance is fault-tolerant according to the specific business logic.
|
||||
|
||||
|
||||
|
||||
- Master fault tolerance flow chart:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/architecture-design/fault-tolerant_master.png" alt="Master Fault Tolerance Flowchart" width="70%" />
|
||||
</p>
|
||||
|
||||
After the ZooKeeper Master is fault-tolerant, it is rescheduled by the Scheduler thread in DolphinScheduler. It traverses the DAG to find the "Running" and "Submit Successful" tasks, and monitors the status of its task instance for the "Running" task. You need to determine whether the Task Queue already exists. If it exists, monitor the status of the task instance. If it does not exist, resubmit the task instance.
|
||||
|
||||
|
||||
|
||||
- Worker fault tolerance flow chart:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/architecture-design/fault-tolerant_worker.png" alt="Worker Fault Tolerance Flowchart" width="70%" />
|
||||
</p>
|
||||
|
||||
Once the Master Scheduler thread finds the task instance as "need to be fault tolerant", it takes over the task and resubmits.
|
||||
|
||||
Note: Because the "network jitter" may cause the node to lose the heartbeat of ZooKeeper in a short time, the node's remove event occurs. In this case, we use the easiest way, that is, once the node has timeout connection with ZooKeeper, it will directly stop the Master or Worker service.
|
||||
|
||||
###### 2. Task failure retry
|
||||
|
||||
Here we must first distinguish between the concept of task failure retry, process failure recovery, and process failure rerun:
|
||||
|
||||
- Task failure Retry is task level, which is automatically performed by the scheduling system. For example, if a shell task sets the number of retries to 3 times, then the shell task will try to run up to 3 times after failing to run.
|
||||
- Process failure recovery is process level, is done manually, recovery can only be performed **from the failed node** or **from the current node**
|
||||
- Process failure rerun is also process level, is done manually, rerun is from the start node
|
||||
|
||||
|
||||
|
||||
Next, let's talk about the topic, we divided the task nodes in the workflow into two types.
|
||||
|
||||
- One is a business node, which corresponds to an actual script or processing statement, such as a Shell node, an MR node, a Spark node, a dependent node, and so on.
|
||||
- There is also a logical node, which does not do the actual script or statement processing, but the logical processing of the entire process flow, such as sub-flow sections.
|
||||
|
||||
Each **service node** can configure the number of failed retries. When the task node fails, it will automatically retry until it succeeds or exceeds the configured number of retries. **Logical node** does not support failed retry. But the tasks in the logical nodes support retry.
|
||||
|
||||
If there is a task failure in the workflow that reaches the maximum number of retries, the workflow will fail to stop, and the failed workflow can be manually rerun or process resumed.
|
||||
|
||||
|
||||
|
||||
##### V. Task priority design
|
||||
|
||||
In the early scheduling design, if there is no priority design and fair scheduling design, it will encounter the situation that the task submitted first may be completed simultaneously with the task submitted subsequently, but the priority of the process or task cannot be set. We have redesigned this, and we are currently designing it as follows:
|
||||
|
||||
- According to **different process instance priority** prioritizes **same process instance priority** prioritizes **task priority within the same process** takes precedence over **same process** commit order from high Go to low for task processing.
|
||||
|
||||
- The specific implementation is to resolve the priority according to the json of the task instance, and then save the **process instance priority _ process instance id_task priority _ task id** information in the ZooKeeper task queue, when obtained from the task queue, Through string comparison, you can get the task that needs to be executed first.
|
||||
|
||||
- The priority of the process definition is that some processes need to be processed before other processes. This can be configured at the start of the process or at the time of scheduled start. There are 5 levels, followed by HIGHEST, HIGH, MEDIUM, LOW, and LOWEST. As shown below
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/architecture-design/process_priority.png" alt="Process Priority Configuration" width="40%" />
|
||||
</p>
|
||||
|
||||
- The priority of the task is also divided into 5 levels, followed by HIGHEST, HIGH, MEDIUM, LOW, and LOWEST. As shown below
|
||||
|
||||
<p align="center">`
|
||||
<img src="/img/architecture-design/task_priority.png" alt="task priority configuration" width="35%" />
|
||||
</p>
|
||||
|
||||
##### VI. Logback and gRPC implement log access
|
||||
|
||||
- Since the Web (UI) and Worker are not necessarily on the same machine, viewing the log is not as it is for querying local files. There are two options:
|
||||
- Put the logs on the ES search engine
|
||||
- Obtain remote log information through gRPC communication
|
||||
- Considering the lightweightness of DolphinScheduler as much as possible, gRPC was chosen to implement remote access log information.
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/architecture-design/grpc.png" alt="grpc remote access" width="50%" />
|
||||
</p>
|
||||
|
||||
- We use a custom Logback FileAppender and Filter function to generate a log file for each task instance.
|
||||
- The main implementation of FileAppender is as follows:
|
||||
|
||||
```java
|
||||
/**
|
||||
* task log appender
|
||||
*/
|
||||
Public class TaskLogAppender extends FileAppender<ILoggingEvent> {
|
||||
|
||||
...
|
||||
|
||||
@Override
|
||||
Protected void append(ILoggingEvent event) {
|
||||
|
||||
If (currentlyActiveFile == null){
|
||||
currentlyActiveFile = getFile();
|
||||
}
|
||||
String activeFile = currentlyActiveFile;
|
||||
// thread name: taskThreadName-processDefineId_processInstanceId_taskInstanceId
|
||||
String threadName = event.getThreadName();
|
||||
String[] threadNameArr = threadName.split("-");
|
||||
// logId = processDefineId_processInstanceId_taskInstanceId
|
||||
String logId = threadNameArr[1];
|
||||
...
|
||||
super.subAppend(event);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Generate a log in the form of /process definition id/process instance id/task instance id.log
|
||||
|
||||
- Filter matches the thread name starting with TaskLogInfo:
|
||||
- TaskLogFilter is implemented as follows:
|
||||
|
||||
```java
|
||||
/**
|
||||
* task log filter
|
||||
*/
|
||||
Public class TaskLogFilter extends Filter<ILoggingEvent> {
|
||||
|
||||
@Override
|
||||
Public FilterReply decide(ILoggingEvent event) {
|
||||
If (event.getThreadName().startsWith("TaskLogInfo-")){
|
||||
Return FilterReply.ACCEPT;
|
||||
}
|
||||
Return FilterReply.DENY;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
|
||||
### summary
|
||||
|
||||
Starting from the scheduling, this paper introduces the architecture principle and implementation ideas of the big data distributed workflow scheduling system-DolphinScheduler. To be continued
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
# Global Parameter development document
|
||||
|
||||
After the user defines the parameter with the direction OUT, it is saved in the localParam of the task.
|
||||
|
||||
## Usage of parameters
|
||||
|
||||
Getting the direct predecessor node `preTasks` of the current `taskInstance` to be created from the DAG, get the `varPool` of `preTasks`, merge this varPool (List) into one `varPool`, and in the merging process, if parameters with the same parameter name are found, they will be handled according to the following logics:
|
||||
|
||||
* If all the values are null, the merged value is null
|
||||
* If one and only one value is non-null, then the merged value is the non-null value
|
||||
* If all the values are not null, it would be the earliest value of the endtime of taskInstance taken by VarPool.
|
||||
|
||||
The direction of all the merged properties is updated to IN during the merge process.
|
||||
|
||||
The result of the merge is saved in taskInstance.varPool.
|
||||
|
||||
The worker receives and parses the varPool into the format of `Map<String,Property>`, where the key of the map is property.prop, which is the parameter name.
|
||||
|
||||
When the processor processes the parameters, it will merge the varPool and localParam and globalParam parameters, and if there are parameters with duplicate names during the merging process, they will be replaced according to the following priorities, with the higher priority being retained and the lower priority being replaced:
|
||||
|
||||
* globalParam: high
|
||||
* varPool: middle
|
||||
* localParam: low
|
||||
|
||||
The parameters are replaced with the corresponding values using regular expressions compared to ${parameter name} before the node content is executed.
|
||||
|
||||
## Parameter setting
|
||||
|
||||
Currently, only SQL and SHELL nodes are supported to get parameters.
|
||||
|
||||
Get the parameter with direction OUT from localParam, and do the following way according to the type of different nodes.
|
||||
|
||||
### SQL node
|
||||
|
||||
The structure returned by the parameter is List<Map<String,String>>, where the elements of List are each row of data, the key of Map is the column name, and the value is the value corresponding to the column.
|
||||
|
||||
* If the SQL statement returns one row of data, match the OUT parameter name based on the OUT parameter name defined by the user when defining the task, or discard it if it does not match.
|
||||
* If the SQL statement returns multiple rows of data, the column names are matched based on the OUT parameter names defined by the user when defining the task of type LIST. All rows of the corresponding column are converted to `List<String>` as the value of this parameter. If there is no match, it is discarded.
|
||||
|
||||
### SHELL node
|
||||
|
||||
The result of the processor execution is returned as `Map<String,String>`.
|
||||
|
||||
The user needs to define `${setValue(key=value)}` in the output when defining the shell script.
|
||||
|
||||
Remove `${setValue()}` when processing parameters, split by "=", with the 0th being the key and the 1st being the value.
|
||||
|
||||
Similarly match the OUT parameter name and key defined by the user when defining the task, and use value as the value of that parameter.
|
||||
|
||||
Return parameter processing
|
||||
|
||||
* The result of acquired Processor is String.
|
||||
* Determine whether the processor is empty or not, and exit if it is empty.
|
||||
* Determine whether the localParam is empty or not, and exit if it is empty.
|
||||
* Get the parameter of localParam which is OUT, and exit if it is empty.
|
||||
* Format String as per appeal format (`List<Map<String,String>>` for SQL, `Map<String,String>>` for shell).
|
||||
|
||||
Assign the parameters with matching values to varPool (List, which contains the original IN's parameters)
|
||||
|
||||
* Format the varPool as json and pass it to master.
|
||||
* The parameters that are OUT would be written into the localParam after the master has received the varPool.
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
# Overview
|
||||
|
||||
<!-- TODO Since the side menu does not support multiple levels, add new page to keep all sub page here -->
|
||||
|
||||
* [Global Parameter](global-parameter.md)
|
||||
* [Switch Task type](task/switch.md)
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
# SWITCH Task development
|
||||
|
||||
Switch task workflow step as follows
|
||||
|
||||
* User-defined expressions and branch information are stored in `taskParams` in `taskdefinition`. When the switch is executed, it will be formatted as `SwitchParameters`
|
||||
* `SwitchTaskExecThread` processes the expressions defined in `switch` from top to bottom, obtains the value of the variable from `varPool`, and parses the expression through `javascript`. If the expression returns true, stop checking and record The order of the expression, here we record as resultConditionLocation. The task of SwitchTaskExecThread is over
|
||||
* After the `switch` task runs, if there is no error (more commonly, the user-defined expression is out of specification or there is a problem with the parameter name), then `MasterExecThread.submitPostNode` will obtain the downstream node of the `DAG` to continue execution.
|
||||
* If it is found in `DagHelper.parsePostNodes` that the current node (the node that has just completed the work) is a `switch` node, the `resultConditionLocation` will be obtained, and all branches except `resultConditionLocation` in the SwitchParameters will be skipped. In this way, only the branches that need to be executed are left
|
||||
|
|
@ -1,101 +0,0 @@
|
|||
### DolphinScheduler Alert SPI main design
|
||||
|
||||
#### DolphinScheduler SPI Design
|
||||
|
||||
DolphinScheduler is undergoing a microkernel + plug-in architecture change. All core capabilities such as tasks, resource storage, registration centers, etc. will be designed as extension points. We hope to use SPI to improve DolphinScheduler’s own flexibility and friendliness (extended sex).
|
||||
|
||||
For alarm-related codes, please refer to the `dolphinscheduler-alert-api` module. This module defines the extension interface of the alarm plug-in and some basic codes. When we need to realize the plug-inization of related functions, it is recommended to read the code of this block first. Of course, it is recommended that you read the document. This will reduce a lot of time, but the document There is a certain degree of lag. When the document is missing, it is recommended to take the source code as the standard (if you are interested, we also welcome you to submit related documents). In addition, we will hardly make changes to the extended interface (excluding new additions) , Unless there is a major structural adjustment, there is an incompatible upgrade version, so the existing documents can generally be satisfied.
|
||||
|
||||
We use the native JAVA-SPI, when you need to extend, in fact, you only need to pay attention to the extension of the `org.apache.dolphinscheduler.alert.api.AlertChannelFactory` interface, the underlying logic such as plug-in loading, and other kernels have been implemented, Which makes our development more focused and simple.
|
||||
|
||||
By the way, we have adopted an excellent front-end component form-create, which supports the generation of front-end UI components based on JSON. If plug-in development involves the front-end, we will use JSON to generate related front-end UI components, org.apache.dolphinscheduler. The parameters of the plug-in are encapsulated in spi.params, which will convert all the relevant parameters into the corresponding JSON, which means that you can complete the drawing of the front-end components by way of Java code (here is mainly the form, we only care Data exchanged between the front and back ends).
|
||||
|
||||
This article mainly focuses on the design and development of Alert.
|
||||
|
||||
#### Main Modules
|
||||
|
||||
If you don't care about its internal design, but simply want to know how to develop your own alarm plug-in, you can skip this content.
|
||||
|
||||
* dolphinscheduler-alert-api
|
||||
|
||||
This module is the core module of ALERT SPI. This module defines the interface of the alarm plug-in extension and some basic codes. The extension plug-in must implement the interface defined by this module: `org.apache.dolphinscheduler.alert.api.AlertChannelFactory`
|
||||
|
||||
* dolphinscheduler-alert-plugins
|
||||
|
||||
This module is currently a plug-in provided by us, and now we have supported dozens of plug-ins, such as Email, DingTalk, Script, etc.
|
||||
|
||||
|
||||
#### Alert SPI Main class information.
|
||||
AlertChannelFactory
|
||||
Alarm plug-in factory interface. All alarm plug-ins need to implement this interface. This interface is used to define the name of the alarm plug-in and the required parameters. The create method is used to create a specific alarm plug-in instance.
|
||||
|
||||
AlertChannel
|
||||
The interface of the alert plug-in. The alert plug-in needs to implement this interface. There is only one method process in this interface. The upper-level alert system will call this method and obtain the return information of the alert through the AlertResult returned by this method.
|
||||
|
||||
AlertData
|
||||
Alarm content information, including id, title, content, log.
|
||||
|
||||
AlertInfo
|
||||
For alarm-related information, when the upper-level system calls an instance of the alarm plug-in, the instance of this class is passed to the specific alarm plug-in through the process method. It contains the alert content AlertData and the parameter information filled in by the front end of the called alert plug-in instance.
|
||||
|
||||
AlertResult
|
||||
The alarm plug-in sends alarm return information.
|
||||
|
||||
org.apache.dolphinscheduler.spi.params
|
||||
This package is a plug-in parameter definition. Our front-end uses the from-create front-end library http://www.form-create.com, which can dynamically generate the front-end UI based on the parameter list json returned by the plug-in definition, so We don't need to care about the front end when we are doing SPI plug-in development.
|
||||
|
||||
Under this package, we currently only encapsulate RadioParam, TextParam, and PasswordParam, which are used to define text type parameters, radio parameters and password type parameters, respectively.
|
||||
|
||||
AbsPluginParams This class is the base class of all parameters, RadioParam these classes all inherit this class. Each DS alert plug-in will return a list of AbsPluginParams in the implementation of AlertChannelFactory.
|
||||
|
||||
The specific design of alert_spi can be seen in the issue: [Alert Plugin Design](https://github.com/apache/incubator-dolphinscheduler/issues/3049)
|
||||
|
||||
#### Alert SPI built-in implementation
|
||||
|
||||
* Email
|
||||
|
||||
Email alert notification
|
||||
|
||||
* DingTalk
|
||||
|
||||
Alert for DingTalk group chat bots
|
||||
|
||||
Related parameter configuration can refer to the DingTalk robot document.
|
||||
|
||||
* EnterpriseWeChat
|
||||
|
||||
EnterpriseWeChat alert notifications
|
||||
|
||||
Related parameter configuration can refer to the EnterpriseWeChat robot document.
|
||||
|
||||
* Script
|
||||
|
||||
We have implemented a shell script for alerting. We will pass the relevant alert parameters to the script and you can implement your alert logic in the shell. This is a good way to interface with internal alerting applications.
|
||||
|
||||
* SMS
|
||||
|
||||
SMS alerts
|
||||
* FeiShu
|
||||
|
||||
FeiShu alert notification
|
||||
* Slack
|
||||
|
||||
Slack alert notification
|
||||
* PagerDuty
|
||||
|
||||
PagerDuty alert notification
|
||||
* WebexTeams
|
||||
|
||||
WebexTeams alert notification
|
||||
|
||||
Related parameter configuration can refer to the WebexTeams document.
|
||||
|
||||
* Telegram
|
||||
|
||||
Telegram alert notification
|
||||
|
||||
Related parameter configuration can refer to the Telegram document.
|
||||
|
||||
* Http
|
||||
|
||||
We have implemented a Http script for alerting. And calling most of the alerting plug-ins end up being Http requests, if we not support your alert plug-in yet, you can use Http to realize your alert login. Also welcome to contribute your common plug-ins to the community :)
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
## DolphinScheduler Datasource SPI main design
|
||||
|
||||
#### How do I use data sources?
|
||||
|
||||
The data source center supports POSTGRESQL, HIVE/IMPALA, SPARK, CLICKHOUSE, SQLSERVER data sources by default.
|
||||
|
||||
If you are using MySQL or ORACLE data source, you need to place the corresponding driver package in the lib directory
|
||||
|
||||
#### How to do Datasource plugin development?
|
||||
|
||||
org.apache.dolphinscheduler.spi.datasource.DataSourceChannel
|
||||
org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory
|
||||
org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient
|
||||
|
||||
1. In the first step, the data source plug-in can implement the above interfaces and inherit the general client. For details, refer to the implementation of data source plug-ins such as sqlserver and mysql. The addition methods of all RDBMS plug-ins are the same.
|
||||
|
||||
2. Add the driver configuration in the data source plug-in pom.xml
|
||||
|
||||
We provide APIs for external access of all data sources in the dolphin scheduler data source API module
|
||||
|
||||
#### **Future plan**
|
||||
|
||||
Support data sources such as kafka, http, files, sparkSQL, FlinkSQL, etc.
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
### DolphinScheduler Registry SPI Extension
|
||||
|
||||
#### how to use?
|
||||
|
||||
Make the following configuration (take zookeeper as an example)
|
||||
|
||||
* Registry plug-in configuration, take Zookeeper as an example (registry.properties)
|
||||
dolphinscheduler-service/src/main/resources/registry.properties
|
||||
```registry.properties
|
||||
registry.plugin.name=zookeeper
|
||||
registry.servers=127.0.0.1:2181
|
||||
```
|
||||
|
||||
For specific configuration information, please refer to the parameter information provided by the specific plug-in, for example zk: `org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConfiguration.java`
|
||||
All configuration information prefixes need to be +registry, such as base.sleep.time.ms, which should be configured in the registry as follows: registry.base.sleep.time.ms=100
|
||||
|
||||
#### How to expand
|
||||
|
||||
`dolphinscheduler-registry-api` defines the standard for implementing plugins. When you need to extend plugins, you only need to implement `org.apache.dolphinscheduler.registry.api.RegistryFactory`.
|
||||
|
||||
Under the `dolphinscheduler-registry-plugin` module is the registry plugin we currently provide.
|
||||
|
||||
#### FAQ
|
||||
|
||||
1: registry connect timeout
|
||||
|
||||
You can increase the relevant timeout parameters.
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
## DolphinScheduler Task SPI extension
|
||||
|
||||
#### How to develop task plugins?
|
||||
|
||||
org.apache.dolphinscheduler.spi.task.TaskChannel
|
||||
|
||||
The plug-in can implement the above interface. It mainly includes creating tasks (task initialization, task running, etc.) and task cancellation. If it is a yarn task, you need to implement org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask.
|
||||
|
||||
We provide APIs for external access to all tasks in the dolphinscheduler-task-api module, while the dolphinscheduler-spi module is the spi general code library, which defines all the plug-in modules, such as the alarm module, the registry module, etc., you can read and view in detail .
|
||||
|
||||
*NOTICE*
|
||||
|
||||
Since the task plug-in involves the front-end page, the front-end SPI has not yet been implemented, so you need to implement the front-end page corresponding to the plug-in separately.
|
||||
|
||||
If there is a class conflict in the task plugin, you can use [Shade-Relocating Classes](https://maven.apache.org/plugins/maven-shade-plugin/) to solve this problem.
|
||||
|
|
@ -1,159 +0,0 @@
|
|||
# DolphinScheduler development
|
||||
|
||||
## Software Requests
|
||||
|
||||
Before setting up the DolphinScheduler development environment, please make sure you have installed the software as below:
|
||||
|
||||
* [Git](https://git-scm.com/downloads): DolphinScheduler version control system
|
||||
* [JDK](https://www.oracle.com/technetwork/java/javase/downloads/index.html): DolphinScheduler backend language
|
||||
* [Maven](http://maven.apache.org/download.cgi): Java Package Management System
|
||||
* [Node](https://nodejs.org/en/download): DolphinScheduler frontend
|
||||
language
|
||||
|
||||
### Clone Git Repository
|
||||
|
||||
Download the git repository through your git management tool, here we use git-core as an example
|
||||
|
||||
```shell
|
||||
mkdir dolphinscheduler
|
||||
cd dolphinscheduler
|
||||
git clone git@github.com:apache/dolphinscheduler.git
|
||||
```
|
||||
### compile source code
|
||||
|
||||
i. If you use MySQL database, pay attention to modify pom.xml in the root project, and change the scope of the mysql-connector-java dependency to compile.
|
||||
|
||||
ii. Run `mvn clean install -Prelease -Dmaven.test.skip=true`
|
||||
|
||||
|
||||
## Notice
|
||||
|
||||
There are two ways to configure the DolphinScheduler development environment, standalone mode and normal mode
|
||||
|
||||
* [Standalone mode](#dolphinscheduler-standalone-quick-start): **Recommended**，more convenient to build development environment, it can cover most scenes.
|
||||
* [Normal mode](#dolphinscheduler-normal-mode): Separate server master, worker, api, which can cover more test environments than standalone, and it is more like production environment in real life.
|
||||
|
||||
## DolphinScheduler Standalone Quick Start
|
||||
|
||||
> **_Note:_** Standalone server only for development and debugging, cause it use H2 Database, Zookeeper Testing Server which may not stable in production
|
||||
> Standalone is only supported in DolphinScheduler 1.3.9 and later versions
|
||||
|
||||
### Git Branch Choose
|
||||
|
||||
Use different Git branch to develop different codes
|
||||
|
||||
* If you want to develop based on a binary package, switch git branch to specific release branch, for example, if you want to develop base on 1.3.9, you should choose branch `1.3.9-release`.
|
||||
* If you want to develop the latest code, choose branch branch `dev`.
|
||||
|
||||
### Start backend server
|
||||
|
||||
Find the class `org.apache.dolphinscheduler.server.StandaloneServer` in Intellij IDEA and clikc run main function to startup.
|
||||
|
||||
### Start frontend server
|
||||
|
||||
Install frontend dependencies and run it
|
||||
|
||||
```shell
|
||||
cd dolphinscheduler-ui
|
||||
npm install
|
||||
npm run start
|
||||
```
|
||||
|
||||
The browser access address http://localhost:12345/dolphinscheduler can login DolphinScheduler UI. The default username and password are **admin/dolphinscheduler123**
|
||||
|
||||
## DolphinScheduler Normal Mode
|
||||
|
||||
### Prepare
|
||||
|
||||
#### zookeeper
|
||||
|
||||
Download [ZooKeeper](https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.6.3), and extract it.
|
||||
|
||||
* Create directory `zkData` and `zkLog`
|
||||
* Go to the zookeeper installation directory, copy configure file `zoo_sample.cfg` to `conf/zoo.cfg`, and change value of dataDir in conf/zoo.cfg to dataDir=./tmp/zookeeper
|
||||
|
||||
```shell
|
||||
# We use path /data/zookeeper/data and /data/zookeeper/datalog here as example
|
||||
dataDir=/data/zookeeper/data
|
||||
dataLogDir=/data/zookeeper/datalog
|
||||
```
|
||||
|
||||
* Run `./bin/zkServer.sh` in terminal by command `./bin/zkServer.sh start`.
|
||||
|
||||
#### Database
|
||||
|
||||
The DolphinScheduler's metadata is stored in relational database. Currently supported MySQL and Postgresql. We use MySQL as an example. Start the database and create a new database named dolphinscheduler as DolphinScheduler metabase
|
||||
|
||||
After creating the new database, run the sql file under `dolphinscheduler/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql` directly in MySQL to complete the database initialization
|
||||
|
||||
#### Start Backend Server
|
||||
|
||||
Following steps will guide how to start the DolphinScheduler backend service
|
||||
|
||||
##### Backend Start Prepare
|
||||
|
||||
* Open project: Use IDE open the project, here we use Intellij IDEA as an example, after opening it will take a while for Intellij IDEA to complete the dependent download
|
||||
* Plugin installation(**Only required for 2.0 or later**)
|
||||
|
||||
* Registry plug-in configuration, take Zookeeper as an example (registry.properties)
|
||||
dolphinscheduler-service/src/main/resources/registry.properties
|
||||
```registry.properties
|
||||
registry.plugin.name=zookeeper
|
||||
registry.servers=127.0.0.1:2181
|
||||
```
|
||||
* File change
|
||||
* If you use MySQL as your metadata database, you need to modify `dolphinscheduler/pom.xml` and change the `scope` of the `mysql-connector-java` dependency to `compile`. This step is not necessary to use PostgreSQL
|
||||
* Modify database configuration, modify the database configuration in the `dolphinscheduler-dao/src/main/resources/application-mysql.yaml`
|
||||
|
||||
|
||||
We here use MySQL with database, username, password named dolphinscheduler as an example
|
||||
```application-mysql.yaml
|
||||
spring:
|
||||
datasource:
|
||||
driver-class-name: com.mysql.jdbc.Driver
|
||||
url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
|
||||
username: ds_user
|
||||
password: dolphinscheduler
|
||||
```
|
||||
|
||||
* Log level: add a line `<appender-ref ref="STDOUT"/>` to the following configuration to enable the log to be displayed on the command line
|
||||
|
||||
`dolphinscheduler-server/src/main/resources/logback-worker.xml`
|
||||
|
||||
`dolphinscheduler-server/src/main/resources/logback-master.xml`
|
||||
|
||||
`dolphinscheduler-api/src/main/resources/logback-api.xml`
|
||||
|
||||
here we add the result after modify as below:
|
||||
|
||||
```diff
|
||||
<root level="INFO">
|
||||
+ <appender-ref ref="STDOUT"/>
|
||||
<appender-ref ref="APILOGFILE"/>
|
||||
<appender-ref ref="SKYWALKING-LOG"/>
|
||||
</root>
|
||||
```
|
||||
|
||||
> **_Note:_** Only DolphinScheduler 2.0 and later versions need to inatall plugin before start server. It not need before version 2.0.
|
||||
|
||||
##### Server start
|
||||
|
||||
There are three services that need to be started, including MasterServer, WorkerServer, ApiApplicationServer.
|
||||
|
||||
* MasterServer：Execute function `main` in the class `org.apache.dolphinscheduler.server.master.MasterServer` by Intellij IDEA, with the configuration *VM Options* `-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false -Dspring.profiles.active=mysql`
|
||||
* WorkerServer：Execute function `main` in the class `org.apache.dolphinscheduler.server.worker.WorkerServer` by Intellij IDEA, with the configuration *VM Options* `-Dlogging.config=classpath:logback-worker.xml -Ddruid.mysql.usePingMethod=false -Dspring.profiles.active=mysql`
|
||||
* ApiApplicationServer：Execute function `main` in the class `org.apache.dolphinscheduler.api.ApiApplicationServer` by Intellij IDEA, with the configuration *VM Options* `-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api,mysql`. After it started, you could find Open API documentation in http://localhost:12345/dolphinscheduler/doc.html
|
||||
|
||||
> The `mysql` in the VM Options `-Dspring.profiles.active=mysql` means specified configuration file
|
||||
|
||||
### Start Frontend Server
|
||||
|
||||
Install frontend dependencies and run it
|
||||
|
||||
```shell
|
||||
cd dolphinscheduler-ui
|
||||
npm install
|
||||
npm run start
|
||||
```
|
||||
|
||||
The browser access address http://localhost:12345/dolphinscheduler can login DolphinScheduler UI. The default username and password are **admin/dolphinscheduler123**
|
||||
|
|
@ -1,197 +0,0 @@
|
|||
# DolphinScheduler E2E Automation Test
|
||||
|
||||
## I. Preparatory knowledge
|
||||
|
||||
### 1. The difference between E2E Test and Unit Test
|
||||
|
||||
E2E, which stands for "End to End", can be translated as "end-to-end" testing. It imitates the user, starting from a certain entry point and progressively performing actions until a certain job is completed. And unit tests are different, the latter usually requires testing parameters, types and parameter values, the number of arguments, the return value, throw an error, and so on, the purpose is to ensure that a specific function to finishing the work is stable and reliable in all cases. Unit testing assumes that if all functions work correctly, then the whole product will work.
|
||||
|
||||
In contrast, E2E test does not emphasize so much the need to cover all usage scenarios, it focuses on whether a complete chain of operations can be completed. For the web front-end, it is also concerned with the layout of the interface and whether the content information meets expectations.
|
||||
|
||||
For example, E2E test of the login page is concerned with whether the user is able to enter and log in normally, and whether the error message is correctly displayed if the login fails. It is not a major concern whether input that is not legal is processed.
|
||||
|
||||
### 2. Selenium test framework
|
||||
|
||||
[Selenium](https://www.selenium.dev) is an open source testing tool for executing automated tests on a web browser. The framework uses WebDriver to transform Web Service commands into browser native calls through the browser's native components to complete operations. In simple words, it simulates the browser and makes selection operations on the elements of the page.
|
||||
|
||||
A WebDriver is an API and protocol which defines a language-neutral interface for controlling the behavior of a web browser. Every browser has a specific WebDriver implementation, called a driver. The driver is the component responsible for delegating to the browser and handling the communication with Selenium and the browser.
|
||||
|
||||
The Selenium framework links all these components together through a user-facing interface that allows transparent work with different browser backends, enabling cross-browser and cross-platform automation.
|
||||
|
||||
## II. E2E Test
|
||||
|
||||
### 1. E2E-Pages
|
||||
|
||||
DolphinScheduler's E2E tests are deployed using docker-compose. The current tests are in standalone mode and are mainly used to check some basic functions such as "add, delete, change and check". For further cluster validation, such as collaboration between services or communication mechanisms between services, refer to `deploy/docker/docker-compose.yml` for configuration.
|
||||
|
||||
For E2E test (the front-end part), the [page model](https://www.selenium.dev/documentation/guidelines/page_object_models/) form is used, mainly to create a corresponding model for each page. The following is an example of a login page.
|
||||
|
||||
```java
|
||||
package org.apache.dolphinscheduler.e2e.pages;
|
||||
|
||||
import org.apache.dolphinscheduler.e2e.pages.common.NavBarPage;
|
||||
import org.apache.dolphinscheduler.e2e.pages.security.TenantPage;
|
||||
|
||||
import org.openqa.selenium.WebElement;
|
||||
import org.openqa.selenium.remote.RemoteWebDriver;
|
||||
import org.openqa.selenium.support.FindBy;
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions;
|
||||
import org.openqa.selenium.support.ui.WebDriverWait;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@Getter
|
||||
public final class LoginPage extends NavBarPage {
|
||||
@FindBy(id = "inputUsername")
|
||||
private WebElement inputUsername;
|
||||
|
||||
@FindBy(id = "inputPassword")
|
||||
private WebElement inputPassword;
|
||||
|
||||
@FindBy(id = "btnLogin")
|
||||
private WebElement buttonLogin;
|
||||
|
||||
public LoginPage(RemoteWebDriver driver) {
|
||||
super(driver);
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
public TenantPage login(String username, String password) {
|
||||
inputUsername().sendKeys(username);
|
||||
inputPassword().sendKeys(password);
|
||||
buttonLogin().click();
|
||||
|
||||
new WebDriverWait(driver, 10)
|
||||
.until(ExpectedConditions.urlContains("/#/security"));
|
||||
|
||||
return new TenantPage(driver);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
During the test process, we only test the elements we need to focus on, not all elements of the page. So on the login page only the username, password and login button elements are declared. The FindBy interface is provided by the Selenium test framework to find the corresponding id or class in a Vue file.
|
||||
|
||||
In addition, during the testing process, the elements are not manipulated directly. The general choice is to package the corresponding methods to achieve the effect of reuse. For example, if you want to log in, you input your username and password through the `public TenantPage login()` method to manipulate the elements you pass in to achieve the effect of logging in. That is, when the user finishes logging in, he or she jumps to the Security Centre (which goes to the Tenant Management page by default).
|
||||
|
||||
The goToTab method is provided in SecurityPage to test the corresponding sidebar jumps, which include TenantPage, UserPage and WorkerGroupPge and QueuePage. These pages are implemented in the same way, to test that the form's input, add and delete buttons return the corresponding pages.
|
||||
|
||||
```java
|
||||
public <T extends SecurityPage.Tab> T goToTab(Class<T> tab) {
|
||||
if (tab == TenantPage.class) {
|
||||
WebElement menuTenantManageElement = new WebDriverWait(driver, 60)
|
||||
.until(ExpectedConditions.elementToBeClickable(menuTenantManage));
|
||||
((JavascriptExecutor)driver).executeScript("arguments[0].click();", menuTenantManageElement);
|
||||
return tab.cast(new TenantPage(driver));
|
||||
}
|
||||
if (tab == UserPage.class) {
|
||||
WebElement menUserManageElement = new WebDriverWait(driver, 60)
|
||||
.until(ExpectedConditions.elementToBeClickable(menUserManage));
|
||||
((JavascriptExecutor)driver).executeScript("arguments[0].click();", menUserManageElement);
|
||||
return tab.cast(new UserPage(driver));
|
||||
}
|
||||
if (tab == WorkerGroupPage.class) {
|
||||
WebElement menWorkerGroupManageElement = new WebDriverWait(driver, 60)
|
||||
.until(ExpectedConditions.elementToBeClickable(menWorkerGroupManage));
|
||||
((JavascriptExecutor)driver).executeScript("arguments[0].click();", menWorkerGroupManageElement);
|
||||
return tab.cast(new WorkerGroupPage(driver));
|
||||
}
|
||||
if (tab == QueuePage.class) {
|
||||
menuQueueManage().click();
|
||||
return tab.cast(new QueuePage(driver));
|
||||
}
|
||||
throw new UnsupportedOperationException("Unknown tab: " + tab.getName());
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
|
||||
For navigation bar options jumping, the goToNav method is provided in `org/apache/dolphinscheduler/e2e/pages/common/NavBarPage.java`. The currently supported pages are: ProjectPage, SecurityPage and ResourcePage.
|
||||
|
||||
```java
|
||||
public <T extends NavBarItem> T goToNav(Class<T> nav) {
|
||||
if (nav == ProjectPage.class) {
|
||||
WebElement projectTabElement = new WebDriverWait(driver, 60)
|
||||
.until(ExpectedConditions.elementToBeClickable(projectTab));
|
||||
((JavascriptExecutor)driver).executeScript("arguments[0].click();", projectTabElement);
|
||||
return nav.cast(new ProjectPage(driver));
|
||||
}
|
||||
|
||||
if (nav == SecurityPage.class) {
|
||||
WebElement securityTabElement = new WebDriverWait(driver, 60)
|
||||
.until(ExpectedConditions.elementToBeClickable(securityTab));
|
||||
((JavascriptExecutor)driver).executeScript("arguments[0].click();", securityTabElement);
|
||||
return nav.cast(new SecurityPage(driver));
|
||||
}
|
||||
|
||||
if (nav == ResourcePage.class) {
|
||||
WebElement resourceTabElement = new WebDriverWait(driver, 60)
|
||||
.until(ExpectedConditions.elementToBeClickable(resourceTab));
|
||||
((JavascriptExecutor)driver).executeScript("arguments[0].click();", resourceTabElement);
|
||||
return nav.cast(new ResourcePage(driver));
|
||||
}
|
||||
|
||||
throw new UnsupportedOperationException("Unknown nav bar");
|
||||
}
|
||||
```
|
||||
|
||||
### E2E-Cases
|
||||
|
||||
Current E2E test cases supported include: File Management, Project Management, Queue Management, Tenant Management, User Management, Worker Group Management and Workflow Test.
|
||||
|
||||
|
||||
|
||||
The following is an example of a tenant management test. As explained earlier, we use docker-compose for deployment, so for each test case, we need to import the corresponding file in the form of an annotation.
|
||||
|
||||
The browser is loaded using the RemoteWebDriver provided with Selenium. Before each test case is started there is some preparation work that needs to be done. For example: logging in the user, jumping to the corresponding page (depending on the specific test case).
|
||||
|
||||
```java
|
||||
@BeforeAll
|
||||
public static void setup() {
|
||||
new LoginPage(browser)
|
||||
.login("admin", "dolphinscheduler123")
|
||||
.goToNav(SecurityPage.class)
|
||||
.goToTab(TenantPage.class)
|
||||
;
|
||||
}
|
||||
```
|
||||
|
||||
When the preparation is complete, it is time for the formal test case writing. We use a form of @Order() annotation for modularity, to confirm the order of the tests. After the tests have been run, assertions are used to determine if the tests were successful, and if the assertion returns true, the tenant creation was successful. The following code can be used as a reference:
|
||||
|
||||
```java
|
||||
@Test
|
||||
@Order(10)
|
||||
void testCreateTenant() {
|
||||
final TenantPage page = new TenantPage(browser);
|
||||
page.create(tenant);
|
||||
|
||||
await().untilAsserted(() -> assertThat(page.tenantList())
|
||||
.as("Tenant list should contain newly-created tenant")
|
||||
.extracting(WebElement::getText)
|
||||
.anyMatch(it -> it.contains(tenant)));
|
||||
}
|
||||
```
|
||||
|
||||
The rest are similar cases and can be understood by referring to the specific source code.
|
||||
|
||||
https://github.com/apache/dolphinscheduler/tree/dev/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases
|
||||
|
||||
## III. Supplements
|
||||
|
||||
When running E2E tests locally, First, you need to start the local service, you can refer to this page:
|
||||
[development-environment-setup](https://dolphinscheduler.apache.org/en-us/development/development-environment-setup.html)
|
||||
|
||||
When running E2E tests locally, the `-Dlocal=true` parameter can be configured to connect locally and facilitate changes to the UI.
|
||||
|
||||
When running E2E tests with `M1` chip, you can use `-Dm1_chip=true` parameter to configure containers supported by
|
||||
`ARM64`.
|
||||
|
||||
|
||||
|
||||
If a connection timeout occurs during a local run, increase the load time to a recommended 30 and above.
|
||||
|
||||
|
||||
|
||||
The test run will be available as an MP4 file.
|
||||
|
||||
|
||||
|
|
@ -1,639 +0,0 @@
|
|||
# Front-end development documentation
|
||||
|
||||
### Technical selection
|
||||
```
|
||||
Vue mvvm framework
|
||||
|
||||
Es6 ECMAScript 6.0
|
||||
|
||||
Ans-ui Analysys-ui
|
||||
|
||||
D3 Visual Library Chart Library
|
||||
|
||||
Jsplumb connection plugin library
|
||||
|
||||
Lodash high performance JavaScript utility library
|
||||
```
|
||||
|
||||
### Development environment
|
||||
|
||||
- #### Node installation
|
||||
Node package download (note version v12.20.2) `https://nodejs.org/download/release/v12.20.2/`
|
||||
|
||||
- #### Front-end project construction
|
||||
Use the command line mode `cd` enter the `dolphinscheduler-ui` project directory and execute `npm install` to pull the project dependency package.
|
||||
|
||||
> If `npm install` is very slow, you can set the taobao mirror
|
||||
|
||||
```
|
||||
npm config set registry http://registry.npm.taobao.org/
|
||||
```
|
||||
|
||||
- Modify `API_BASE` in the file `dolphinscheduler-ui/.env` to interact with the backend:
|
||||
|
||||
```
|
||||
# back end interface address
|
||||
API_BASE = http://127.0.0.1:12345
|
||||
```
|
||||
|
||||
> ##### ! ! ! Special attention here. If the project reports a "node-sass error" error while pulling the dependency package, execute the following command again after execution.
|
||||
|
||||
```bash
|
||||
npm install node-sass --unsafe-perm #Install node-sass dependency separately
|
||||
```
|
||||
|
||||
- #### Development environment operation
|
||||
- `npm start` project development environment (after startup address http://localhost:8888)
|
||||
|
||||
#### Front-end project release
|
||||
|
||||
- `npm run build` project packaging (after packaging, the root directory will create a folder called dist for publishing Nginx online)
|
||||
|
||||
Run the `npm run build` command to generate a package file (dist) package
|
||||
|
||||
Copy it to the corresponding directory of the server (front-end service static page storage directory)
|
||||
|
||||
Visit address` http://localhost:8888`
|
||||
|
||||
#### Start with node and daemon under Linux
|
||||
|
||||
Install pm2 `npm install -g pm2`
|
||||
|
||||
Execute `pm2 start npm -- run dev` to start the project in the project `dolphinscheduler-ui `root directory
|
||||
|
||||
#### command
|
||||
|
||||
- Start `pm2 start npm -- run dev`
|
||||
|
||||
- Stop `pm2 stop npm`
|
||||
|
||||
- delete `pm2 delete npm`
|
||||
|
||||
- Status `pm2 list`
|
||||
|
||||
```
|
||||
|
||||
[root@localhost dolphinscheduler-ui]# pm2 start npm -- run dev
|
||||
[PM2] Applying action restartProcessId on app [npm](ids: 0)
|
||||
[PM2] [npm](0) ✓
|
||||
[PM2] Process successfully started
|
||||
┌──────────┬────┬─────────┬──────┬──────┬────────┬─────────┬────────┬─────┬──────────┬──────┬──────────┐
|
||||
│ App name │ id │ version │ mode │ pid │ status │ restart │ uptime │ cpu │ mem │ user │ watching │
|
||||
├──────────┼────┼─────────┼──────┼──────┼────────┼─────────┼────────┼─────┼──────────┼──────┼──────────┤
|
||||
│ npm │ 0 │ N/A │ fork │ 6168 │ online │ 31 │ 0s │ 0% │ 5.6 MB │ root │ disabled │
|
||||
└──────────┴────┴─────────┴──────┴──────┴────────┴─────────┴────────┴─────┴──────────┴──────┴──────────┘
|
||||
Use `pm2 show <id|name>` to get more details about an app
|
||||
|
||||
```
|
||||
|
||||
### Project directory structure
|
||||
|
||||
`build` some webpack configurations for packaging and development environment projects
|
||||
|
||||
`node_modules` development environment node dependency package
|
||||
|
||||
`src` project required documents
|
||||
|
||||
`src => combo` project third-party resource localization `npm run combo` specific view `build/combo.js`
|
||||
|
||||
`src => font` Font icon library can be added by visiting https://www.iconfont.cn Note: The font library uses its own secondary development to reintroduce its own library `src/sass/common/_font.scss`
|
||||
|
||||
`src => images` public image storage
|
||||
|
||||
`src => js` js/vue
|
||||
|
||||
`src => lib` internal components of the company (company component library can be deleted after open source)
|
||||
|
||||
`src => sass` sass file One page corresponds to a sass file
|
||||
|
||||
`src => view` page file One page corresponds to an html file
|
||||
|
||||
```
|
||||
> Projects are developed using vue single page application (SPA)
|
||||
- All page entry files are in the `src/js/conf/${ corresponding page filename => home} index.js` entry file
|
||||
- The corresponding sass file is in `src/sass/conf/${corresponding page filename => home}/index.scss`
|
||||
- The corresponding html file is in `src/view/${corresponding page filename => home}/index.html`
|
||||
```
|
||||
|
||||
Public module and utill `src/js/module`
|
||||
|
||||
`components` => internal project common components
|
||||
|
||||
`download` => download component
|
||||
|
||||
`echarts` => chart component
|
||||
|
||||
`filter` => filter and vue pipeline
|
||||
|
||||
`i18n` => internationalization
|
||||
|
||||
`io` => io request encapsulation based on axios
|
||||
|
||||
`mixin` => vue mixin public part for disabled operation
|
||||
|
||||
`permissions` => permission operation
|
||||
|
||||
`util` => tool
|
||||
|
||||
### System function module
|
||||
|
||||
Home => `http://localhost:8888/#/home`
|
||||
|
||||
Project Management => `http://localhost:8888/#/projects/list`
|
||||
```
|
||||
| Project Home
|
||||
| Workflow
|
||||
- Workflow definition
|
||||
- Workflow instance
|
||||
- Task instance
|
||||
```
|
||||
|
||||
Resource Management => `http://localhost:8888/#/resource/file`
|
||||
```
|
||||
| File Management
|
||||
| udf Management
|
||||
- Resource Management
|
||||
- Function management
|
||||
```
|
||||
|
||||
Data Source Management => `http://localhost:8888/#/datasource/list`
|
||||
|
||||
Security Center => `http://localhost:8888/#/security/tenant`
|
||||
```
|
||||
| Tenant Management
|
||||
| User Management
|
||||
| Alarm Group Management
|
||||
- master
|
||||
- worker
|
||||
```
|
||||
|
||||
User Center => `http://localhost:8888/#/user/account`
|
||||
|
||||
## Routing and state management
|
||||
|
||||
The project `src/js/conf/home` is divided into
|
||||
|
||||
`pages` => route to page directory
|
||||
```
|
||||
The page file corresponding to the routing address
|
||||
```
|
||||
|
||||
`router` => route management
|
||||
```
|
||||
vue router, the entry file index.js in each page will be registered. Specific operations: https://router.vuejs.org/zh/
|
||||
```
|
||||
|
||||
`store` => status management
|
||||
```
|
||||
The page corresponding to each route has a state management file divided into:
|
||||
|
||||
actions => mapActions => Details：https://vuex.vuejs.org/zh/guide/actions.html
|
||||
|
||||
getters => mapGetters => Details：https://vuex.vuejs.org/zh/guide/getters.html
|
||||
|
||||
index => entrance
|
||||
|
||||
mutations => mapMutations => Details：https://vuex.vuejs.org/zh/guide/mutations.html
|
||||
|
||||
state => mapState => Details：https://vuex.vuejs.org/zh/guide/state.html
|
||||
|
||||
Specific action：https://vuex.vuejs.org/zh/
|
||||
```
|
||||
|
||||
## specification
|
||||
## Vue specification
|
||||
##### 1.Component name
|
||||
The component is named multiple words and is connected with a wire (-) to avoid conflicts with HTML tags and a clearer structure.
|
||||
```
|
||||
// positive example
|
||||
export default {
|
||||
name: 'page-article-item'
|
||||
}
|
||||
```
|
||||
|
||||
##### 2.Component files
|
||||
The internal common component of the `src/js/module/components` project writes the folder name with the same name as the file name. The subcomponents and util tools that are split inside the common component are placed in the internal `_source` folder of the component.
|
||||
```
|
||||
└── components
|
||||
├── header
|
||||
├── header.vue
|
||||
└── _source
|
||||
└── nav.vue
|
||||
└── util.js
|
||||
├── conditions
|
||||
├── conditions.vue
|
||||
└── _source
|
||||
└── search.vue
|
||||
└── util.js
|
||||
```
|
||||
|
||||
##### 3.Prop
|
||||
When you define Prop, you should always name it in camel format (camelCase) and use the connection line (-) when assigning values to the parent component.
|
||||
This follows the characteristics of each language, because it is case-insensitive in HTML tags, and the use of links is more friendly; in JavaScript, the more natural is the hump name.
|
||||
|
||||
```
|
||||
// Vue
|
||||
props: {
|
||||
articleStatus: Boolean
|
||||
}
|
||||
// HTML
|
||||
<article-item :article-status="true"></article-item>
|
||||
```
|
||||
|
||||
The definition of Prop should specify its type, defaults, and validation as much as possible.
|
||||
|
||||
Example：
|
||||
|
||||
```
|
||||
props: {
|
||||
attrM: Number,
|
||||
attrA: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
attrZ: {
|
||||
type: Object,
|
||||
// The default value of the array/object should be returned by a factory function
|
||||
default: function () {
|
||||
return {
|
||||
msg: 'achieve you and me'
|
||||
}
|
||||
}
|
||||
},
|
||||
attrE: {
|
||||
type: String,
|
||||
validator: function (v) {
|
||||
return !(['success', 'fail'].indexOf(v) === -1)
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
##### 4.v-for
|
||||
When performing v-for traversal, you should always bring a key value to make rendering more efficient when updating the DOM.
|
||||
```
|
||||
<ul>
|
||||
<li v-for="item in list" :key="item.id">
|
||||
{{ item.title }}
|
||||
</li>
|
||||
</ul>
|
||||
```
|
||||
|
||||
v-for should be avoided on the same element as v-if (`for example: <li>`) because v-for has a higher priority than v-if. To avoid invalid calculations and rendering, you should try to use v-if Put it on top of the container's parent element.
|
||||
```
|
||||
<ul v-if="showList">
|
||||
<li v-for="item in list" :key="item.id">
|
||||
{{ item.title }}
|
||||
</li>
|
||||
</ul>
|
||||
```
|
||||
|
||||
##### 5.v-if / v-else-if / v-else
|
||||
If the elements in the same set of v-if logic control are logically identical, Vue reuses the same part for more efficient element switching, `such as: value`. In order to avoid the unreasonable effect of multiplexing, you should add key to the same element for identification.
|
||||
```
|
||||
<div v-if="hasData" key="mazey-data">
|
||||
<span>{{ mazeyData }}</span>
|
||||
</div>
|
||||
<div v-else key="mazey-none">
|
||||
<span>no data</span>
|
||||
</div>
|
||||
```
|
||||
|
||||
##### 6.Instruction abbreviation
|
||||
In order to unify the specification, the instruction abbreviation is always used. Using `v-bind`, `v-on` is not bad. Here is only a unified specification.
|
||||
```
|
||||
<input :value="mazeyUser" @click="verifyUser">
|
||||
```
|
||||
|
||||
##### 7.Top-level element order of single file components
|
||||
Styles are packaged in a file, all the styles defined in a single vue file, the same name in other files will also take effect. All will have a top class name before creating a component.
|
||||
Note: The sass plugin has been added to the project, and the sas syntax can be written directly in a single vue file.
|
||||
For uniformity and ease of reading, they should be placed in the order of `<template>`、`<script>`、`<style>`.
|
||||
|
||||
```
|
||||
<template>
|
||||
<div class="test-model">
|
||||
test
|
||||
</div>
|
||||
</template>
|
||||
<script>
|
||||
export default {
|
||||
name: "test",
|
||||
data() {
|
||||
return {}
|
||||
},
|
||||
props: {},
|
||||
methods: {},
|
||||
watch: {},
|
||||
beforeCreate() {
|
||||
},
|
||||
created() {
|
||||
},
|
||||
beforeMount() {
|
||||
},
|
||||
mounted() {
|
||||
},
|
||||
beforeUpdate() {
|
||||
},
|
||||
updated() {
|
||||
},
|
||||
beforeDestroy() {
|
||||
},
|
||||
destroyed() {
|
||||
},
|
||||
computed: {},
|
||||
components: {},
|
||||
}
|
||||
</script>
|
||||
|
||||
<style lang="scss" rel="stylesheet/scss">
|
||||
.test-model {
|
||||
|
||||
}
|
||||
</style>
|
||||
|
||||
```
|
||||
|
||||
## JavaScript specification
|
||||
|
||||
##### 1.var / let / const
|
||||
It is recommended to no longer use var, but use let / const, prefer const. The use of any variable must be declared in advance, except that the function defined by function can be placed anywhere.
|
||||
|
||||
##### 2.quotes
|
||||
```
|
||||
const foo = 'after division'
|
||||
const bar = `${foo}，ront-end engineer`
|
||||
```
|
||||
|
||||
##### 3.function
|
||||
Anonymous functions use the arrow function uniformly. When multiple parameters/return values are used, the object's structure assignment is used first.
|
||||
```
|
||||
function getPersonInfo ({name, sex}) {
|
||||
// ...
|
||||
return {name, gender}
|
||||
}
|
||||
```
|
||||
The function name is uniformly named with a camel name. The beginning of the capital letter is a constructor. The lowercase letters start with ordinary functions, and the new operator should not be used to operate ordinary functions.
|
||||
|
||||
##### 4.object
|
||||
```
|
||||
const foo = {a: 0, b: 1}
|
||||
const bar = JSON.parse(JSON.stringify(foo))
|
||||
|
||||
const foo = {a: 0, b: 1}
|
||||
const bar = {...foo, c: 2}
|
||||
|
||||
const foo = {a: 3}
|
||||
Object.assign(foo, {b: 4})
|
||||
|
||||
const myMap = new Map([])
|
||||
for (let [key, value] of myMap.entries()) {
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
##### 5.module
|
||||
Unified management of project modules using import / export.
|
||||
```
|
||||
// lib.js
|
||||
export default {}
|
||||
|
||||
// app.js
|
||||
import app from './lib'
|
||||
```
|
||||
|
||||
Import is placed at the top of the file.
|
||||
|
||||
If the module has only one output value, use `export default`，otherwise no.
|
||||
|
||||
## HTML / CSS
|
||||
|
||||
##### 1.Label
|
||||
|
||||
Do not write the type attribute when referencing external CSS or JavaScript. The HTML5 default type is the text/css and text/javascript properties, so there is no need to specify them.
|
||||
```
|
||||
<link rel="stylesheet" href="//www.test.com/css/test.css">
|
||||
<script src="//www.test.com/js/test.js"></script>
|
||||
```
|
||||
|
||||
##### 2.Naming
|
||||
The naming of Class and ID should be semantic, and you can see what you are doing by looking at the name; multiple words are connected by a link.
|
||||
```
|
||||
// positive example
|
||||
.test-header{
|
||||
font-size: 20px;
|
||||
}
|
||||
```
|
||||
|
||||
##### 3.Attribute abbreviation
|
||||
CSS attributes use abbreviations as much as possible to improve the efficiency and ease of understanding of the code.
|
||||
|
||||
```
|
||||
// counter example
|
||||
border-width: 1px;
|
||||
border-style: solid;
|
||||
border-color: #ccc;
|
||||
|
||||
// positive example
|
||||
border: 1px solid #ccc;
|
||||
```
|
||||
|
||||
##### 4.Document type
|
||||
The HTML5 standard should always be used.
|
||||
|
||||
```
|
||||
<!DOCTYPE html>
|
||||
```
|
||||
|
||||
##### 5.Notes
|
||||
A block comment should be written to a module file.
|
||||
```
|
||||
/**
|
||||
* @module mazey/api
|
||||
* @author Mazey <mazey@mazey.net>
|
||||
* @description test.
|
||||
* */
|
||||
```
|
||||
|
||||
## interface
|
||||
|
||||
##### All interfaces are returned as Promise
|
||||
Note that non-zero is wrong for catching catch
|
||||
|
||||
```
|
||||
const test = () => {
|
||||
return new Promise((resolve, reject) => {
|
||||
resolve({
|
||||
a:1
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// transfer
|
||||
test.then(res => {
|
||||
console.log(res)
|
||||
// {a:1}
|
||||
})
|
||||
```
|
||||
|
||||
Normal return
|
||||
```
|
||||
{
|
||||
code:0,
|
||||
data:{}
|
||||
msg:'success'
|
||||
}
|
||||
```
|
||||
|
||||
Error return
|
||||
```
|
||||
{
|
||||
code:10000,
|
||||
data:{}
|
||||
msg:'failed'
|
||||
}
|
||||
```
|
||||
If the interface is a post request, the Content-Type defaults to application/x-www-form-urlencoded; if the Content-Type is changed to application/json,
|
||||
Interface parameter transfer needs to be changed to the following way
|
||||
```
|
||||
io.post('url', payload, null, null, { emulateJSON: false } res => {
|
||||
resolve(res)
|
||||
}).catch(e => {
|
||||
reject(e)
|
||||
})
|
||||
```
|
||||
|
||||
##### Related interface path
|
||||
|
||||
dag related interface `src/js/conf/home/store/dag/actions.js`
|
||||
|
||||
Data Source Center Related Interfaces `src/js/conf/home/store/datasource/actions.js`
|
||||
|
||||
Project Management Related Interfaces `src/js/conf/home/store/projects/actions.js`
|
||||
|
||||
Resource Center Related Interfaces `src/js/conf/home/store/resource/actions.js`
|
||||
|
||||
Security Center Related Interfaces `src/js/conf/home/store/security/actions.js`
|
||||
|
||||
User Center Related Interfaces `src/js/conf/home/store/user/actions.js`
|
||||
|
||||
## Extended development
|
||||
|
||||
##### 1.Add node
|
||||
|
||||
(1) First place the icon icon of the node in the `src/js/conf/home/pages/dag/img `folder, and note the English name of the node defined by the `toolbar_${in the background. For example: SHELL}.png`
|
||||
|
||||
(2) Find the `tasksType` object in `src/js/conf/home/pages/dag/_source/config.js` and add it to it.
|
||||
```
|
||||
'DEPENDENT': { // The background definition node type English name is used as the key value
|
||||
desc: 'DEPENDENT', // tooltip desc
|
||||
color: '#2FBFD8' // The color represented is mainly used for tree and gantt
|
||||
}
|
||||
```
|
||||
|
||||
(3) Add a `${node type (lowercase)}`.vue file in `src/js/conf/home/pages/dag/_source/formModel/tasks`. The contents of the components related to the current node are written here. Must belong to a node component must have a function _verification () After the verification is successful, the relevant data of the current component is thrown to the parent component.
|
||||
```
|
||||
/**
|
||||
* Verification
|
||||
*/
|
||||
_verification () {
|
||||
// datasource subcomponent verification
|
||||
if (!this.$refs.refDs._verifDatasource()) {
|
||||
return false
|
||||
}
|
||||
|
||||
// verification function
|
||||
if (!this.method) {
|
||||
this.$message.warning(`${i18n.$t('Please enter method')}`)
|
||||
return false
|
||||
}
|
||||
|
||||
// localParams subcomponent validation
|
||||
if (!this.$refs.refLocalParams._verifProp()) {
|
||||
return false
|
||||
}
|
||||
// store
|
||||
this.$emit('on-params', {
|
||||
type: this.type,
|
||||
datasource: this.datasource,
|
||||
method: this.method,
|
||||
localParams: this.localParams
|
||||
})
|
||||
return true
|
||||
}
|
||||
```
|
||||
|
||||
(4) Common components used inside the node component are under` _source`, and `commcon.js` is used to configure public data.
|
||||
|
||||
##### 2.Increase the status type
|
||||
(1) Find the `tasksState` object in `src/js/conf/home/pages/dag/_source/config.js` and add it to it.
|
||||
|
||||
```
|
||||
'WAITTING_DEPEND': { // 'WAITTING_DEPEND': { //Backend defines state type, frontend is used as key value
|
||||
id: 11, // front-end definition id is used as a sort
|
||||
desc: `${i18n.$t('waiting for dependency')}`, // tooltip desc
|
||||
color: '#5101be', // The color represented is mainly used for tree and gantt
|
||||
icoUnicode: '', // font icon
|
||||
isSpin: false // whether to rotate (requires code judgment)
|
||||
}
|
||||
```
|
||||
|
||||
##### 3.Add the action bar tool
|
||||
(1) Find the `toolOper` object in `src/js/conf/home/pages/dag/_source/config.js` and add it to it.
|
||||
```
|
||||
{
|
||||
code: 'pointer', // tool identifier
|
||||
icon: '', // tool icon
|
||||
disable: disable, // disable
|
||||
desc: `${i18n.$t('Drag node and selected item')}` // tooltip desc
|
||||
}
|
||||
```
|
||||
|
||||
(2) Tool classes are returned as a constructor `src/js/conf/home/pages/dag/_source/plugIn`
|
||||
|
||||
`downChart.js` => dag image download processing
|
||||
|
||||
`dragZoom.js` => mouse zoom effect processing
|
||||
|
||||
`jsPlumbHandle.js` => drag and drop line processing
|
||||
|
||||
`util.js` => belongs to the `plugIn` tool class
|
||||
|
||||
|
||||
The operation is handled in the `src/js/conf/home/pages/dag/_source/dag.js` => `toolbarEvent` event.
|
||||
|
||||
|
||||
##### 3.Add a routing page
|
||||
|
||||
(1) First add a routing address`src/js/conf/home/router/index.js` in route management
|
||||
```
|
||||
routing address{
|
||||
path: '/test', // routing address
|
||||
name: 'test', // alias
|
||||
component: resolve => require(['../pages/test/index'], resolve), // route corresponding component entry file
|
||||
meta: {
|
||||
title: `${i18n.$t('test')} - EasyScheduler` // title display
|
||||
}
|
||||
},
|
||||
```
|
||||
|
||||
(2)Create a `test` folder in `src/js/conf/home/pages` and create an `index.vue `entry file in the folder.
|
||||
|
||||
This will give you direct access to`http://localhost:8888/#/test`
|
||||
|
||||
|
||||
##### 4.Increase the preset mailbox
|
||||
|
||||
Find the `src/lib/localData/email.js` startup and timed email address input to automatically pull down the match.
|
||||
```
|
||||
export default ["test@analysys.com.cn","test1@analysys.com.cn","test3@analysys.com.cn"]
|
||||
```
|
||||
|
||||
##### 5.Authority management and disabled state processing
|
||||
|
||||
The permission gives the userType according to the backUser interface `getUserInfo` interface: `"ADMIN_USER/GENERAL_USER" `permission to control whether the page operation button is `disabled`.
|
||||
|
||||
specific operation：`src/js/module/permissions/index.js`
|
||||
|
||||
disabled processing：`src/js/module/mixin/disabledState.js`
|
||||
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
# Have Questions?
|
||||
|
||||
## StackOverflow
|
||||
|
||||
For usage questions, it is recommended you use the StackOverflow tag [apache-dolphinscheduler](https://stackoverflow.com/questions/tagged/apache-dolphinscheduler) as it is an active forum for DolphinScheduler users’ questions and answers.
|
||||
|
||||
Some quick tips when using StackOverflow:
|
||||
|
||||
- Prior to asking submitting questions, please:
|
||||
- Search StackOverflow’s [apache-dolphinscheduler](https://stackoverflow.com/questions/tagged/apache-dolphinscheduler) tag to see if your question has already been answered
|
||||
- Please follow the StackOverflow [code of conduct](https://stackoverflow.com/help/how-to-ask)
|
||||
- Always use the apache-dolphinscheduler tag when asking questions
|
||||
- Please do not cross-post between [StackOverflow](https://stackoverflow.com/questions/tagged/apache-dolphinscheduler) and [GitHub issues](https://github.com/apache/dolphinscheduler/issues/new/choose)
|
||||
|
||||
Question template:
|
||||
|
||||
> **Describe the question**
|
||||
>
|
||||
> A clear and concise description of what the question is.
|
||||
>
|
||||
> **Which version of DolphinScheduler:**
|
||||
>
|
||||
> -[1.3.0-preview]
|
||||
>
|
||||
> **Additional context**
|
||||
>
|
||||
> Add any other context about the problem here.
|
||||
>
|
||||
> **Requirement or improvement**
|
||||
>
|
||||
> \- Please describe about your requirements or improvement suggestions.
|
||||
|
||||
For broad, opinion based, ask for external resources, debug issues, bugs, contributing to the project, and scenarios, it is recommended you use the[ GitHub issues ](https://github.com/apache/dolphinscheduler/issues/new/choose)or dev@dolphinscheduler.apache.org mailing list.
|
||||
|
||||
## Mailing Lists
|
||||
|
||||
- [dev@dolphinscheduler.apache.org](https://lists.apache.org/list.html?dev@dolphinscheduler.apache.org) is for people who want to contribute code to DolphinScheduler. [(subscribe)](mailto:dev-subscribe@dolphinscheduler.apache.org?subject=(send%20this%20email%20to%20subscribe)) [(unsubscribe)](mailto:dev-unsubscribe@dolphinscheduler.apache.org?subject=(send%20this%20email%20to%20unsubscribe)) [(archives)](http://lists.apache.org/list.html?dev@dolphinscheduler.apache.org)
|
||||
|
||||
Some quick tips when using email:
|
||||
|
||||
- Prior to asking submitting questions, please:
|
||||
- Search StackOverflow at [apache-dolphinscheduler](https://stackoverflow.com/questions/tagged/apache-dolphinscheduler) to see if your question has already been answered
|
||||
|
||||
- Tagging the subject line of your email will help you get a faster response, e.g. [api-server]: How to get open api interface?
|
||||
|
||||
- Tags may help identify a topic by:
|
||||
- Component: MasterServer,ApiServer,WorkerServer,AlertServer, etc
|
||||
- Level: Beginner, Intermediate, Advanced
|
||||
- Scenario: Debug, How-to
|
||||
|
||||
- For error logs or long code examples, please use [GitHub gist](https://gist.github.com/) and include only a few lines of the pertinent code / log within the email.
|
||||
|
||||
## Chat Rooms
|
||||
|
||||
Chat rooms are great for quick questions or discussions on specialized topics.
|
||||
|
||||
The following chat rooms are officially part of Apache DolphinScheduler:
|
||||
|
||||
The Slack workspace URL: http://asf-dolphinscheduler.slack.com/.
|
||||
|
||||
You can join through invitation url: https://s.apache.org/dolphinscheduler-slack.
|
||||
|
||||
This chat room is used for questions and discussions related to using DolphinScheduler.
|
||||
|
||||
|
||||
|
|
@ -1,712 +0,0 @@
|
|||
<!-- markdown-link-check-disable -->
|
||||
## Q: What's the name of this project?
|
||||
|
||||
A: DolphinScheduler
|
||||
|
||||
---
|
||||
|
||||
## Q: DolphinScheduler service introduction and recommended running memory
|
||||
|
||||
A: DolphinScheduler consists of 5 services, MasterServer, WorkerServer, ApiServer, AlertServer, LoggerServer and UI.
|
||||
|
||||
| Service | Description |
|
||||
| ------------------------- | ------------------------------------------------------------ |
|
||||
| MasterServer | Mainly responsible for DAG segmentation and task status monitoring |
|
||||
| WorkerServer/LoggerServer | Mainly responsible for the submission, execution and update of task status. LoggerServer is used for Rest Api to view logs through RPC |
|
||||
| ApiServer | Provides the Rest Api service for the UI to call |
|
||||
| AlertServer | Provide alarm service |
|
||||
| UI | Front page display |
|
||||
|
||||
Note：**Due to the large number of services, it is recommended that the single-machine deployment is preferably 4 cores and 16G or more.**
|
||||
|
||||
---
|
||||
|
||||
## Q: Which mailboxes does the system support?
|
||||
|
||||
A: Support most mailboxes, qq, 163, 126, 139, outlook, aliyun, etc. are supported. Support TLS and SSL protocols, optionally configured in alert.properties
|
||||
|
||||
---
|
||||
|
||||
## Q: What are the common system variable time parameters and how do I use them?
|
||||
|
||||
A: Please refer to 'System parameter' in the system-manual
|
||||
|
||||
---
|
||||
|
||||
## Q: pip install kazoo This installation gives an error. Is it necessary to install?
|
||||
|
||||
A: This is the python connection Zookeeper needs to use, it is used to delete the master/worker temporary node info in the Zookeeper. so you can ignore error if it's your first install. after version 1.3.0, kazoo is not been needed, we use program to replace what kazoo done
|
||||
|
||||
---
|
||||
|
||||
## Q: How to specify the machine running task
|
||||
|
||||
A: version 1.2 and berfore, Use **the administrator** to create a Worker group, **specify the Worker group** when the **process definition starts**, or **specify the Worker group on the task node**. If not specified, use Default, **Default is to select one of all the workers in the cluster to use for task submission and execution.**
|
||||
version 1.3, you can set worker group for the worker
|
||||
|
||||
---
|
||||
|
||||
## Q: Priority of the task
|
||||
|
||||
A: We also support **the priority of processes and tasks**. Priority We have five levels of **HIGHEST, HIGH, MEDIUM, LOW and LOWEST**. **You can set the priority between different process instances, or you can set the priority of different task instances in the same process instance.** For details, please refer to the task priority design in the architecture-design.
|
||||
|
||||
---
|
||||
|
||||
## Q: dolphinscheduler-grpc gives an error
|
||||
|
||||
A: Execute in the root directory: mvn -U clean package assembly:assembly -Dmaven.test.skip=true , then refresh the entire project.
|
||||
version 1.3 not use grpc, we use netty directly
|
||||
|
||||
---
|
||||
|
||||
## Q: Does DolphinScheduler support running on windows?
|
||||
|
||||
A: In theory, **only the Worker needs to run on Linux**. Other services can run normally on Windows. But it is still recommended to deploy on Linux.
|
||||
|
||||
---
|
||||
|
||||
## Q: UI compiles node-sass prompt in linux: Error: EACCESS: permission denied, mkdir xxxx
|
||||
|
||||
A: Install **npm install node-sass --unsafe-perm** separately, then **npm install**
|
||||
|
||||
---
|
||||
|
||||
## Q: UI cannot log in normally.
|
||||
|
||||
A: 1, if it is node startup, check whether the .env API_BASE configuration under dolphinscheduler-ui is the Api Server service address.
|
||||
|
||||
2, If it is nginx booted and installed via **install-dolphinscheduler-ui.sh**, check if the proxy_pass configuration in **/etc/nginx/conf.d/dolphinscheduler.conf** is the Api Server service address
|
||||
|
||||
3, if the above configuration is correct, then please check if the Api Server service is normal,
|
||||
|
||||
curl http://localhost:12345/dolphinscheduler/users/get-user-info, check the Api Server log,
|
||||
|
||||
if Prompt cn.dolphinscheduler.api.interceptor.LoginHandlerInterceptor:[76] - session info is null, which proves that the Api Server service is normal.
|
||||
|
||||
4, if there is no problem above, you need to check if **server.context-path and server.port configuration** in **application.properties** is correct
|
||||
|
||||
---
|
||||
|
||||
## Q: After the process definition is manually started or scheduled, no process instance is generated.
|
||||
|
||||
A: 1, first **check whether the MasterServer service exists through jps**, or directly check whether there is a master service in zk from the service monitoring.
|
||||
|
||||
2,If there is a master service, check **the command status statistics** or whether new records are added in **t_ds_error_command**. If it is added, **please check the message field.**
|
||||
|
||||
---
|
||||
|
||||
## Q : The task status is always in the successful submission status.
|
||||
|
||||
A: 1, **first check whether the WorkerServer service exists through jps**, or directly check whether there is a worker service in zk from the service monitoring.
|
||||
|
||||
2,If the **WorkerServer** service is normal, you need to **check whether the MasterServer puts the task task in the zk queue. You need to check whether the task is blocked in the MasterServer log and the zk queue.**
|
||||
|
||||
3, if there is no problem above, you need to locate whether the Worker group is specified, but **the machine grouped by the worker is not online**.
|
||||
|
||||
---
|
||||
|
||||
## Q: Is there a Docker image and a Dockerfile?
|
||||
|
||||
A: Provide Docker image and Dockerfile.
|
||||
|
||||
Docker image address: https://hub.docker.com/r/escheduler/escheduler_images
|
||||
|
||||
Dockerfile address: https://github.com/qiaozhanwei/escheduler_dockerfile/tree/master/docker_escheduler
|
||||
|
||||
---
|
||||
|
||||
## Q : Need to pay attention to the problem in install.sh
|
||||
|
||||
A: 1, if the replacement variable contains special characters, **use the \ transfer character to transfer**
|
||||
|
||||
2, installPath="/data1_1T/dolphinscheduler", **this directory can not be the same as the install.sh directory currently installed with one click.**
|
||||
|
||||
3, deployUser = "dolphinscheduler", **the deployment user must have sudo privileges**, because the worker is executed by sudo -u tenant sh xxx.command
|
||||
|
||||
4, monitorServerState = "false", whether the service monitoring script is started, the default is not to start the service monitoring script. **If the service monitoring script is started, the master and worker services are monitored every 5 minutes, and if the machine is down, it will automatically restart.**
|
||||
|
||||
5, hdfsStartupSate="false", whether to enable HDFS resource upload function. The default is not enabled. **If it is not enabled, the resource center cannot be used.** If enabled, you need to configure the configuration of fs.defaultFS and yarn in conf/common/hadoop/hadoop.properties. If you use namenode HA, you need to copy core-site.xml and hdfs-site.xml to the conf root directory.
|
||||
|
||||
Note: **The 1.0.x version does not automatically create the hdfs root directory, you need to create it yourself, and you need to deploy the user with hdfs operation permission.**
|
||||
|
||||
---
|
||||
|
||||
## Q : Process definition and process instance offline exception
|
||||
|
||||
A : For **versions prior to 1.0.4**, modify the code under the escheduler-api cn.escheduler.api.quartz package.
|
||||
|
||||
```
|
||||
public boolean deleteJob(String jobName, String jobGroupName) {
|
||||
lock.writeLock().lock();
|
||||
try {
|
||||
JobKey jobKey = new JobKey(jobName,jobGroupName);
|
||||
if(scheduler.checkExists(jobKey)){
|
||||
logger.info("try to delete job, job name: {}, job group name: {},", jobName, jobGroupName);
|
||||
return scheduler.deleteJob(jobKey);
|
||||
}else {
|
||||
return true;
|
||||
}
|
||||
|
||||
} catch (SchedulerException e) {
|
||||
logger.error(String.format("delete job : %s failed",jobName), e);
|
||||
} finally {
|
||||
lock.writeLock().unlock();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Q: Can the tenant created before the HDFS startup use the resource center normally?
|
||||
|
||||
A: No. Because the tenant created by HDFS is not started, the tenant directory will not be registered in HDFS. So the last resource will report an error.
|
||||
|
||||
---
|
||||
|
||||
## Q: In the multi-master and multi-worker state, the service is lost, how to be fault-tolerant
|
||||
|
||||
A: **Note:** **Master monitors Master and Worker services.**
|
||||
|
||||
1，If the Master service is lost, other Masters will take over the process of the hanged Master and continue to monitor the Worker task status.
|
||||
|
||||
2，If the Worker service is lost, the Master will monitor that the Worker service is gone. If there is a Yarn task, the Kill Yarn task will be retried.
|
||||
|
||||
Please see the fault-tolerant design in the architecture for details.
|
||||
|
||||
---
|
||||
|
||||
## Q : Fault tolerance for a machine distributed by Master and Worker
|
||||
|
||||
A: The 1.0.3 version only implements the fault tolerance of the Master startup process, and does not take the Worker Fault Tolerance. That is to say, if the Worker hangs, no Master exists. There will be problems with this process. We will add Master and Worker startup fault tolerance in version **1.1.0** to fix this problem. If you want to manually modify this problem, you need to **modify the running task for the running worker task that is running the process across the restart and has been dropped. The running process is set to the failed state across the restart**. Then resume the process from the failed node.
|
||||
|
||||
---
|
||||
|
||||
## Q : Timing is easy to set to execute every second
|
||||
|
||||
A : Note when setting the timing. If the first digit (* * * * * ? *) is set to *, it means execution every second. **We will add a list of recently scheduled times in version 1.1.0.** You can see the last 5 running times online at http://cron.qqe2.com/
|
||||
|
||||
---
|
||||
|
||||
## Q: Is there a valid time range for timing?
|
||||
|
||||
A: Yes, **if the timing start and end time is the same time, then this timing will be invalid timing. If the end time of the start and end time is smaller than the current time, it is very likely that the timing will be automatically deleted.**
|
||||
|
||||
---
|
||||
|
||||
## Q : There are several implementations of task dependencies
|
||||
|
||||
A: 1, the task dependency between **DAG**, is **from the zero degree** of the DAG segmentation
|
||||
|
||||
2, there are **task dependent nodes**, you can achieve cross-process tasks or process dependencies, please refer to the (DEPENDENT) node design in the system-manual.
|
||||
|
||||
Note: **Cross-project processes or task dependencies are not supported**
|
||||
|
||||
---
|
||||
|
||||
## Q: There are several ways to start the process definition.
|
||||
|
||||
A: 1, in **the process definition list**, click the **Start** button.
|
||||
|
||||
2, **the process definition list adds a timer**, scheduling start process definition.
|
||||
|
||||
3, process definition **view or edit** the DAG page, any **task node right click** Start process definition.
|
||||
|
||||
4, you can define DAG editing for the process, set the running flag of some tasks to **prohibit running**, when the process definition is started, the connection of the node will be removed from the DAG.
|
||||
|
||||
---
|
||||
|
||||
## Q : Python task setting Python version
|
||||
|
||||
A: 1，**for the version after 1.0.3** only need to modify PYTHON_HOME in `bin/env/dolphinscheduler_env.sh`
|
||||
|
||||
```
|
||||
export PYTHON_HOME=/bin/python
|
||||
```
|
||||
|
||||
Note: This is **PYTHON_HOME** , which is the absolute path of the python command, not the simple PYTHON_HOME. Also note that when exporting the PATH, you need to directly
|
||||
|
||||
```
|
||||
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH
|
||||
```
|
||||
|
||||
2，For versions prior to 1.0.3, the Python task only supports the Python version of the system. It does not support specifying the Python version.
|
||||
|
||||
---
|
||||
|
||||
## Q：Worker Task will generate a child process through sudo -u tenant sh xxx.command, will kill when kill
|
||||
|
||||
A： We will add the kill task in 1.0.4 and kill all the various child processes generated by the task.
|
||||
|
||||
---
|
||||
|
||||
## Q ： How to use the queue in DolphinScheduler, what does the user queue and tenant queue mean?
|
||||
|
||||
A ： The queue in the DolphinScheduler can be configured on the user or the tenant. **The priority of the queue specified by the user is higher than the priority of the tenant queue.** For example, to specify a queue for an MR task, the queue is specified by mapreduce.job.queuename.
|
||||
|
||||
Note: When using the above method to specify the queue, the MR uses the following methods:
|
||||
|
||||
```
|
||||
Configuration conf = new Configuration();
|
||||
GenericOptionsParser optionParser = new GenericOptionsParser(conf, args);
|
||||
String[] remainingArgs = optionParser.getRemainingArgs();
|
||||
```
|
||||
|
||||
|
||||
|
||||
If it is a Spark task --queue mode specifies the queue
|
||||
|
||||
---
|
||||
|
||||
## Q : Master or Worker reports the following alarm
|
||||
|
||||
<p align="center">
|
||||
<img src="https://analysys.github.io/easyscheduler_docs/images/master_worker_lack_res.png" width="60%" />
|
||||
</p>
|
||||
|
||||
|
||||
|
||||
A ： Change the value of master.properties **master.reserved.memory** under conf to a smaller value, say 0.1 or the value of worker.properties **worker.reserved.memory** is a smaller value, say 0.1
|
||||
|
||||
---
|
||||
|
||||
## Q: The hive version is 1.1.0+cdh5.15.0, and the SQL hive task connection is reported incorrectly.
|
||||
|
||||
<p align="center">
|
||||
<img src="https://analysys.github.io/easyscheduler_docs/images/cdh_hive_error.png" width="60%" />
|
||||
</p>
|
||||
|
||||
|
||||
A ： Will hive pom
|
||||
|
||||
```
|
||||
<dependency>
|
||||
<groupId>org.apache.hive</groupId>
|
||||
<artifactId>hive-jdbc</artifactId>
|
||||
<version>2.1.0</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
change into
|
||||
|
||||
```
|
||||
<dependency>
|
||||
<groupId>org.apache.hive</groupId>
|
||||
<artifactId>hive-jdbc</artifactId>
|
||||
<version>1.1.0</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Q : how to add a worker server
|
||||
A: 1, Create deployment user and hosts mapping, please refer 1.3 part of [cluster deployment](https://dolphinscheduler.apache.org/en-us/docs/laster/user_doc/installation/cluster.html)
|
||||
|
||||
2, Configure hosts mapping and ssh access and modify directory permissions. please refer 1.4 part of [cluster deployment](https://dolphinscheduler.apache.org/en-us/docs/laster/user_doc/installation/cluster.html)
|
||||
|
||||
3, Copy the deployment directory from worker server that has already deployed
|
||||
|
||||
4, Go to bin dir, then start worker server and logger server
|
||||
|
||||
```
|
||||
./dolphinscheduler-daemon.sh start worker-server
|
||||
./dolphinscheduler-daemon.sh start logger-server
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Q : When DolphinScheduler release a new version, and the change between current version and latest, and how to upgrade, and version number specification
|
||||
A: 1, The release process of Apache Project happens in the mailing list. You can subscribe DolphinScheduler's mailing list and then when the release is in process, you'll receive release emails. Please follow this [introduction](https://github.com/apache/dolphinscheduler#get-help) to subscribe DolphinScheduler's mailing list.
|
||||
|
||||
2, When new version published, there would be release note which describe the change log, and there also have upgrade document for the previous version to new's.
|
||||
|
||||
3, Version number is x.y.z, when x is increased, it represents the version of the new architecture. When y is increased, it means that it is incompatible with the y version before it needs to be upgraded by script or other manual processing. When the z increase represents a bug fix, the upgrade is fully compatible. No additional processing is required. Remaining problem, the 1.0.2 upgrade is not compatible with 1.0.1 and requires an upgrade script.
|
||||
|
||||
---
|
||||
|
||||
## Q : Subsequent tasks can execute even front task failed
|
||||
A: When start the workflow, you can set the task failure strategy: continue or failure.
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Q : Workflow template DAG, workflow instance, work task and what is the relationship among them? A DAG supports a maximum concurrency of 100, does it mean that 100 workflow instances are generated and run concurrently? A task node in a DAG also has a concurrent number configuration. Does it mean that tasks can run concurrently with multiple threads? Is the maximum number 100?
|
||||
A:
|
||||
|
||||
1.2.1 version
|
||||
```
|
||||
master.properties
|
||||
Control the max parallel number of master node workflows
|
||||
master.exec.threads=100
|
||||
|
||||
Control the max number of parallel tasks in each workflow
|
||||
master.exec.task.number=20
|
||||
|
||||
worker.properties
|
||||
Control the max parallel number of worker node tasks
|
||||
worker.exec.threads=100
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Q : Worker group manage page no buttons displayed
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/39816903/81903776-d8cb9180-95f4-11ea-98cb-94ca1e6a1db5.png" width="60%" />
|
||||
</p>
|
||||
A: For version 1.3.0, we want to support k8s, while the ip always will be changed, so can't config on the UI, worker can config group name in the worker.properties.
|
||||
|
||||
---
|
||||
|
||||
## Q : Why not add mysql jdbc connector to docker image
|
||||
A: The license of mysql jdbc connector is not compatible with apache v2 license, so it can't be included by docker image.
|
||||
|
||||
---
|
||||
|
||||
## Q : Allways fail when a task instance submit multiple yarn application
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/16174111/81312485-476e9380-90b9-11ea-9aad-ed009db899b1.png" width="60%" />
|
||||
</p>
|
||||
A： This bug have fix in dev and in Requirement/TODO list.
|
||||
|
||||
---
|
||||
|
||||
## Q : Master server and worker server stop abnormally after run for a few days
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/18378986/81293969-c3101680-90a0-11ea-87e5-ac9f0dd53f5e.png" width="60%" />
|
||||
</p>
|
||||
A: Session timeout is too short, only 0.3 seconds. Change the config item in zookeeper.properties:
|
||||
|
||||
```
|
||||
zookeeper.session.timeout=60000
|
||||
zookeeper.connection.timeout=30000
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Q : Started using the docker-compose default configuration and display zookeeper errors
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/42579056/80374318-13c98780-88c9-11ea-8d5f-53448b957f02.png" width="60%" />
|
||||
</p>
|
||||
A: This problem is solved in dev-1.3.0. This [pr](https://github.com/apache/dolphinscheduler/pull/2595) has solved this bug, brief change log:
|
||||
|
||||
```
|
||||
1. add zookeeper environment variable ZOO_4LW_COMMANDS_WHITELIST in docker-compose.yml file.
|
||||
2. change the data type of minLatency, avgLatency and maxLatency from int to float.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Q : Interface show some task would be running all the time when db delayed and log show task instance is null
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/51871547/80302626-b1478d00-87dd-11ea-97d4-08aa2244a6d0.jpg" width="60%" />
|
||||
</p>
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/51871547/80302626-b1478d00-87dd-11ea-97d4-08aa2244a6d0.jpg" width="60%" />
|
||||
</p>
|
||||
|
||||
A: This [bug](https://github.com/apache/dolphinscheduler/issues/1477) describe the problem detail and it has been been solved in version 1.2.1.
|
||||
|
||||
For version under 1.2.1, some tips for this situation:
|
||||
|
||||
```
|
||||
1. clear the task queue in zk for path: /dolphinscheduler/task_queue
|
||||
2. change the state of the task to failed( integer value: 6).
|
||||
3. run the work flow by recover from failed
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Q : Zookeeper masters znode list ip address is 127.0.0.1, instead of wanted ip eth0 or eth1, and may can't see task log
|
||||
A: bug fix:
|
||||
```
|
||||
1, confirm hostname
|
||||
$hostname
|
||||
hadoop1
|
||||
2, hostname -i
|
||||
127.0.0.1 10.3.57.15
|
||||
3, edit /etc/hosts,delete hadoop1 from 127.0.0.1 record
|
||||
$cat /etc/hosts
|
||||
127.0.0.1 localhost
|
||||
10.3.57.15 ds1 hadoop1
|
||||
4, hostname -i
|
||||
10.3.57.15
|
||||
```
|
||||
|
||||
Hostname cmd return server hostname, hostname -i return all matched ips configured in /etc/hosts. So after I delete the hostname matched with 127.0.0.1, and only remain internal ip resolution, instead of remove all the 127.0.0.1 resolution record. As long as hostname cmd return the correct internal ip configured in /etc/hosts can fix this bug. DolphinScheduler use the first record returned by hostname -i command. In my opion, DS should not use hostname -i to get the ip , as in many companies the devops configured the server name, we suggest use ip configured in configuration file or znode instead of /etc/hosts.
|
||||
|
||||
---
|
||||
|
||||
## Q : The scheduling system set a second frequency task, causing the system to crash
|
||||
A: The scheduling system not support second frequency task.
|
||||
|
||||
---
|
||||
|
||||
## Q : Compile front-end code(dolphinscheduler-ui) show error cannot download "https://github.com/sass/node-sass/releases/download/v4.13.1/darwin-x64-72_binding.node"
|
||||
A: 1, cd dolphinscheduler-ui and delete node_modules directory
|
||||
```
|
||||
sudo rm -rf node_modules
|
||||
```
|
||||
2, install node-sass through npm.taobao.org
|
||||
```
|
||||
sudo npm uninstall node-sass
|
||||
sudo npm i node-sass --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/
|
||||
```
|
||||
3, if the 2nd step failure, please, [referer url](https://dolphinscheduler.apache.org/en-us/development/frontend-development.html)
|
||||
```
|
||||
sudo npm rebuild node-sass
|
||||
```
|
||||
When solved this problem, if you don't want to download this node every time, you can set system environment variable: SASS_BINARY_PATH= /xxx/xxx/xxx/xxx.node.
|
||||
|
||||
---
|
||||
|
||||
## Q : How to config when use mysql as database instead of postgres
|
||||
A: 1, Edit project root dir maven config file, remove scope test property so that mysql driver can be loaded.
|
||||
```
|
||||
<dependency>
|
||||
<groupId>mysql</groupId>
|
||||
<artifactId>mysql-connector-java</artifactId>
|
||||
<version>${mysql.connector.version}</version>
|
||||
<scope>test<scope>
|
||||
</dependency>
|
||||
```
|
||||
2, Edit application-dao.properties and quzrtz.properties config file to use mysql driver.
|
||||
Default is postgresSql driver because of license problem.
|
||||
|
||||
---
|
||||
|
||||
## Q : How does a shell task run
|
||||
A: 1, Where is the executed server? Specify one worker to run the task, you can create worker group in Security Center, then the task can be send to the particular worker. If a worker group have multiple servers, which server actually execute is determined by scheduling and has randomness.
|
||||
|
||||
2, If it is a shell file of a path on the server, how to point to the path? The server shell file, involving permissions issues, it is not recommended to do so. It is recommended that you use the storage function of the resource center, and then use the resource reference in the shell editor. The system will help you download the script to the execution directory. If the task dependent on resource center files, worker use "hdfs dfs -get" to get the resource files in HDFS, then run the task in /tmp/escheduler/exec/process, this path can be customized when installtion dolphinscheduler.
|
||||
|
||||
3, Which user execute the task? Task is run by the tenant through "sudo -u ${tenant}", tenant is a linux user.
|
||||
|
||||
---
|
||||
|
||||
## Q : What’s the best deploy mode you suggest in production env
|
||||
A: 1, I suggest you use 3 nodes for stability if you don't have too many tasks to run. And deploy Master/Worker server on different nodes is better. If you only have one node, you of course only can deploy them together! By the way, how many machines you need is determined by your business. The DolphinScheduler system itself does not use too many resources. Test more, and you'll find the right way to use a few machines.
|
||||
|
||||
---
|
||||
|
||||
## Q : DEPENDENT Task Node
|
||||
A: 1, DEPENDENT task node actually does not have script, it used for config data cycle dependent logic, and then add task node after that to realize task cycle dependent.
|
||||
|
||||
---
|
||||
|
||||
## Q : How to change the boot port of the master
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/8263441/62352160-0f3e9100-b53a-11e9-95ba-3ae3dde49c72.png" width="60%" />
|
||||
</p>
|
||||
A: 1, modify application_master.properties, for example: server.port=12345.
|
||||
|
||||
---
|
||||
|
||||
## Q : Scheduled tasks cannot be online
|
||||
A: 1, We can successly create scheduled task and add one record into t_scheduler_schedules table, but when I click online, front page no reaction and will lock table t_scheduler_schedules, and tested set field release_state value to 1 in table t_scheduler_schedules, and task display online state. For DS version above 1.2 table name is t_ds_schedules, other version table name is t_scheduler_schedules.
|
||||
|
||||
---
|
||||
|
||||
## Q : What is the address of swagger ui
|
||||
A: 1, For version 1.2+ is http://apiServerIp:apiServerPort/dolphinscheduler/doc.html others is http://apiServerIp:apiServerPort/escheduler/doc.html.
|
||||
|
||||
---
|
||||
|
||||
## Q : Front-end installation package is missing files
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/41460919/61437083-d960b080-a96e-11e9-87f1-297ba3aca5e3.png" width="60%" />
|
||||
</p>
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/41460919/61437218-1b89f200-a96f-11e9-8e48-3fac47eb2389.png" width="60%" />
|
||||
</p>
|
||||
|
||||
A: 1, User changed the config api server config file and item
|
||||
, thus lead to the problem. After resume to the default value and problem solved.
|
||||
|
||||
---
|
||||
|
||||
## Q : Upload a relatively large file blocked
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/21357069/58231400-805b0e80-7d69-11e9-8107-7f37b06a95df.png" width="60%" />
|
||||
</p>
|
||||
A: 1, Edit ngnix config file, edit upload max size client_max_body_size 1024m.
|
||||
|
||||
2, the version of Google Chrome is old, and the latest version of the browser has been updated.
|
||||
|
||||
---
|
||||
|
||||
## Q : Create a spark data source, click "Test Connection", the system will fall back to the login page
|
||||
A: 1, edit nginx config file /etc/nginx/conf.d/escheduler.conf
|
||||
```
|
||||
proxy_connect_timeout 300s;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_send_timeout 300s;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Q : Workflow Dependency
|
||||
A: 1, It is currently judged according to natural days, at the end of last month: the judgment time is the workflow A start_time/scheduler_time between '2019-05-31 00:00:00' and '2019-05-31 23:59:59'. Last month: It is judged that there is an A instance completed every day from the 1st to the end of the month. Last week: There are completed A instances 7 days last week. The first two days: Judging yesterday and the day before yesterday, there must be a completed A instance for two days.
|
||||
|
||||
---
|
||||
|
||||
## Q : DS Backend Inteface Document
|
||||
A: 1, http://106.75.43.194:8888/dolphinscheduler/doc.html?language=en.
|
||||
|
||||
## During the operation of dolphinscheduler, the ip address is obtained incorrectly
|
||||
|
||||
When the master service and worker service are registered with zookeeper, relevant information will be created in the form of ip:port
|
||||
|
||||
If the ip address is obtained incorrectly, please check the network information. For example, in the Linux system, use the `ifconfig` command to view the network information. The following figure is an example:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/network/network_config.png" width="60%" />
|
||||
</p>
|
||||
|
||||
You can use the three strategies provided by dolphinscheduler to get the available ip:
|
||||
|
||||
* default: First using internal network card to obtain the IP address, and then using external network card. If all above fail, use the address of the first available network card
|
||||
* inner: Use the internal network card to obtain the ip address, if fails thrown an exception.
|
||||
* outer: Use the external network card to obtain the ip address, if fails thrown an exception.
|
||||
|
||||
Modify the configuration in `common.properties`:
|
||||
|
||||
```shell
|
||||
# network IP gets priority, default: inner outer
|
||||
# dolphin.scheduler.network.priority.strategy=default
|
||||
```
|
||||
|
||||
After configuration is modified, restart the service to activation
|
||||
|
||||
If the ip address is still wrong, please download [dolphinscheduler-netutils.jar](/asset/dolphinscheduler-netutils.jar) to the machine, execute the following commands and feedback the output to the community developers:
|
||||
|
||||
```shell
|
||||
java -jar target/dolphinscheduler-netutils.jar
|
||||
```
|
||||
|
||||
## Configure sudo to be secret free, which is used to solve the problem of using the default configuration sudo authority to be too large or unable to apply for root authority
|
||||
|
||||
Configure the sudo permission of the dolphinscheduler account to be an ordinary user manager within the scope of some ordinary users, and restrict specified users to run certain commands on the specified host. For detailed configuration, please see sudo rights management
|
||||
For example, sudo permission management configuration dolphinscheduler OS account can only operate the permissions of users userA, userB, userC (users userA, userB, and userC are used for multi-tenant submitting jobs to the big data cluster)
|
||||
|
||||
```shell
|
||||
echo 'dolphinscheduler ALL=(userA,userB,userC) NOPASSWD: NOPASSWD: ALL' >> /etc/sudoers
|
||||
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Q：Deploy for multiple YARN clusters
|
||||
A：By deploying different worker in different yarn clusters，the steps are as follows(eg: AWS EMR):
|
||||
|
||||
1. Deploying the worker server on the master node of the EMR cluster
|
||||
|
||||
2. Changing `yarn.application.status.address` to current emr's yarn url in the `conf/common.properties`
|
||||
|
||||
3. Execute command `bin/dolphinscheduler-daemon.sh start worker-server` and `bin/dolphinscheduler-daemon.sh start logger-server` to start worker-server and logger-server
|
||||
|
||||
---
|
||||
|
||||
## Q：Update process definition error: Duplicate key TaskDefinition
|
||||
|
||||
A：Before DS 2.0.4 (after 2.0.0-alpha), there may be a problem of duplicate keys TaskDefinition due to version switching, which may cause the update workflow to fail; you can refer to the following SQL to delete duplicate data, taking MySQL as an example: (Note: Before operating, be sure to back up the original data, the SQL from pr[#8408](https://github.com/apache/dolphinscheduler/pull/8408))
|
||||
|
||||
```SQL
|
||||
DELETE FROM t_ds_process_task_relation_log WHERE id IN
|
||||
(
|
||||
SELECT
|
||||
x.id
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
aa.id
|
||||
FROM
|
||||
t_ds_process_task_relation_log aa
|
||||
JOIN
|
||||
(
|
||||
SELECT
|
||||
a.process_definition_code
|
||||
,MAX(a.id) as min_id
|
||||
,a.pre_task_code
|
||||
,a.pre_task_version
|
||||
,a.post_task_code
|
||||
,a.post_task_version
|
||||
,a.process_definition_version
|
||||
,COUNT(*) cnt
|
||||
FROM
|
||||
t_ds_process_task_relation_log a
|
||||
JOIN (
|
||||
SELECT
|
||||
code
|
||||
FROM
|
||||
t_ds_process_definition
|
||||
GROUP BY code
|
||||
)b ON b.code = a.process_definition_code
|
||||
WHERE 1=1
|
||||
GROUP BY a.pre_task_code
|
||||
,a.post_task_code
|
||||
,a.pre_task_version
|
||||
,a.post_task_version
|
||||
,a.process_definition_code
|
||||
,a.process_definition_version
|
||||
HAVING COUNT(*) > 1
|
||||
)bb ON bb.process_definition_code = aa.process_definition_code
|
||||
AND bb.pre_task_code = aa.pre_task_code
|
||||
AND bb.post_task_code = aa.post_task_code
|
||||
AND bb.process_definition_version = aa.process_definition_version
|
||||
AND bb.pre_task_version = aa.pre_task_version
|
||||
AND bb.post_task_version = aa.post_task_version
|
||||
AND bb.min_id != aa.id
|
||||
)x
|
||||
)
|
||||
;
|
||||
|
||||
DELETE FROM t_ds_task_definition_log WHERE id IN
|
||||
(
|
||||
SELECT
|
||||
x.id
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
a.id
|
||||
FROM
|
||||
t_ds_task_definition_log a
|
||||
JOIN
|
||||
(
|
||||
SELECT
|
||||
code
|
||||
,name
|
||||
,version
|
||||
,MAX(id) AS min_id
|
||||
FROM
|
||||
t_ds_task_definition_log
|
||||
GROUP BY code
|
||||
,name
|
||||
,version
|
||||
HAVING COUNT(*) > 1
|
||||
)b ON b.code = a.code
|
||||
AND b.name = a.name
|
||||
AND b.version = a.version
|
||||
AND b.min_id != a.id
|
||||
)x
|
||||
)
|
||||
;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Q：Upgrade from 2.0.1 to 2.0.5 using PostgreSQL database failed
|
||||
|
||||
A：The repair can be completed by executing the following SQL in the database:
|
||||
```SQL
|
||||
update t_ds_version set version='2.0.1';
|
||||
```
|
||||
|
||||
## Can not find python-gateway-server in distribute package
|
||||
|
||||
After version 3.0.0-alpha, Python gateway server integrate into API server, and Python gateway service will start when you
|
||||
start API server. If you want disabled when Python gateway service you could change API server configuration in path
|
||||
`api-server/conf/application.yaml` and change attribute `python-gateway.enabled : false`.
|
||||
|
||||
---
|
||||
|
||||
## We will collect more FAQ later
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Alert Component User Guide
|
||||
|
||||
## How to Create Alert Plugins and Alert Groups
|
||||
|
||||
In version 2.0.0, users need to create alert instances, and needs to choose an alarm policy when defining an alarm instance, there are three options: send if the task succeeds, send on failure, and send on both success and failure. when the workflow or task is executed, if an alarm is triggered, calling the alarm instance send method needs a logical judgment, which matches the alarm instance with the task status, executes the alarm instance sending logic if it matches, and filters if it does not match. When create alert instances then associate them with alert groups. Alert group can use multiple alert instances.
|
||||
The alarm module supports the following scenarios:
|
||||
<img src="/img/alert/alert_scenarios_en.png">
|
||||
|
||||
The steps to use are as follows:
|
||||
|
||||
First, go to the Security Center page. Select Alarm Group Management, click Alarm Instance Management on the left and create an alarm instance. Select the corresponding alarm plug-in and fill in the relevant alarm parameters.
|
||||
|
||||
Then select Alarm Group Management, create an alarm group, and choose the corresponding alarm instance.
|
||||
|
||||
<img src="/img/alert/alert_step_1.png">
|
||||
<img src="/img/alert/alert_step_2.png">
|
||||
<img src="/img/alert/alert_step_3.png">
|
||||
<img src="/img/alert/alert_step_4.png">
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
# DingTalk
|
||||
|
||||
If you need to use `DingTalk` for alerting, create an alert instance in the alert instance management and select the DingTalk plugin.
|
||||
The following shows the `DingTalk` configuration example:
|
||||
|
||||
|
||||
|
||||
## Parameter Configuration
|
||||
|
||||
* Webhook
|
||||
> The format is: https://oapi.dingtalk.com/robot/send?access_token=XXXXXX
|
||||
* Keyword
|
||||
> Custom keywords for security settings
|
||||
* Secret
|
||||
> Signature of security settings
|
||||
* MessageType
|
||||
> Support both text and markdown types
|
||||
|
||||
When a custom bot sends a message, you can specify the "@person list" by their mobile phone number. When the selected people in the "@people list" receive the message, there will be a `@` message reminder. `No disturb` mode always receives reminders, and "someone @ you" appears in the message.
|
||||
* @Mobiles
|
||||
> The mobile phone number of the "@person"
|
||||
* @UserIds
|
||||
> The user ID by "@person"
|
||||
* @All
|
||||
> @Everyone
|
||||
|
||||
[DingTalk Custom Robot Access Development Documentation](https://open.dingtalk.com/document/robots/custom-robot-access)
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
# Webex Teams
|
||||
|
||||
If you need to use `Webex Teams` to alert, create an alert instance in the alert instance management, and choose the WebexTeams plugin.
|
||||
You can pick private alert or room group chat alert.
|
||||
The following is the `WebexTeams` configuration example:
|
||||
|
||||
|
||||
|
||||
## Parameter Configuration
|
||||
|
||||
* botAccessToken
|
||||
> The robot's access token
|
||||
* roomID
|
||||
> The ID of the room that receives message (only support one room ID)
|
||||
* toPersonId
|
||||
> The person ID of the recipient when sending a private 1:1 message
|
||||
* toPersonEmail
|
||||
> The email address of the recipient when sending a private 1:1 message
|
||||
* atSomeoneInRoom
|
||||
> If the message destination is room, the emails of the person being @, use `,` (eng commas) to separate multiple emails
|
||||
* destination
|
||||
> The destination of the message (one message only support one destination)
|
||||
|
||||
## Create a Bot
|
||||
|
||||
Create a bot visit [Official Website My-Apps](https://developer.webex.com/my-apps) to `Create a New APP` and select `Create a Bot`, fill in the bot information and acquire `bot username` and `bot ID` for further usage.
|
||||
|
||||
|
||||
|
||||
## Create a Room
|
||||
|
||||
Create a root visit [Official Website for Developer APIs](https://developer.webex.com/docs/api/v1/rooms/create-a-room) to create a new room, fill in the room name and acquire `id`(room ID) and `creatorId` for further usage.
|
||||
|
||||
|
||||
|
||||
### Invite Bot to the Room
|
||||
|
||||
Invite bot to the room by invite bot Email (bot username).
|
||||
|
||||
## Send Private Message
|
||||
|
||||
In this way, you can send private message to a person by `User Email` or `UserId` in a private conversation. Fill in the `To Person Id` or `To Person Email`(recommended) and `Bot Access Token` and select `Destination` `personEmail` or `personId`.
|
||||
The `user Email` is user register Email.
|
||||
The `userId` we can acquire it from the `creatorId` of creating a new group chat room API.
|
||||
|
||||
|
||||
|
||||
### Private Alert Message Example
|
||||
|
||||
|
||||
|
||||
## Send Group Room Message
|
||||
|
||||
In this way, you can send group room message to a room by `Room ID`. Fill in the `Room Id` and `Bot Access Token` and select `Destination` `roomId`.
|
||||
The `Room ID` we can acquire it from the `id` of creating a new group chat room API.
|
||||
|
||||
|
||||
|
||||
### Group Room Alert Message Example
|
||||
|
||||
|
||||
|
||||
[WebexTeams Application Bot Guide](https://developer.webex.com/docs/bots)
|
||||
[WebexTeams Message Guide](https://developer.webex.com/docs/api/v1/messages/create-a-message)
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
# Enterprise WeChat
|
||||
|
||||
If you need to use `Enterprise WeChat` to alert, create an alert instance in the alert instance management, and choose the `WeChat` plugin.
|
||||
The following is the `WeChat` configuration example:
|
||||
|
||||
|
||||
|
||||
## Send Type
|
||||
|
||||
The parameter `send.type` corresponds to sending messages to Enterprise WeChat customized APP and group chat created by API respectively.
|
||||
|
||||
### APP
|
||||
|
||||
The APP sends type means to notify the alert results via Enterprise WeChat customized APPs, supports sending messages to both specified users and all members. Currently, send to specified enterprise department and tags are not supported, a new PR to contribute is welcomed.
|
||||
The following is the `APP` alert config example:
|
||||
|
||||
|
||||
|
||||
The following is the `APP` `MARKDOWN` alert message example:
|
||||
|
||||
|
||||
|
||||
The following is the `APP` `TEXT` alert message example:
|
||||
|
||||
|
||||
|
||||
#### Prerequisites
|
||||
|
||||
Need to create a new customized APP in Enterprise WeChat before sending messages to APP, create at the [APP Page](https://work.weixin.qq.com/wework_admin/frame#apps) and acquire the APP `AgentId` and set its visible scope to the root of the hierarchy.
|
||||
|
||||
#### Send Messages to Specified Users
|
||||
|
||||
The Enterprise WeChat APPs support sending messages to both specified users and all members, using `|` to separate multiple `userIds` and using `@all` to send messages to everyone.
|
||||
To acquire user `userId` refer to [Official Doc](https://developer.work.weixin.qq.com/document/path/95402), acquire `userId` by user phone number.
|
||||
The following is the `query userId` API example:
|
||||
|
||||
|
||||
|
||||
#### References
|
||||
|
||||
APP: https://work.weixin.qq.com/api/doc/90000/90135/90236
|
||||
|
||||
|
||||
### Group Chat
|
||||
|
||||
The Group Chat send type means to notify the alert results via group chat created by Enterprise WeChat API, sending messages to all members of the group and specified users are not supported.
|
||||
The following is the `Group Chat` alert config example:
|
||||
|
||||
|
||||
|
||||
The following is the `APP` `MARKDOWN` alert message example:
|
||||
|
||||
|
||||
|
||||
The following is the `Group Chat` `TEXT` alert message example:
|
||||
|
||||
|
||||
|
||||
#### Prerequisites
|
||||
|
||||
Before sending messages to group chat, create a new group chat by Enterprise WeChat API, refer to [Official Doc](https://developer.work.weixin.qq.com/document/path/90245) to create a new one and acquire `chatid`.
|
||||
To acquire user `userId` refer to [Official Doc](https://developer.work.weixin.qq.com/document/path/95402), acquire `userId` by user phone number.
|
||||
The following is the `create new group chat` API and `query userId` API example:
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#### References
|
||||
|
||||
Group Chat：https://work.weixin.qq.com/api/doc/90000/90135/90248
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
# Telegram
|
||||
|
||||
If you need `Telegram` to alert, create an alert instance in the alert instance management, and choose the `Telegram` plugin.
|
||||
The following shows the `Telegram` configuration example:
|
||||
|
||||
|
||||
|
||||
## Parameter Configuration
|
||||
|
||||
* WebHook:
|
||||
> The WebHook of Telegram when use robot to send message
|
||||
* botToken
|
||||
> The robot's access token
|
||||
* chatId
|
||||
> Sub Telegram Channel
|
||||
* parseMode
|
||||
> Message parse type (support txt, markdown, markdownV2, html)
|
||||
* EnableProxy
|
||||
> Enable proxy sever
|
||||
* Proxy
|
||||
> the proxy address of the proxy server
|
||||
* Port
|
||||
> the proxy port of Proxy-Server
|
||||
* User
|
||||
> Authentication(Username) for the proxy server
|
||||
* Password
|
||||
> Authentication(Password) for the proxy server
|
||||
|
||||
**NOTICE**：The webhook needs to be able to receive and use the same JSON body of HTTP POST that DolphinScheduler constructs and the following shows the JSON body:
|
||||
|
||||
```json
|
||||
{
|
||||
"text": "[{\"projectId\":1,\"projectName\":\"p1\",\"owner\":\"admin\",\"processId\":35,\"processDefinitionCode\":4928367293568,\"processName\":\"s11-3-20220324084708668\",\"taskCode\":4928359068928,\"taskName\":\"s1\",\"taskType\":\"SHELL\",\"taskState\":\"FAILURE\",\"taskStartTime\":\"2022-03-24 08:47:08\",\"taskEndTime\":\"2022-03-24 08:47:09\",\"taskHost\":\"192.168.1.103:1234\",\"logPath\":\"\"}]",
|
||||
"chat_id": "chat id number"
|
||||
}
|
||||
```
|
||||
|
||||
References:
|
||||
- [Telegram Application Bot Guide](https://core.telegram.org/bots)
|
||||
- [Telegram Bots Api](https://core.telegram.org/bots/api)
|
||||
- [Telegram SendMessage Api](https://core.telegram.org/bots/api#sendmessage)
|
||||
|
||||
|
|
@ -1,310 +0,0 @@
|
|||
# Overview
|
||||
## Introduction
|
||||
|
||||
The data quality task is used to check the data accuracy during the integration and processing of data. Data quality tasks in this release include single-table checking, single-table custom SQL checking, multi-table accuracy, and two-table value comparisons. The running environment of the data quality task is Spark 2.4.0, and other versions have not been verified, and users can verify by themselves.
|
||||
- The execution flow of the data quality task is as follows:
|
||||
|
||||
> The user defines the task in the interface, and the user input value is stored in `TaskParam`
|
||||
When running a task, `Master` will parse `TaskParam`, encapsulate the parameters required by `DataQualityTask` and send it to `Worker`.
|
||||
Worker runs the data quality task. After the data quality task finishes running, it writes the statistical results to the specified storage engine. The current data quality task result is stored in the `t_ds_dq_execute_result` table of `dolphinscheduler`
|
||||
`Worker` sends the task result to `Master`, after `Master` receives `TaskResponse`, it will judge whether the task type is `DataQualityTask`, if so, it will read the corresponding result from `t_ds_dq_execute_result` according to `taskInstanceId`, and then The result is judged according to the check mode, operator and threshold configured by the user. If the result is a failure, the corresponding operation, alarm or interruption will be performed according to the failure policy configured by the user.
|
||||
|
||||
Add config : `<server-name>/conf/common.properties`
|
||||
|
||||
```properties
|
||||
data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar
|
||||
```
|
||||
|
||||
Please fill in `data-quality.jar.name` according to the actual package name,
|
||||
If you package `data-quality` separately, remember to modify the package name to be consistent with `data-quality.jar.name`.
|
||||
If the old version is upgraded and used, you need to execute the `sql` update script to initialize the database before running.
|
||||
If you want to use `MySQL` data, you need to comment out the `scope` of `MySQL` in `pom.xml`
|
||||
Currently only `MySQL`, `PostgreSQL` and `HIVE` data sources have been tested, other data sources have not been tested yet
|
||||
`Spark` needs to be configured to read `Hive` metadata, `Spark` does not use `jdbc` to read `Hive`
|
||||
|
||||
## Detail
|
||||
|
||||
- CheckMethod: [CheckFormula][Operator][Threshold], if the result is true, it indicates that the data does not meet expectations, and the failure strategy is executed.
|
||||
- CheckFormula：
|
||||
- Expected-Actual
|
||||
- Actual-Expected
|
||||
- (Actual/Expected)x100%
|
||||
- (Expected-Actual)/Expected x100%
|
||||
- Operator：=、>、>=、<、<=、!=
|
||||
- ExpectedValue
|
||||
- FixValue
|
||||
- DailyAvg
|
||||
- WeeklyAvg
|
||||
- MonthlyAvg
|
||||
- Last7DayAvg
|
||||
- Last30DayAvg
|
||||
- SrcTableTotalRows
|
||||
- TargetTableTotalRows
|
||||
|
||||
- example
|
||||
- CheckFormula：Expected-Actual
|
||||
- Operator：>
|
||||
- Threshold：0
|
||||
- ExpectedValue：FixValue=9。
|
||||
|
||||
Assuming that the actual value is 10, the operator is >, and the expected value is 9, then the result 10 -9 > 0 is true, which means that the row data in the empty column has exceeded the threshold, and the task is judged to fail
|
||||
# Guide
|
||||
## NullCheck
|
||||
### Introduction
|
||||
The goal of the null value check is to check the number of empty rows in the specified column. The number of empty rows can be compared with the total number of rows or a specified threshold. If it is greater than a certain threshold, it will be judged as failure.
|
||||
- Calculate the SQL statement that the specified column is empty as follows:
|
||||
|
||||
```sql
|
||||
SELECT COUNT(*) AS miss FROM ${src_table} WHERE (${src_field} is null or ${src_field} = '') AND (${src_filter})
|
||||
```
|
||||
|
||||
- The SQL to calculate the total number of rows in the table is as follows:
|
||||
|
||||
```sql
|
||||
SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})
|
||||
```
|
||||
|
||||
### UI Guide
|
||||
|
||||
- Source data type: select MySQL, PostgreSQL, etc.
|
||||
- Source data source: the corresponding data source under the source data type
|
||||
- Source data table: drop-down to select the table where the validation data is located
|
||||
- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
|
||||
- Src table check column: drop-down to select the check column name
|
||||
- Check method:
|
||||
- [Expected-Actual]
|
||||
- [Actual-Expected]
|
||||
- [Actual/Expected]x100%
|
||||
- [(Expected-Actual)/Expected]x100%
|
||||
- Check operators: =, >, >=, <, <=, ! =
|
||||
- Threshold: The value used in the formula for comparison
|
||||
- Failure strategy
|
||||
- Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
|
||||
- Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
|
||||
- Expected value type: select the desired type from the drop-down menu
|
||||
|
||||
## Timeliness Check
|
||||
### Introduction
|
||||
The timeliness check is used to check whether the data is processed within the expected time. The start time and end time can be specified to define the time range. If the amount of data within the time range does not reach the set threshold, the check task will be judged as fail
|
||||
### UI Guide
|
||||
|
||||
- Source data type: select MySQL, PostgreSQL, etc.
|
||||
- Source data source: the corresponding data source under the source data type
|
||||
- Source data table: drop-down to select the table where the validation data is located
|
||||
- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
|
||||
- Src table check column: drop-down to select check column name
|
||||
- start time: the start time of a time range
|
||||
- end time: the end time of a time range
|
||||
- Time Format: Set the corresponding time format
|
||||
- Check method:
|
||||
- [Expected-Actual]
|
||||
- [Actual-Expected]
|
||||
- [Actual/Expected]x100%
|
||||
- [(Expected-Actual)/Expected]x100%
|
||||
- Check operators: =, >, >=, <, <=, ! =
|
||||
- Threshold: The value used in the formula for comparison
|
||||
- Failure strategy
|
||||
- Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
|
||||
- Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
|
||||
- Expected value type: select the desired type from the drop-down menu
|
||||
|
||||
## Field Length Check
|
||||
### Introduction
|
||||
The goal of field length verification is to check whether the length of the selected field meets the expectations. If there is data that does not meet the requirements, and the number of rows exceeds the threshold, the task will be judged to fail
|
||||
### UI Guide
|
||||
|
||||
- Source data type: select MySQL, PostgreSQL, etc.
|
||||
- Source data source: the corresponding data source under the source data type
|
||||
- Source data table: drop-down to select the table where the validation data is located
|
||||
- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
|
||||
- Src table check column: drop-down to select the check column name
|
||||
- Logical operators: =, >, >=, <, <=, ! =
|
||||
- Field length limit: like the title
|
||||
- Check method:
|
||||
- [Expected-Actual]
|
||||
- [Actual-Expected]
|
||||
- [Actual/Expected]x100%
|
||||
- [(Expected-Actual)/Expected]x100%
|
||||
- Check operators: =, >, >=, <, <=, ! =
|
||||
- Threshold: The value used in the formula for comparison
|
||||
- Failure strategy
|
||||
- Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
|
||||
- Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
|
||||
- Expected value type: select the desired type from the drop-down menu
|
||||
|
||||
## Uniqueness Check
|
||||
### Introduction
|
||||
The goal of the uniqueness check is to check whether the field is duplicated. It is generally used to check whether the primary key is duplicated. If there is duplication and the threshold is reached, the check task will be judged to be failed.
|
||||
### UI Guide
|
||||
|
||||
- Source data type: select MySQL, PostgreSQL, etc.
|
||||
- Source data source: the corresponding data source under the source data type
|
||||
- Source data table: drop-down to select the table where the validation data is located
|
||||
- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
|
||||
- Src table check column: drop-down to select the check column name
|
||||
- Check method:
|
||||
- [Expected-Actual]
|
||||
- [Actual-Expected]
|
||||
- [Actual/Expected]x100%
|
||||
- [(Expected-Actual)/Expected]x100%
|
||||
- Check operators: =, >, >=, <, <=, ! =
|
||||
- Threshold: The value used in the formula for comparison
|
||||
- Failure strategy
|
||||
- Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
|
||||
- Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
|
||||
- Expected value type: select the desired type from the drop-down menu
|
||||
|
||||
## Regular Expression Check
|
||||
### Introduction
|
||||
The goal of regular expression verification is to check whether the format of the value of a field meets the requirements, such as time format, email format, ID card format, etc. If there is data that does not meet the format and exceeds the threshold, the task will be judged as failed.
|
||||
### UI Guide
|
||||
|
||||
- Source data type: select MySQL, PostgreSQL, etc.
|
||||
- Source data source: the corresponding data source under the source data type
|
||||
- Source data table: drop-down to select the table where the validation data is located
|
||||
- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
|
||||
- Src table check column: drop-down to select check column name
|
||||
- Regular expression: as title
|
||||
- Check method:
|
||||
- [Expected-Actual]
|
||||
- [Actual-Expected]
|
||||
- [Actual/Expected]x100%
|
||||
- [(Expected-Actual)/Expected]x100%
|
||||
- Check operators: =, >, >=, <, <=, ! =
|
||||
- Threshold: The value used in the formula for comparison
|
||||
- Failure strategy
|
||||
- Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
|
||||
- Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
|
||||
- Expected value type: select the desired type from the drop-down menu
|
||||
|
||||
## Enumeration Check
|
||||
### Introduction
|
||||
The goal of enumeration value verification is to check whether the value of a field is within the range of enumeration values. If there is data that is not in the range of enumeration values and exceeds the threshold, the task will be judged to fail
|
||||
### UI Guide
|
||||
|
||||
- Source data type: select MySQL, PostgreSQL, etc.
|
||||
- Source data source: the corresponding data source under the source data type
|
||||
- Source data table: drop-down to select the table where the validation data is located
|
||||
- Src table filter conditions: such as title, also used when counting the total number of rows in the table, optional
|
||||
- Src table check column: drop-down to select the check column name
|
||||
- List of enumeration values: separated by commas
|
||||
- Check method:
|
||||
- [Expected-Actual]
|
||||
- [Actual-Expected]
|
||||
- [Actual/Expected]x100%
|
||||
- [(Expected-Actual)/Expected]x100%
|
||||
- Check operators: =, >, >=, <, <=, ! =
|
||||
- Threshold: The value used in the formula for comparison
|
||||
- Failure strategy
|
||||
- Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
|
||||
- Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
|
||||
- Expected value type: select the desired type from the drop-down menu
|
||||
## Table Count Check
|
||||
### Introduction
|
||||
The goal of table row number verification is to check whether the number of rows in the table reaches the expected value. If the number of rows does not meet the standard, the task will be judged as failed.
|
||||
### UI Guide
|
||||
|
||||
- Source data type: select MySQL, PostgreSQL, etc.
|
||||
- Source data source: the corresponding data source under the source data type
|
||||
- Source data table: drop-down to select the table where the validation data is located
|
||||
- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
|
||||
- Src table check column: drop-down to select the check column name
|
||||
- Check method:
|
||||
- [Expected-Actual]
|
||||
- [Actual-Expected]
|
||||
- [Actual/Expected]x100%
|
||||
- [(Expected-Actual)/Expected]x100%
|
||||
- Check operators: =, >, >=, <, <=, ! =
|
||||
- Threshold: The value used in the formula for comparison
|
||||
- Failure strategy
|
||||
- Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
|
||||
- Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
|
||||
- Expected value type: select the desired type from the drop-down menu
|
||||
|
||||
## Custom SQL Check
|
||||
### Introduction
|
||||
### UI Guide
|
||||
|
||||
- Source data type: select MySQL, PostgreSQL, etc.
|
||||
- Source data source: the corresponding data source under the source data type
|
||||
- Source data table: drop-down to select the table where the data to be verified is located
|
||||
- Actual value name: alias in SQL for statistical value calculation, such as max_num
|
||||
- Actual value calculation SQL: SQL for outputting actual values,
|
||||
- Note: The SQL must be statistical SQL, such as counting the number of rows, calculating the maximum value, minimum value, etc.
|
||||
- select max(a) as max_num from ${src_table}, the table name must be filled like this
|
||||
- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
|
||||
- Check method:
|
||||
- Check operators: =, >, >=, <, <=, ! =
|
||||
- Threshold: The value used in the formula for comparison
|
||||
- Failure strategy
|
||||
- Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
|
||||
- Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
|
||||
- Expected value type: select the desired type from the drop-down menu
|
||||
|
||||
## Accuracy check of multi-table
|
||||
### Introduction
|
||||
Accuracy checks are performed by comparing the accuracy differences of data records for selected fields between two tables, examples are as follows
|
||||
- table test1
|
||||
|
||||
| c1 | c2 |
|
||||
| :---: | :---: |
|
||||
| a | 1 |
|
||||
|b|2|
|
||||
|
||||
- table test2
|
||||
|
||||
| c21 | c22 |
|
||||
| :---: | :---: |
|
||||
| a | 1 |
|
||||
|b|3|
|
||||
If you compare the data in c1 and c21, the tables test1 and test2 are exactly the same. If you compare c2 and c22, the data in table test1 and table test2 are inconsistent.
|
||||
### UI Guide
|
||||
|
||||
- Source data type: select MySQL, PostgreSQL, etc.
|
||||
- Source data source: the corresponding data source under the source data type
|
||||
- Source data table: drop-down to select the table where the data to be verified is located
|
||||
- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
|
||||
- Target data type: choose MySQL, PostgreSQL, etc.
|
||||
- Target data source: the corresponding data source under the source data type
|
||||
- Target data table: drop-down to select the table where the data to be verified is located
|
||||
- Target filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
|
||||
- Check column:
|
||||
- Fill in the source data column, operator and target data column respectively
|
||||
- Verification method: select the desired verification method
|
||||
- Operators: =, >, >=, <, <=, ! =
|
||||
- Failure strategy
|
||||
- Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
|
||||
- Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
|
||||
- Expected value type: select the desired type in the drop-down menu, only SrcTableTotalRow, TargetTableTotalRow and fixed value are suitable for selection here
|
||||
## Comparison of the values checked by the two tables
|
||||
### Introduction
|
||||
Two-table value comparison allows users to customize different SQL statistics for two tables and compare the corresponding values. For example, for the source table A, the total amount of a certain column is calculated, and for the target table, the total amount of a certain column is calculated. value sum2, compare sum1 and sum2 to determine the check result
|
||||
### UI Guide
|
||||
|
||||
- Source data type: select MySQL, PostgreSQL, etc.
|
||||
- Source data source: the corresponding data source under the source data type
|
||||
- Source data table: the table where the data is to be verified
|
||||
- Actual value name: Calculate the alias in SQL for the actual value, such as max_age1
|
||||
- Actual value calculation SQL: SQL for outputting actual values,
|
||||
- Note: The SQL must be statistical SQL, such as counting the number of rows, calculating the maximum value, minimum value, etc.
|
||||
- select max(age) as max_age1 from ${src_table} The table name must be filled like this
|
||||
- Target data type: choose MySQL, PostgreSQL, etc.
|
||||
- Target data source: the corresponding data source under the source data type
|
||||
- Target data table: the table where the data is to be verified
|
||||
- Expected value name: Calculate the alias in SQL for the expected value, such as max_age2
|
||||
- Expected value calculation SQL: SQL for outputting expected value,
|
||||
- Note: The SQL must be statistical SQL, such as counting the number of rows, calculating the maximum value, minimum value, etc.
|
||||
- select max(age) as max_age2 from ${target_table} The table name must be filled like this
|
||||
- Verification method: select the desired verification method
|
||||
- Operators: =, >, >=, <, <=, ! =
|
||||
- Failure strategy
|
||||
- Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
|
||||
- Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
|
||||
|
||||
## Task result view
|
||||
|
||||
## Rule View
|
||||
### List of rules
|
||||
|
||||
### Rules Details
|
||||
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
# HIVE
|
||||
|
||||
## Use HiveServer2
|
||||
|
||||
|
||||
|
||||
- Datasource: select `HIVE`
|
||||
- Datasource name: enter the name of the DataSource
|
||||
- Description: enter a description of the DataSource
|
||||
- IP/Host Name: enter the HIVE service IP
|
||||
- Port: enter the HIVE service port
|
||||
- Username: set the username for HIVE connection
|
||||
- Password: set the password for HIVE connection
|
||||
- Database name: enter the database name of the HIVE connection
|
||||
- Jdbc connection parameters: parameter settings for HIVE connection, in JSON format
|
||||
|
||||
> NOTICE: If you wish to execute multiple HIVE SQL in the same session, you could set `support.hive.oneSession = true` in `common.properties`.
|
||||
> It is helpful when you try to set env variables before running HIVE SQL. Default value of `support.hive.oneSession` is `false` and multi-SQLs run in different sessions.
|
||||
|
||||
## Use HiveServer2 HA ZooKeeper
|
||||
|
||||
|
||||
|
||||
NOTICE: If Kerberos is disabled, ensure the parameter `hadoop.security.authentication.startup.state` is false, and parameter `java.security.krb5.conf.path` value sets null.
|
||||
If **Kerberos** is enabled, needs to set the following parameters in `common.properties`:
|
||||
|
||||
```conf
|
||||
# whether to startup kerberos
|
||||
hadoop.security.authentication.startup.state=true
|
||||
|
||||
# java.security.krb5.conf path
|
||||
java.security.krb5.conf.path=/opt/krb5.conf
|
||||
|
||||
# login user from keytab username
|
||||
login.user.keytab.username=hdfs-mycluster@ESZ.COM
|
||||
|
||||
# login user from keytab path
|
||||
login.user.keytab.path=/opt/hdfs.headless.keytab
|
||||
```
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
# DataSource
|
||||
|
||||
DataSource supports MySQL, PostgreSQL, Hive/Impala, Spark, ClickHouse, Oracle, SQL Server and other DataSources.
|
||||
|
||||
- Click "Data Source Center -> Create Data Source" to create different types of DataSources according to requirements.
|
||||
- Click "Test Connection" to test whether the DataSource can connect successfully.
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
# MySQL
|
||||
|
||||
|
||||
|
||||
- Datasource: select MYSQL
|
||||
- Datasource name: enter the name of the DataSource
|
||||
- Description: enter a description of the DataSource
|
||||
- IP/Host Name: enter the MYSQL service IP
|
||||
- Port: enter the MYSQL service port
|
||||
- Username: set the username for MYSQL connection
|
||||
- Password: set the password for MYSQL connection
|
||||
- Database name: enter the database name of the MYSQL connection
|
||||
- Jdbc connection parameters: parameter settings for MYSQL connection, in JSON format
|
||||
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
# PostgreSQL
|
||||
|
||||
|
||||
|
||||
- Datasource: select POSTGRESQL
|
||||
- Datasource name: enter the name of the DataSource
|
||||
- Description: enter a description of the DataSource
|
||||
- IP/Host Name: enter the PostgreSQL service IP
|
||||
- Port: enter the PostgreSQL service port
|
||||
- Username: set the username for PostgreSQL connection
|
||||
- Password: set the password for PostgreSQL connection
|
||||
- Database name: enter the database name of the PostgreSQL connection
|
||||
- Jdbc connection parameters: parameter settings for PostgreSQL connection, in JSON format
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
# Spark
|
||||
|
||||
|
||||
|
||||
- Datasource: select Spark
|
||||
- Datasource name: enter the name of the DataSource
|
||||
- Description: enter a description of the DataSource
|
||||
- IP/Host Name: enter the Spark service IP
|
||||
- Port: enter the Spark service port
|
||||
- Username: set the username for Spark connection
|
||||
- Password: set the password for Spark connection
|
||||
- Database name: enter the database name of the Spark connection
|
||||
- Jdbc connection parameters: parameter settings for Spark connection, in JSON format
|
||||
|
|
@ -1,248 +0,0 @@
|
|||
# DolphinScheduler Expansion and Reduction
|
||||
|
||||
## Expansion
|
||||
|
||||
This article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.
|
||||
|
||||
```
|
||||
Attention: There cannot be more than one master service process or worker service process on a physical machine.
|
||||
If the physical machine which locate the expansion master or worker node has already installed the scheduled service, check the [1.4 Modify configuration] and edit the configuration file `conf/config/install_config.conf` on ** all ** nodes, add masters or workers parameter, and restart the scheduling cluster.
|
||||
```
|
||||
|
||||
### Basic software installation
|
||||
|
||||
* [required] [JDK](https://www.oracle.com/technetwork/java/javase/downloads/index.html) (version 1.8+): must install, install and configure `JAVA_HOME` and `PATH` variables under `/etc/profile`
|
||||
* [optional] If the expansion is a worker node, you need to consider whether to install an external client, such as Hadoop, Hive, Spark Client.
|
||||
|
||||
|
||||
```markdown
|
||||
Attention: DolphinScheduler itself does not depend on Hadoop, Hive, Spark, but will only call their Client for the corresponding task submission.
|
||||
```
|
||||
|
||||
### Get Installation Package
|
||||
|
||||
- Check the version of DolphinScheduler used in your existing environment, and get the installation package of the corresponding version, if the versions are different, there may be compatibility problems.
|
||||
- Confirm the unified installation directory of other nodes, this article assumes that DolphinScheduler is installed in `/opt/` directory, and the full path is `/opt/dolphinscheduler`.
|
||||
- Please download the corresponding version of the installation package to the server installation directory, uncompress it and rename it to `dolphinscheduler` and store it in the `/opt` directory.
|
||||
- Add database dependency package, this document uses Mysql database, add `mysql-connector-java` driver package to `/opt/dolphinscheduler/lib` directory.
|
||||
|
||||
```shell
|
||||
# create the installation directory, please do not create the installation directory in /root, /home and other high privilege directories
|
||||
mkdir -p /opt
|
||||
cd /opt
|
||||
# decompress
|
||||
tar -zxvf apache-dolphinscheduler-<version>-bin.tar.gz -C /opt
|
||||
cd /opt
|
||||
mv apache-dolphinscheduler-<version>-bin dolphinscheduler
|
||||
```
|
||||
|
||||
```markdown
|
||||
Attention: You can copy the installation package directly from an existing environment to an expanded physical machine.
|
||||
```
|
||||
|
||||
### Create Deployment Users
|
||||
|
||||
- Create deployment user on **all** expansion machines, and make sure to configure sudo-free. If we plan to deploy scheduling on four expansion machines, ds1, ds2, ds3, and ds4, create deployment users on each machine is prerequisite.
|
||||
|
||||
```shell
|
||||
# to create a user, you need to log in with root and set the deployment user name, modify it by yourself, the following take `dolphinscheduler` as an example:
|
||||
useradd dolphinscheduler;
|
||||
|
||||
# set the user password, please change it by yourself, the following take `dolphinscheduler123` as an example
|
||||
echo "dolphinscheduler123" | passwd --stdin dolphinscheduler
|
||||
|
||||
# configure sudo password-free
|
||||
echo 'dolphinscheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL' >> /etc/sudoers
|
||||
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
|
||||
|
||||
```
|
||||
|
||||
```markdown
|
||||
Attention:
|
||||
- Since it is `sudo -u {linux-user}` to switch between different Linux users to run multi-tenant jobs, the deploying user needs to have sudo privileges and be password free.
|
||||
- If you find the line `Default requiretty` in the `/etc/sudoers` file, please also comment it out.
|
||||
- If have needs to use resource uploads, you also need to assign read and write permissions to the deployment user on `HDFS or MinIO`.
|
||||
```
|
||||
|
||||
### Modify Configuration
|
||||
|
||||
- From an existing node such as `Master/Worker`, copy the configuration directory directly to replace the configuration directory in the new node. After finishing the file copy, check whether the configuration items are correct.
|
||||
|
||||
```markdown
|
||||
Highlights:
|
||||
datasource.properties: database connection information
|
||||
zookeeper.properties: information for connecting zk
|
||||
common.properties: Configuration information about the resource store (if hadoop is set up, please check if the core-site.xml and hdfs-site.xml configuration files exist).
|
||||
dolphinscheduler_env.sh: environment Variables
|
||||
````
|
||||
|
||||
- Modify the `dolphinscheduler_env.sh` environment variable in the `bin/env/dolphinscheduler_env.sh` directory according to the machine configuration (the following is the example that all the used software install under `/opt/soft`)
|
||||
|
||||
```shell
|
||||
export HADOOP_HOME=/opt/soft/hadoop
|
||||
export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
|
||||
# export SPARK_HOME1=/opt/soft/spark1
|
||||
export SPARK_HOME2=/opt/soft/spark2
|
||||
export PYTHON_HOME=/opt/soft/python
|
||||
export JAVA_HOME=/opt/soft/jav
|
||||
export HIVE_HOME=/opt/soft/hive
|
||||
export FLINK_HOME=/opt/soft/flink
|
||||
export DATAX_HOME=/opt/soft/datax/bin/datax.py
|
||||
export PATH=$HADOOP_HOME/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
|
||||
|
||||
```
|
||||
|
||||
`Attention: This step is very important, such as `JAVA_HOME` and `PATH` is necessary to configure if haven not used just ignore or comment out`
|
||||
|
||||
|
||||
- Soft link the `JDK` to `/usr/bin/java` (still using `JAVA_HOME=/opt/soft/java` as an example)
|
||||
|
||||
```shell
|
||||
sudo ln -s /opt/soft/java/bin/java /usr/bin/java
|
||||
```
|
||||
|
||||
- Modify the configuration file `conf/config/install_config.conf` on the **all** nodes, synchronizing the following configuration.
|
||||
|
||||
* To add a new master node, you need to modify the IPs and masters parameters.
|
||||
* To add a new worker node, modify the IPs and workers parameters.
|
||||
|
||||
```shell
|
||||
# which machines to deploy DS services on, separated by commas between multiple physical machines
|
||||
ips="ds1,ds2,ds3,ds4"
|
||||
|
||||
# ssh port,default 22
|
||||
sshPort="22"
|
||||
|
||||
# which machine the master service is deployed on
|
||||
masters="existing master01,existing master02,ds1,ds2"
|
||||
|
||||
# the worker service is deployed on which machine, and specify the worker belongs to which worker group, the following example of "default" is the group name
|
||||
workers="existing worker01:default,existing worker02:default,ds3:default,ds4:default"
|
||||
|
||||
```
|
||||
- If the expansion is for worker nodes, you need to set the worker group, refer to the security of the [Worker grouping](./security.md)
|
||||
|
||||
- On all new nodes, change the directory permissions so that the deployment user has access to the DolphinScheduler directory
|
||||
|
||||
```shell
|
||||
sudo chown -R dolphinscheduler:dolphinscheduler dolphinscheduler
|
||||
```
|
||||
|
||||
### Restart the Cluster and Verify
|
||||
|
||||
- Restart the cluster
|
||||
|
||||
```shell
|
||||
# stop command:
|
||||
|
||||
bin/stop-all.sh # stop all services
|
||||
|
||||
sh bin/dolphinscheduler-daemon.sh stop master-server # stop master service
|
||||
sh bin/dolphinscheduler-daemon.sh stop worker-server # stop worker service
|
||||
sh bin/dolphinscheduler-daemon.sh stop api-server # stop api service
|
||||
sh bin/dolphinscheduler-daemon.sh stop alert-server # stop alert service
|
||||
|
||||
|
||||
# start command::
|
||||
bin/start-all.sh # start all services
|
||||
|
||||
sh bin/dolphinscheduler-daemon.sh start master-server # start master service
|
||||
sh bin/dolphinscheduler-daemon.sh start worker-server # start worker service
|
||||
sh bin/dolphinscheduler-daemon.sh start api-server # start api service
|
||||
sh bin/dolphinscheduler-daemon.sh start alert-server # start alert service
|
||||
|
||||
```
|
||||
|
||||
```
|
||||
Attention: When using `stop-all.sh` or `stop-all.sh`, if the physical machine execute the command is not configured to be ssh-free on all machines, it will prompt to enter the password
|
||||
```
|
||||
|
||||
- After completing the script, use the `jps` command to see if every node service is started (`jps` comes with the `Java JDK`)
|
||||
|
||||
```
|
||||
MasterServer ----- master service
|
||||
WorkerServer ----- worker service
|
||||
ApiApplicationServer ----- api service
|
||||
AlertServer ----- alert service
|
||||
```
|
||||
|
||||
After successful startup, you can view the logs, which are stored in the `logs` folder.
|
||||
|
||||
```Log Path
|
||||
logs/
|
||||
├── dolphinscheduler-alert-server.log
|
||||
├── dolphinscheduler-master-server.log
|
||||
├── dolphinscheduler-worker-server.log
|
||||
├── dolphinscheduler-api-server.log
|
||||
```
|
||||
|
||||
If the above services start normally and the scheduling system page is normal, check whether there is an expanded Master or Worker service in the [Monitor] of the web system. If it exists, the expansion is complete.
|
||||
|
||||
-----------------------------------------------------------------------------
|
||||
|
||||
## Reduction
|
||||
|
||||
The reduction is to reduce the master or worker services for the existing DolphinScheduler cluster.
|
||||
There are two steps for shrinking. After performing the following two steps, the shrinking operation can be completed.
|
||||
|
||||
### Stop the Service on the Scaled-Down Node
|
||||
|
||||
* If you are scaling down the master node, identify the physical machine where the master service is located, and stop the master service on the physical machine.
|
||||
* If scale down the worker node, determine the physical machine where the worker service scale down and stop the worker services on the physical machine.
|
||||
|
||||
```shell
|
||||
# stop command:
|
||||
bin/stop-all.sh # stop all services
|
||||
|
||||
sh bin/dolphinscheduler-daemon.sh stop master-server # stop master service
|
||||
sh bin/dolphinscheduler-daemon.sh stop worker-server # stop worker service
|
||||
sh bin/dolphinscheduler-daemon.sh stop api-server # stop api service
|
||||
sh bin/dolphinscheduler-daemon.sh stop alert-server # stop alert service
|
||||
|
||||
|
||||
# start command:
|
||||
bin/start-all.sh # start all services
|
||||
|
||||
sh bin/dolphinscheduler-daemon.sh start master-server # start master service
|
||||
sh bin/dolphinscheduler-daemon.sh start worker-server # start worker service
|
||||
sh bin/dolphinscheduler-daemon.sh start api-server # start api service
|
||||
sh bin/dolphinscheduler-daemon.sh start alert-server # start alert service
|
||||
|
||||
```
|
||||
|
||||
```
|
||||
Attention: When using `stop-all.sh` or `stop-all.sh`, if the machine without the command is not configured to be ssh-free for all machines, it will prompt to enter the password
|
||||
```
|
||||
|
||||
- After the script is completed, use the `jps` command to see if every node service was successfully shut down (`jps` comes with the `Java JDK`)
|
||||
|
||||
```
|
||||
MasterServer ----- master service
|
||||
WorkerServer ----- worker service
|
||||
ApiApplicationServer ----- api service
|
||||
AlertServer ----- alert service
|
||||
```
|
||||
If the corresponding master service or worker service does not exist, then the master or worker service is successfully shut down.
|
||||
|
||||
|
||||
### Modify the Configuration File
|
||||
|
||||
- modify the configuration file `conf/config/install_config.conf` on the **all** nodes, synchronizing the following configuration.
|
||||
|
||||
* to scale down the master node, modify the IPs and masters parameters.
|
||||
* to scale down worker nodes, modify the IPs and workers parameters.
|
||||
|
||||
```shell
|
||||
# which machines to deploy DS services on, "localhost" for this machine
|
||||
ips="ds1,ds2,ds3,ds4"
|
||||
|
||||
# ssh port,default: 22
|
||||
sshPort="22"
|
||||
|
||||
# which machine the master service is deployed on
|
||||
masters="existing master01,existing master02,ds1,ds2"
|
||||
|
||||
# The worker service is deployed on which machine, and specify which worker group this worker belongs to, the following example of "default" is the group name
|
||||
workers="existing worker01:default,existing worker02:default,ds3:default,ds4:default"
|
||||
|
||||
```
|
||||
|
|
@ -1,123 +0,0 @@
|
|||
# Flink Calls Operating Steps
|
||||
|
||||
## Create a Queue
|
||||
|
||||
1. Log in to the scheduling system, click `Security`, then click `Queue manage` on the left, and click `Create queue` to create a queue.
|
||||
2. Fill in the name and value of the queue, and click "Submit"
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/create_queue.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## Create a Tenant
|
||||
|
||||
```
|
||||
1. The tenant corresponds to a Linux user, which the user worker uses to submit jobs. If the Linux OS environment does not have this user, the worker will create this user when executing the script.
|
||||
2. Both the tenant and the tenant code are unique and cannot be repeated, just like a person only has one name and one ID number.
|
||||
3. After creating a tenant, there will be a folder in the HDFS relevant directory.
|
||||
```
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/create_tenant.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## Create a User
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/create_user.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## Create a Token
|
||||
|
||||
1. Log in to the scheduling system, click `Security`, then click `Token manage` on the left, and click `Create token` to create a token.
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/token-management-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
|
||||
2. Select the `Expiration time` (token validity time), select `User` (choose the specified user to perform the API operation), click "Generate token", copy the `Token` string, and click "Submit".
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/create-token-en1.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## Token Usage
|
||||
|
||||
1. Open the API documentation page
|
||||
|
||||
> Address：http://{api server ip}:12345/dolphinscheduler/doc.html?language=en_US&lang=en
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api-documentation-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
|
||||
2. Select a test API, the API selected for this test is `queryAllProjectList`
|
||||
|
||||
> projects/query-project-list
|
||||
|
||||
3. Open `Postman`, fill in the API address, and enter the `Token` in `Headers`, and then send the request to view the result:
|
||||
|
||||
```
|
||||
token: The Token just generated
|
||||
```
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/test-api.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## User Authorization
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/user_authorization.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## User Login
|
||||
|
||||
```
|
||||
http://192.168.1.163:12345/dolphinscheduler/ui/#/monitor/servers/master
|
||||
```
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/user_login.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## Upload the Resource
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/upload_resource.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## Create a Workflow
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/create_workflow1.png" width="80%" />
|
||||
</p>
|
||||
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/create_workflow2.png" width="80%" />
|
||||
</p>
|
||||
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/create_workflow3.png" width="80%" />
|
||||
</p>
|
||||
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/create_workflow4.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## View the Execution Result
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/execution_result.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## View Log
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/log.png" width="80%" />
|
||||
</p>
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
# Home Page
|
||||
|
||||
The home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.
|
||||
|
||||
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
# General Setting
|
||||
|
||||
## Language
|
||||
|
||||
DolphinScheduler supports two types of built-in language which include `English` and `Chinese`. You could click the button
|
||||
on the top control bar named `English` and `Chinese` and change it to another one when you want to switch the language.
|
||||
The entire DolphinScheduler page language will shift when you switch the language selection.
|
||||
|
||||
## Theme
|
||||
|
||||
DolphinScheduler supports two types of built-in theme which include `Dark` and `Light`. When you want to change the theme
|
||||
of DolphinScheduler, all you have to do is click the button named `Dark`(or `Light`) on the top control bar and on the left
|
||||
of to [language](#language) control button.
|
||||
|
||||
## Time Zone
|
||||
|
||||
DolphinScheduler support time zone setting.
|
||||
|
||||
Server Time Zone
|
||||
|
||||
The default time zone is UTC when using `bin/dolphinshceduler_daemon.sh` to start the server, you could update `SPRING_JACKSON_TIME_ZONE` in `bin/env/dolphinscheduler_env.sh`, such as `export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-Asia/Shanghai}`.<br>
|
||||
If you start server in IDEA, the default time zone is your local time zone, you could add the JVM parameter to update server time zone, such as `-Duser.timezone=UTC`. Time zone list refer to [List of tz database time zones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
|
||||
|
||||
User Time zone
|
||||
|
||||
The user's default time zone is based on the time zone which you run the DolphinScheduler service.You could
|
||||
click the button on the right of the [language](#language) button and then click `Choose timeZone` to choose the time zone
|
||||
you want to switch. All time related components will adjust their time zone according to the time zone setting you select.
|
||||
|
||||
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
# Cluster Deployment
|
||||
|
||||
Cluster deployment is to deploy the DolphinScheduler on multiple machines for running massive tasks in production.
|
||||
|
||||
If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md).
|
||||
|
||||
## Deployment Steps
|
||||
|
||||
Cluster deployment uses the same scripts and configuration files as [pseudo-cluster deployment](pseudo-cluster.md), so the preparation and deployment steps are the same as pseudo-cluster deployment. The difference is that pseudo-cluster deployment is for one machine, while cluster deployment (Cluster) is for multiple machines. And steps of "Modify Configuration" are quite different between pseudo-cluster deployment and cluster deployment.
|
||||
|
||||
### Prerequisites and DolphinScheduler Startup Environment Preparations
|
||||
|
||||
Configure all the configurations refer to [pseudo-cluster deployment](pseudo-cluster.md) on every machine, except sections `Prerequisites`, `Start ZooKeeper` and `Initialize the Database` of the `DolphinScheduler Startup Environment`.
|
||||
|
||||
### Modify Configuration
|
||||
|
||||
This step differs quite a lot from [pseudo-cluster deployment](pseudo-cluster.md), because the deployment script transfers the required resources for installation to each deployment machine by using `scp`. So we only need to modify the configuration of the machine that runs `install.sh` script and configurations will dispatch to cluster by `scp`. The configuration file is under the path `conf/config/install_config.conf`, here we only need to modify section **INSTALL MACHINE**, **DolphinScheduler ENV, Database, Registry Server** and keep other sections the same as [pseudo-cluster deployment](pseudo-cluster .md), the following describes the parameters that must be modified:
|
||||
|
||||
```shell
|
||||
# ---------------------------------------------------------
|
||||
# INSTALL MACHINE
|
||||
# ---------------------------------------------------------
|
||||
# Using IP or machine hostname for the server going to deploy master, worker, API server, the IP of the server
|
||||
# If you using a hostname, make sure machines could connect each other by hostname
|
||||
# As below, the hostname of the machine deploying DolphinScheduler is ds1, ds2, ds3, ds4, ds5, where ds1, ds2 install the master server, ds3, ds4, and ds5 installs worker server, the alert server is installed in ds4, and the API server is installed in ds5
|
||||
ips="ds1,ds2,ds3,ds4,ds5"
|
||||
masters="ds1,ds2"
|
||||
workers="ds3:default,ds4:default,ds5:default"
|
||||
alertServer="ds4"
|
||||
apiServers="ds5"
|
||||
```
|
||||
|
||||
## Start and Login DolphinScheduler
|
||||
|
||||
Same as [pseudo-cluster](pseudo-cluster.md)
|
||||
|
||||
## Start and Stop Server
|
||||
|
||||
Same as [pseudo-cluster](pseudo-cluster.md)
|
||||
|
|
@ -1,754 +0,0 @@
|
|||
# QuickStart in Kubernetes
|
||||
|
||||
Kubernetes deployment is DolphinScheduler deployment in a Kubernetes cluster, which can schedule massive tasks and can be used in production.
|
||||
|
||||
If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [Helm](https://helm.sh/) version 3.1.0+
|
||||
- [Kubernetes](https://kubernetes.io/) version 1.12+
|
||||
- PV provisioner support in the underlying infrastructure
|
||||
|
||||
## Install DolphinScheduler
|
||||
|
||||
Please download the source code package `apache-dolphinscheduler-<version>-src.tar.gz`, download address: [download address](/en-us/download/download.html)
|
||||
|
||||
To publish the release name `dolphinscheduler` version, please execute the following commands:
|
||||
|
||||
```
|
||||
$ tar -zxvf apache-dolphinscheduler-<version>-src.tar.gz
|
||||
$ cd apache-dolphinscheduler-<version>-src/docker/kubernetes/dolphinscheduler
|
||||
$ helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
$ helm dependency update .
|
||||
$ helm install dolphinscheduler . --set image.tag=<version>
|
||||
```
|
||||
|
||||
To publish the release name `dolphinscheduler` version to `test` namespace:
|
||||
|
||||
```bash
|
||||
$ helm install dolphinscheduler . -n test
|
||||
```
|
||||
|
||||
> **Tip**: If a namespace named `test` is used, the optional parameter `-n test` needs to be added to the `helm` and `kubectl` commands.
|
||||
|
||||
These commands are used to deploy DolphinScheduler on the Kubernetes cluster by default. The [Appendix-Configuration](#appendix-configuration) section lists the parameters that can be configured during installation.
|
||||
|
||||
> **Tip**: List all releases using `helm list`
|
||||
|
||||
The **PostgreSQL** (with username `root`, password `root` and database `dolphinscheduler`) and **ZooKeeper** services will start by default.
|
||||
|
||||
## Access DolphinScheduler UI
|
||||
|
||||
If `ingress.enabled` in `values.yaml` is set to `true`, you could access `http://${ingress.host}/dolphinscheduler` in browser.
|
||||
|
||||
> **Tip**: If there is a problem with ingress access, please contact the Kubernetes administrator and refer to the [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/).
|
||||
|
||||
Otherwise, when `api.service.type=ClusterIP` you need to execute `port-forward` commands:
|
||||
|
||||
```bash
|
||||
$ kubectl port-forward --address 0.0.0.0 svc/dolphinscheduler-api 12345:12345
|
||||
$ kubectl port-forward --address 0.0.0.0 -n test svc/dolphinscheduler-api 12345:12345 # with test namespace
|
||||
```
|
||||
|
||||
> **Tip**: If the error of `unable to do port forwarding: socat not found` appears, you need to install `socat` first.
|
||||
|
||||
Access the web: `http://localhost:12345/dolphinscheduler` (Modify the IP address if needed).
|
||||
|
||||
Or when `api.service.type=NodePort` you need to execute the command:
|
||||
|
||||
```bash
|
||||
NODE_IP=$(kubectl get no -n {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
|
||||
NODE_PORT=$(kubectl get svc {{ template "dolphinscheduler.fullname" . }}-api -n {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}")
|
||||
echo http://$NODE_IP:$NODE_PORT/dolphinscheduler
|
||||
```
|
||||
|
||||
Access the web: `http://$NODE_IP:$NODE_PORT/dolphinscheduler`.
|
||||
|
||||
The default username is `admin` and the default password is `dolphinscheduler123`.
|
||||
|
||||
Please refer to the `Quick Start` in the chapter [Quick Start](../start/quick-start.md) to explore how to use DolphinScheduler.
|
||||
|
||||
## Uninstall the Chart
|
||||
|
||||
To uninstall or delete the `dolphinscheduler` deployment:
|
||||
|
||||
```bash
|
||||
$ helm uninstall dolphinscheduler
|
||||
```
|
||||
|
||||
The command removes all the Kubernetes components (except PVC) associated with the `dolphinscheduler` and deletes the release.
|
||||
|
||||
Run the command below to delete the PVC's associated with `dolphinscheduler`:
|
||||
|
||||
```bash
|
||||
$ kubectl delete pvc -l app.kubernetes.io/instance=dolphinscheduler
|
||||
```
|
||||
|
||||
> **Note**: Deleting the PVC's will delete all data as well. Please be cautious before doing it.
|
||||
|
||||
## Configuration
|
||||
|
||||
The configuration file is `values.yaml`, and the [Appendix-Configuration](#appendix-configuration) tables lists the configurable parameters of the DolphinScheduler and their default values.
|
||||
|
||||
## Support Matrix
|
||||
|
||||
| Type | Support | Notes |
|
||||
| ------------------------------------------------------------ | ------------ | ------------------------------------- |
|
||||
| Shell | Yes | |
|
||||
| Python2 | Yes | |
|
||||
| Python3 | Indirect Yes | Refer to FAQ |
|
||||
| Hadoop2 | Indirect Yes | Refer to FAQ |
|
||||
| Hadoop3 | Not Sure | Not tested |
|
||||
| Spark-Local(client) | Indirect Yes | Refer to FAQ |
|
||||
| Spark-YARN(cluster) | Indirect Yes | Refer to FAQ |
|
||||
| Spark-Standalone(cluster) | Not Yet | |
|
||||
| Spark-Kubernetes(cluster) | Not Yet | |
|
||||
| Flink-Local(local>=1.11) | Not Yet | Generic CLI mode is not yet supported |
|
||||
| Flink-YARN(yarn-cluster) | Indirect Yes | Refer to FAQ |
|
||||
| Flink-YARN(yarn-session/yarn-per-job/yarn-application>=1.11) | Not Yet | Generic CLI mode is not yet supported |
|
||||
| Flink-Standalone(default) | Not Yet | |
|
||||
| Flink-Standalone(remote>=1.11) | Not Yet | Generic CLI mode is not yet supported |
|
||||
| Flink-Kubernetes(default) | Not Yet | |
|
||||
| Flink-Kubernetes(remote>=1.11) | Not Yet | Generic CLI mode is not yet supported |
|
||||
| Flink-NativeKubernetes(kubernetes-session/application>=1.11) | Not Yet | Generic CLI mode is not yet supported |
|
||||
| MapReduce | Indirect Yes | Refer to FAQ |
|
||||
| Kerberos | Indirect Yes | Refer to FAQ |
|
||||
| HTTP | Yes | |
|
||||
| DataX | Indirect Yes | Refer to FAQ |
|
||||
| Sqoop | Indirect Yes | Refer to FAQ |
|
||||
| SQL-MySQL | Indirect Yes | Refer to FAQ |
|
||||
| SQL-PostgreSQL | Yes | |
|
||||
| SQL-Hive | Indirect Yes | Refer to FAQ |
|
||||
| SQL-Spark | Indirect Yes | Refer to FAQ |
|
||||
| SQL-ClickHouse | Indirect Yes | Refer to FAQ |
|
||||
| SQL-Oracle | Indirect Yes | Refer to FAQ |
|
||||
| SQL-SQLServer | Indirect Yes | Refer to FAQ |
|
||||
| SQL-DB2 | Indirect Yes | Refer to FAQ |
|
||||
|
||||
## FAQ
|
||||
|
||||
### How to View the Logs of a Pod Container?
|
||||
|
||||
List all pods (aka `po`):
|
||||
|
||||
```
|
||||
kubectl get po
|
||||
kubectl get po -n test # with test namespace
|
||||
```
|
||||
|
||||
View the logs of a pod container named `dolphinscheduler-master-0`:
|
||||
|
||||
```
|
||||
kubectl logs dolphinscheduler-master-0
|
||||
kubectl logs -f dolphinscheduler-master-0 # follow log output
|
||||
kubectl logs --tail 10 dolphinscheduler-master-0 -n test # show last 10 lines from the end of the logs
|
||||
```
|
||||
|
||||
### How to Scale API, master and worker on Kubernetes?
|
||||
|
||||
List all deployments (aka `deploy`):
|
||||
|
||||
```
|
||||
kubectl get deploy
|
||||
kubectl get deploy -n test # with test namespace
|
||||
```
|
||||
|
||||
Scale api to 3 replicas:
|
||||
|
||||
```
|
||||
kubectl scale --replicas=3 deploy dolphinscheduler-api
|
||||
kubectl scale --replicas=3 deploy dolphinscheduler-api -n test # with test namespace
|
||||
```
|
||||
|
||||
List all stateful sets (aka `sts`):
|
||||
|
||||
```
|
||||
kubectl get sts
|
||||
kubectl get sts -n test # with test namespace
|
||||
```
|
||||
|
||||
Scale master to 2 replicas:
|
||||
|
||||
```
|
||||
kubectl scale --replicas=2 sts dolphinscheduler-master
|
||||
kubectl scale --replicas=2 sts dolphinscheduler-master -n test # with test namespace
|
||||
```
|
||||
|
||||
Scale worker to 6 replicas:
|
||||
|
||||
```
|
||||
kubectl scale --replicas=6 sts dolphinscheduler-worker
|
||||
kubectl scale --replicas=6 sts dolphinscheduler-worker -n test # with test namespace
|
||||
```
|
||||
|
||||
### How to Use MySQL as the DolphinScheduler's Database Instead of PostgreSQL?
|
||||
|
||||
> Because of the commercial license, we cannot directly use the driver of MySQL.
|
||||
>
|
||||
> If you want to use MySQL, you can build a new image based on the `apache/dolphinscheduler` image follow the following instructions:
|
||||
|
||||
1. Download the MySQL driver [mysql-connector-java-8.0.16.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar).
|
||||
|
||||
2. Create a new `Dockerfile` to add MySQL driver:
|
||||
|
||||
```
|
||||
FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:<version>
|
||||
COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/lib
|
||||
```
|
||||
|
||||
3. Build a new docker image including MySQL driver:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:mysql-driver .
|
||||
```
|
||||
|
||||
4. Push the docker image `apache/dolphinscheduler:mysql-driver` to a docker registry.
|
||||
|
||||
5. Modify image `repository` and update `tag` to `mysql-driver` in `values.yaml`.
|
||||
|
||||
6. Modify postgresql `enabled` to `false` in `values.yaml`.
|
||||
|
||||
7. Modify externalDatabase (especially modify `host`, `username` and `password`) in `values.yaml`:
|
||||
|
||||
```yaml
|
||||
externalDatabase:
|
||||
type: "mysql"
|
||||
driver: "com.mysql.jdbc.Driver"
|
||||
host: "localhost"
|
||||
port: "3306"
|
||||
username: "root"
|
||||
password: "root"
|
||||
database: "dolphinscheduler"
|
||||
params: "useUnicode=true&characterEncoding=UTF-8"
|
||||
```
|
||||
|
||||
8. Run a DolphinScheduler release in Kubernetes (See **Install DolphinScheduler**).
|
||||
|
||||
### How to Support MySQL Datasource in `Datasource manage`?
|
||||
|
||||
> Because of the commercial license, we cannot directly use the driver of MySQL.
|
||||
>
|
||||
> If you want to add MySQL datasource, you can build a new image based on the `apache/dolphinscheduler` image follow the following instructions:
|
||||
|
||||
1. Download the MySQL driver [mysql-connector-java-8.0.16.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar).
|
||||
|
||||
2. Create a new `Dockerfile` to add MySQL driver:
|
||||
|
||||
```
|
||||
FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:<version>
|
||||
COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/lib
|
||||
```
|
||||
|
||||
3. Build a new docker image including MySQL driver:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:mysql-driver .
|
||||
```
|
||||
|
||||
4. Push the docker image `apache/dolphinscheduler:mysql-driver` to a docker registry.
|
||||
|
||||
5. Modify image `repository` and update `tag` to `mysql-driver` in `values.yaml`.
|
||||
|
||||
6. Run a DolphinScheduler release in Kubernetes (See **Install DolphinScheduler**).
|
||||
|
||||
7. Add a MySQL datasource in `Datasource manage`.
|
||||
|
||||
### How to Support Oracle Datasource in `Datasource manage`?
|
||||
|
||||
> Because of the commercial license, we cannot directly use the driver of Oracle.
|
||||
>
|
||||
> If you want to add Oracle datasource, you can build a new image based on the `apache/dolphinscheduler` image follow the following instructions:
|
||||
|
||||
1. Download the Oracle driver [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`)
|
||||
|
||||
2. Create a new `Dockerfile` to add Oracle driver:
|
||||
|
||||
```
|
||||
FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:<version>
|
||||
COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib
|
||||
```
|
||||
|
||||
3. Build a new docker image including Oracle driver:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:oracle-driver .
|
||||
```
|
||||
|
||||
4. Push the docker image `apache/dolphinscheduler:oracle-driver` to a docker registry.
|
||||
|
||||
5. Modify image `repository` and update `tag` to `oracle-driver` in `values.yaml`.
|
||||
|
||||
6. Run a DolphinScheduler release in Kubernetes (See **Install DolphinScheduler**).
|
||||
|
||||
7. Add an Oracle datasource in `Datasource manage`.
|
||||
|
||||
### How to Support Python 2 pip and Custom requirements.txt?
|
||||
|
||||
1. Create a new `Dockerfile` to install pip:
|
||||
|
||||
```
|
||||
FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:<version>
|
||||
COPY requirements.txt /tmp
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends python-pip && \
|
||||
pip install --no-cache-dir -r /tmp/requirements.txt && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
```
|
||||
|
||||
The command will install the default **pip 18.1**. If you upgrade the pip, just add the following command.
|
||||
|
||||
```
|
||||
pip install --no-cache-dir -U pip && \
|
||||
```
|
||||
|
||||
2. Build a new docker image including pip:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:pip .
|
||||
```
|
||||
|
||||
3. Push the docker image `apache/dolphinscheduler:pip` to a docker registry.
|
||||
|
||||
4. Modify image `repository` and update `tag` to `pip` in `values.yaml`.
|
||||
|
||||
5. Run a DolphinScheduler release in Kubernetes (See **Install DolphinScheduler**).
|
||||
|
||||
6. Verify pip under a new Python task.
|
||||
|
||||
### How to Support Python 3?
|
||||
|
||||
1. Create a new `Dockerfile` to install Python 3:
|
||||
|
||||
```
|
||||
FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:<version>
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends python3 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
```
|
||||
|
||||
The command will install the default **Python 3.7.3**. If you also want to install **pip3**, just replace `python3` with `python3-pip` like:
|
||||
|
||||
```
|
||||
apt-get install -y --no-install-recommends python3-pip && \
|
||||
```
|
||||
|
||||
2. Build a new docker image including Python 3:
|
||||
|
||||
```
|
||||
docker build -t apache/dolphinscheduler:python3 .
|
||||
```
|
||||
|
||||
3. Push the docker image `apache/dolphinscheduler:python3` to a docker registry.
|
||||
|
||||
4. Modify image `repository` and update `tag` to `python3` in `values.yaml`.
|
||||
|
||||
5. Modify `PYTHON_HOME` to `/usr/bin/python3` in `values.yaml`.
|
||||
|
||||
6. Run a DolphinScheduler release in Kubernetes (See **Install DolphinScheduler**).
|
||||
|
||||
7. Verify Python 3 under a new Python task.
|
||||
|
||||
### How to Support Hadoop, Spark, Flink, Hive or DataX?
|
||||
|
||||
Take Spark 2.4.7 as an example:
|
||||
|
||||
1. Download the Spark 2.4.7 release binary `spark-2.4.7-bin-hadoop2.7.tgz`.
|
||||
|
||||
2. Ensure that `common.sharedStoragePersistence.enabled` is turned on.
|
||||
|
||||
3. Run a DolphinScheduler release in Kubernetes (See **Install DolphinScheduler**).
|
||||
|
||||
4. Copy the Spark 2.4.7 release binary into the Docker container.
|
||||
|
||||
```bash
|
||||
kubectl cp spark-2.4.7-bin-hadoop2.7.tgz dolphinscheduler-worker-0:/opt/soft
|
||||
kubectl cp -n test spark-2.4.7-bin-hadoop2.7.tgz dolphinscheduler-worker-0:/opt/soft # with test namespace
|
||||
```
|
||||
|
||||
Because the volume `sharedStoragePersistence` is mounted on `/opt/soft`, all files in `/opt/soft` will not be lost.
|
||||
|
||||
5. Attach the container and ensure that `SPARK_HOME2` exists.
|
||||
|
||||
```bash
|
||||
kubectl exec -it dolphinscheduler-worker-0 bash
|
||||
kubectl exec -n test -it dolphinscheduler-worker-0 bash # with test namespace
|
||||
cd /opt/soft
|
||||
tar zxf spark-2.4.7-bin-hadoop2.7.tgz
|
||||
rm -f spark-2.4.7-bin-hadoop2.7.tgz
|
||||
ln -s spark-2.4.7-bin-hadoop2.7 spark2 # or just mv
|
||||
$SPARK_HOME2/bin/spark-submit --version
|
||||
```
|
||||
|
||||
The last command will print the Spark version if everything goes well.
|
||||
|
||||
6. Verify Spark under a Shell task.
|
||||
|
||||
```
|
||||
$SPARK_HOME2/bin/spark-submit --class org.apache.spark.examples.SparkPi $SPARK_HOME2/examples/jars/spark-examples_2.11-2.4.7.jar
|
||||
```
|
||||
|
||||
Check whether the task log contains the output like `Pi is roughly 3.146015`.
|
||||
|
||||
7. Verify Spark under a Spark task.
|
||||
|
||||
The file `spark-examples_2.11-2.4.7.jar` needs to be uploaded to the resources first, and then create a Spark task with:
|
||||
|
||||
- Spark Version: `SPARK2`
|
||||
- Main Class: `org.apache.spark.examples.SparkPi`
|
||||
- Main Package: `spark-examples_2.11-2.4.7.jar`
|
||||
- Deploy Mode: `local`
|
||||
|
||||
Similarly, check whether the task log contains the output like `Pi is roughly 3.146015`.
|
||||
|
||||
8. Verify Spark on YARN.
|
||||
|
||||
Spark on YARN (Deploy Mode is `cluster` or `client`) requires Hadoop support. Similar to Spark support, the operation of supporting Hadoop is almost the same as the previous steps.
|
||||
|
||||
Ensure that `$HADOOP_HOME` and `$HADOOP_CONF_DIR` exists.
|
||||
|
||||
### How to Support Spark 3?
|
||||
|
||||
In fact, the way to submit applications with `spark-submit` is the same, regardless of Spark 1, 2 or 3. In other words, the semantics of `SPARK_HOME2` is the second `SPARK_HOME` instead of `SPARK2`'s `HOME`, so just set `SPARK_HOME2=/path/to/spark3`.
|
||||
|
||||
Take Spark 3.1.1 as an example:
|
||||
|
||||
1. Download the Spark 3.1.1 release binary `spark-3.1.1-bin-hadoop2.7.tgz`.
|
||||
|
||||
2. Ensure that `common.sharedStoragePersistence.enabled` is turned on.
|
||||
|
||||
3. Run a DolphinScheduler release in Kubernetes (See **Install DolphinScheduler**).
|
||||
|
||||
4. Copy the Spark 3.1.1 release binary into the Docker container.
|
||||
|
||||
```bash
|
||||
kubectl cp spark-3.1.1-bin-hadoop2.7.tgz dolphinscheduler-worker-0:/opt/soft
|
||||
kubectl cp -n test spark-3.1.1-bin-hadoop2.7.tgz dolphinscheduler-worker-0:/opt/soft # with test namespace
|
||||
```
|
||||
|
||||
5. Attach the container and ensure that `SPARK_HOME2` exists.
|
||||
|
||||
```bash
|
||||
kubectl exec -it dolphinscheduler-worker-0 bash
|
||||
kubectl exec -n test -it dolphinscheduler-worker-0 bash # with test namespace
|
||||
cd /opt/soft
|
||||
tar zxf spark-3.1.1-bin-hadoop2.7.tgz
|
||||
rm -f spark-3.1.1-bin-hadoop2.7.tgz
|
||||
ln -s spark-3.1.1-bin-hadoop2.7 spark2 # or just mv
|
||||
$SPARK_HOME2/bin/spark-submit --version
|
||||
```
|
||||
|
||||
The last command will print the Spark version if everything goes well.
|
||||
|
||||
6. Verify Spark under a Shell task.
|
||||
|
||||
```
|
||||
$SPARK_HOME2/bin/spark-submit --class org.apache.spark.examples.SparkPi $SPARK_HOME2/examples/jars/spark-examples_2.12-3.1.1.jar
|
||||
```
|
||||
|
||||
Check whether the task log contains the output like `Pi is roughly 3.146015`.
|
||||
|
||||
### How to Support Shared Storage Between Master, Worker and Api Server?
|
||||
|
||||
For example, Master, Worker and API server may use Hadoop at the same time.
|
||||
|
||||
1. Modify the following configurations in `values.yaml`
|
||||
|
||||
```yaml
|
||||
common:
|
||||
sharedStoragePersistence:
|
||||
enabled: false
|
||||
mountPath: "/opt/soft"
|
||||
accessModes:
|
||||
- "ReadWriteMany"
|
||||
storageClassName: "-"
|
||||
storage: "20Gi"
|
||||
```
|
||||
|
||||
Modify `storageClassName` and `storage` to actual environment values.
|
||||
|
||||
> **Note**: `storageClassName` must support the access mode: `ReadWriteMany`.
|
||||
|
||||
2. Copy the Hadoop into the directory `/opt/soft`.
|
||||
|
||||
3. Ensure that `$HADOOP_HOME` and `$HADOOP_CONF_DIR` are correct.
|
||||
|
||||
### How to Support Local File Resource Storage Instead of HDFS and S3?
|
||||
|
||||
Modify the following configurations in `values.yaml`:
|
||||
|
||||
```yaml
|
||||
common:
|
||||
configmap:
|
||||
RESOURCE_STORAGE_TYPE: "HDFS"
|
||||
RESOURCE_UPLOAD_PATH: "/dolphinscheduler"
|
||||
FS_DEFAULT_FS: "file:///"
|
||||
fsFileResourcePersistence:
|
||||
enabled: true
|
||||
accessModes:
|
||||
- "ReadWriteMany"
|
||||
storageClassName: "-"
|
||||
storage: "20Gi"
|
||||
```
|
||||
|
||||
Modify `storageClassName` and `storage` to actual environment values.
|
||||
|
||||
> **Note**: `storageClassName` must support the access mode: `ReadWriteMany`.
|
||||
|
||||
### How to Support S3 Resource Storage Like MinIO?
|
||||
|
||||
Take MinIO as an example: Modify the following configurations in `values.yaml`:
|
||||
|
||||
```yaml
|
||||
common:
|
||||
configmap:
|
||||
RESOURCE_STORAGE_TYPE: "S3"
|
||||
RESOURCE_UPLOAD_PATH: "/dolphinscheduler"
|
||||
FS_DEFAULT_FS: "s3a://BUCKET_NAME"
|
||||
FS_S3A_ENDPOINT: "http://MINIO_IP:9000"
|
||||
FS_S3A_ACCESS_KEY: "MINIO_ACCESS_KEY"
|
||||
FS_S3A_SECRET_KEY: "MINIO_SECRET_KEY"
|
||||
```
|
||||
|
||||
Modify `BUCKET_NAME`, `MINIO_IP`, `MINIO_ACCESS_KEY` and `MINIO_SECRET_KEY` to actual environment values.
|
||||
|
||||
> **Note**: `MINIO_IP` can only use IP instead of the domain name, because DolphinScheduler currently doesn't support S3 path style access.
|
||||
|
||||
### How to Configure SkyWalking?
|
||||
|
||||
Modify SkyWalking configurations in `values.yaml`:
|
||||
|
||||
```yaml
|
||||
common:
|
||||
configmap:
|
||||
SKYWALKING_ENABLE: "true"
|
||||
SW_AGENT_COLLECTOR_BACKEND_SERVICES: "127.0.0.1:11800"
|
||||
SW_GRPC_LOG_SERVER_HOST: "127.0.0.1"
|
||||
SW_GRPC_LOG_SERVER_PORT: "11800"
|
||||
```
|
||||
|
||||
## Appendix-Configuration
|
||||
|
||||
| Parameter | Description | Default |
|
||||
| --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------- |
|
||||
| `timezone` | World time and date for cities in all time zones | `Asia/Shanghai` |
|
||||
| | | |
|
||||
| `image.repository` | Docker image repository for the DolphinScheduler | `apache/dolphinscheduler` |
|
||||
| `image.tag` | Docker image version for the DolphinScheduler | `latest` |
|
||||
| `image.pullPolicy` | Image pull policy. Options: Always, Never, IfNotPresent | `IfNotPresent` |
|
||||
| `image.pullSecret` | Image pull secret. An optional reference to secret in the same namespace to use for pulling any of the images | `nil` |
|
||||
| | | |
|
||||
| `postgresql.enabled` | If not exists external PostgreSQL, by default, the DolphinScheduler will use a internal PostgreSQL | `true` |
|
||||
| `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` |
|
||||
| `postgresql.postgresqlPassword` | The password for internal PostgreSQL | `root` |
|
||||
| `postgresql.postgresqlDatabase` | The database for internal PostgreSQL | `dolphinscheduler` |
|
||||
| `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` |
|
||||
| `postgresql.persistence.size` | `PersistentVolumeClaim` size | `20Gi` |
|
||||
| `postgresql.persistence.storageClass` | PostgreSQL data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database type will use it | `postgresql` |
|
||||
| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database driver will use it | `org.postgresql.Driver` |
|
||||
| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database host will use it | `localhost` |
|
||||
| `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database port will use it | `5432` |
|
||||
| `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database username will use it | `root` |
|
||||
| `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database password will use it | `root` |
|
||||
| `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database database will use it | `dolphinscheduler` |
|
||||
| `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database params will use it | `characterEncoding=utf8` |
|
||||
| | | |
|
||||
| `zookeeper.enabled` | If not exists external ZooKeeper, by default, the DolphinScheduler will use a internal ZooKeeper | `true` |
|
||||
| `zookeeper.fourlwCommandsWhitelist` | A list of comma separated Four Letter Words commands to use | `srvr,ruok,wchs,cons` |
|
||||
| `zookeeper.persistence.enabled` | Set `zookeeper.persistence.enabled` to `true` to mount a new volume for internal ZooKeeper | `false` |
|
||||
| `zookeeper.persistence.size` | `PersistentVolumeClaim` size | `20Gi` |
|
||||
| `zookeeper.persistence.storageClass` | ZooKeeper data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `zookeeper.zookeeperRoot` | Specify dolphinscheduler root directory in ZooKeeper | `/dolphinscheduler` |
|
||||
| `externalZookeeper.zookeeperQuorum` | If exists external ZooKeeper, and set `zookeeper.enabled` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` |
|
||||
| `externalZookeeper.zookeeperRoot` | If exists external ZooKeeper, and set `zookeeper.enabled` value to false. Specify dolphinscheduler root directory in Zookeeper | `/dolphinscheduler` |
|
||||
| | | |
|
||||
| `common.configmap.DOLPHINSCHEDULER_OPTS` | The jvm options for dolphinscheduler, suitable for all servers | `""` |
|
||||
| `common.configmap.DATA_BASEDIR_PATH` | User data directory path, self configuration, please make sure the directory exists and have read write permissions | `/tmp/dolphinscheduler` |
|
||||
| `common.configmap.RESOURCE_STORAGE_TYPE` | Resource storage type: HDFS, S3, NONE | `HDFS` |
|
||||
| `common.configmap.RESOURCE_UPLOAD_PATH` | Resource store on HDFS/S3 path, please make sure the directory exists on hdfs and have read write permissions | `/dolphinscheduler` |
|
||||
| `common.configmap.FS_DEFAULT_FS` | Resource storage file system like `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler` | `file:///` |
|
||||
| `common.configmap.FS_S3A_ENDPOINT` | S3 endpoint when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `s3.xxx.amazonaws.com` |
|
||||
| `common.configmap.FS_S3A_ACCESS_KEY` | S3 access key when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `xxxxxxx` |
|
||||
| `common.configmap.FS_S3A_SECRET_KEY` | S3 secret key when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `xxxxxxx` |
|
||||
| `common.configmap.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE` | Whether to startup kerberos | `false` |
|
||||
| `common.configmap.JAVA_SECURITY_KRB5_CONF_PATH` | The java.security.krb5.conf path | `/opt/krb5.conf` |
|
||||
| `common.configmap.LOGIN_USER_KEYTAB_USERNAME` | The login user from keytab username | `hdfs@HADOOP.COM` |
|
||||
| `common.configmap.LOGIN_USER_KEYTAB_PATH` | The login user from keytab path | `/opt/hdfs.keytab` |
|
||||
| `common.configmap.KERBEROS_EXPIRE_TIME` | The kerberos expire time, the unit is hour | `2` |
|
||||
| `common.configmap.HDFS_ROOT_USER` | The HDFS root user who must have the permission to create directories under the HDFS root path | `hdfs` |
|
||||
| `common.configmap.RESOURCE_MANAGER_HTTPADDRESS_PORT` | Set resource manager httpaddress port for yarn | `8088` |
|
||||
| `common.configmap.YARN_RESOURCEMANAGER_HA_RM_IDS` | If resourcemanager HA is enabled, please set the HA IPs | `nil` |
|
||||
| `common.configmap.YARN_APPLICATION_STATUS_ADDRESS` | If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname, otherwise keep default | `http://ds1:%s/ws/v1/cluster/apps/%s` |
|
||||
| `common.configmap.SKYWALKING_ENABLE` | Set whether to enable skywalking | `false` |
|
||||
| `common.configmap.SW_AGENT_COLLECTOR_BACKEND_SERVICES` | Set agent collector backend services for skywalking | `127.0.0.1:11800` |
|
||||
| `common.configmap.SW_GRPC_LOG_SERVER_HOST` | Set grpc log server host for skywalking | `127.0.0.1` |
|
||||
| `common.configmap.SW_GRPC_LOG_SERVER_PORT` | Set grpc log server port for skywalking | `11800` |
|
||||
| `common.configmap.HADOOP_HOME` | Set `HADOOP_HOME` for DolphinScheduler's task environment | `/opt/soft/hadoop` |
|
||||
| `common.configmap.HADOOP_CONF_DIR` | Set `HADOOP_CONF_DIR` for DolphinScheduler's task environment | `/opt/soft/hadoop/etc/hadoop` |
|
||||
| `common.configmap.SPARK_HOME1` | Set `SPARK_HOME1` for DolphinScheduler's task environment | `/opt/soft/spark1` |
|
||||
| `common.configmap.SPARK_HOME2` | Set `SPARK_HOME2` for DolphinScheduler's task environment | `/opt/soft/spark2` |
|
||||
| `common.configmap.PYTHON_HOME` | Set `PYTHON_HOME` for DolphinScheduler's task environment | `/usr/bin/python` |
|
||||
| `common.configmap.JAVA_HOME` | Set `JAVA_HOME` for DolphinScheduler's task environment | `/usr/local/openjdk-8` |
|
||||
| `common.configmap.HIVE_HOME` | Set `HIVE_HOME` for DolphinScheduler's task environment | `/opt/soft/hive` |
|
||||
| `common.configmap.FLINK_HOME` | Set `FLINK_HOME` for DolphinScheduler's task environment | `/opt/soft/flink` |
|
||||
| `common.configmap.DATAX_HOME` | Set `DATAX_HOME` for DolphinScheduler's task environment | `/opt/soft/datax` |
|
||||
| `common.sharedStoragePersistence.enabled` | Set `common.sharedStoragePersistence.enabled` to `true` to mount a shared storage volume for Hadoop, Spark binary and etc | `false` |
|
||||
| `common.sharedStoragePersistence.mountPath` | The mount path for the shared storage volume | `/opt/soft` |
|
||||
| `common.sharedStoragePersistence.accessModes` | `PersistentVolumeClaim` access modes, must be `ReadWriteMany` | `[ReadWriteMany]` |
|
||||
| `common.sharedStoragePersistence.storageClassName` | Shared Storage persistent volume storage class, must support the access mode: ReadWriteMany | `-` |
|
||||
| `common.sharedStoragePersistence.storage` | `PersistentVolumeClaim` size | `20Gi` |
|
||||
| `common.fsFileResourcePersistence.enabled` | Set `common.fsFileResourcePersistence.enabled` to `true` to mount a new file resource volume for `api` and `worker` | `false` |
|
||||
| `common.fsFileResourcePersistence.accessModes` | `PersistentVolumeClaim` access modes, must be `ReadWriteMany` | `[ReadWriteMany]` |
|
||||
| `common.fsFileResourcePersistence.storageClassName` | Resource persistent volume storage class, must support the access mode: ReadWriteMany | `-` |
|
||||
| `common.fsFileResourcePersistence.storage` | `PersistentVolumeClaim` size | `20Gi` |
|
||||
| | | |
|
||||
| `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
|
||||
| `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
|
||||
| `master.annotations` | The `annotations` for master server | `{}` |
|
||||
| `master.affinity` | If specified, the pod's scheduling constraints | `{}` |
|
||||
| `master.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
|
||||
| `master.tolerations` | If specified, the pod's tolerations | `{}` |
|
||||
| `master.resources` | The `resource` limit and request config for master server | `{}` |
|
||||
| `master.configmap.MASTER_SERVER_OPTS` | The jvm options for master server | `-Xms1g -Xmx1g -Xmn512m` |
|
||||
| `master.configmap.MASTER_EXEC_THREADS` | Master execute thread number to limit process instances | `100` |
|
||||
| `master.configmap.MASTER_EXEC_TASK_NUM` | Master execute task number in parallel per process instance | `20` |
|
||||
| `master.configmap.MASTER_DISPATCH_TASK_NUM` | Master dispatch task number per batch | `3` |
|
||||
| `master.configmap.MASTER_HOST_SELECTOR` | Master host selector to select a suitable worker, optional values include Random, RoundRobin, LowerWeight | `LowerWeight` |
|
||||
| `master.configmap.MASTER_HEARTBEAT_INTERVAL` | Master heartbeat interval, the unit is second | `10` |
|
||||
| `master.configmap.MASTER_TASK_COMMIT_RETRYTIMES` | Master commit task retry times | `5` |
|
||||
| `master.configmap.MASTER_TASK_COMMIT_INTERVAL` | master commit task interval, the unit is second | `1` |
|
||||
| `master.configmap.MASTER_MAX_CPULOAD_AVG` | Master max cpuload avg, only higher than the system cpu load average, master server can schedule | `-1` (`the number of cpu cores * 2`) |
|
||||
| `master.configmap.MASTER_RESERVED_MEMORY` | Master reserved memory, only lower than system available memory, master server can schedule, the unit is G | `0.3` |
|
||||
| `master.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
|
||||
| `master.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
|
||||
| `master.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `master.livenessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `master.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `master.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `master.readinessProbe.enabled` | Turn on and off readiness probe | `true` |
|
||||
| `master.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` |
|
||||
| `master.readinessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `master.readinessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `master.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `master.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `master.persistentVolumeClaim.enabled` | Set `master.persistentVolumeClaim.enabled` to `true` to mount a new volume for `master` | `false` |
|
||||
| `master.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` access modes | `[ReadWriteOnce]` |
|
||||
| `master.persistentVolumeClaim.storageClassName` | `Master` logs data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` size | `20Gi` |
|
||||
| | | |
|
||||
| `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
|
||||
| `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
|
||||
| `worker.annotations` | The `annotations` for worker server | `{}` |
|
||||
| `worker.affinity` | If specified, the pod's scheduling constraints | `{}` |
|
||||
| `worker.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
|
||||
| `worker.tolerations` | If specified, the pod's tolerations | `{}` |
|
||||
| `worker.resources` | The `resource` limit and request config for worker server | `{}` |
|
||||
| `worker.configmap.WORKER_SERVER_OPTS` | The jvm options for worker server | `-Xms1g -Xmx1g -Xmn512m` |
|
||||
| `worker.configmap.WORKER_EXEC_THREADS` | Worker execute thread number to limit task instances | `100` |
|
||||
| `worker.configmap.WORKER_HEARTBEAT_INTERVAL` | Worker heartbeat interval, the unit is second | `10` |
|
||||
| `worker.configmap.WORKER_MAX_CPULOAD_AVG` | Worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks | `-1` (`the number of cpu cores * 2`) |
|
||||
| `worker.configmap.WORKER_RESERVED_MEMORY` | Worker reserved memory, only lower than system available memory, worker server can be dispatched tasks, the unit is G | `0.3` |
|
||||
| `worker.configmap.WORKER_GROUPS` | Worker groups | `default` |
|
||||
| `worker.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
|
||||
| `worker.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
|
||||
| `worker.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `worker.livenessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `worker.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `worker.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `worker.readinessProbe.enabled` | Turn on and off readiness probe | `true` |
|
||||
| `worker.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` |
|
||||
| `worker.readinessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `worker.readinessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `worker.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `worker.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `worker.persistentVolumeClaim.enabled` | Set `worker.persistentVolumeClaim.enabled` to `true` to enable `persistentVolumeClaim` for `worker` | `false` |
|
||||
| `worker.persistentVolumeClaim.dataPersistentVolume.enabled` | Set `worker.persistentVolumeClaim.dataPersistentVolume.enabled` to `true` to mount a data volume for `worker` | `false` |
|
||||
| `worker.persistentVolumeClaim.dataPersistentVolume.accessModes` | `PersistentVolumeClaim` access modes | `[ReadWriteOnce]` |
|
||||
| `worker.persistentVolumeClaim.dataPersistentVolume.storageClassName` | `Worker` data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `worker.persistentVolumeClaim.dataPersistentVolume.storage` | `PersistentVolumeClaim` size | `20Gi` |
|
||||
| `worker.persistentVolumeClaim.logsPersistentVolume.enabled` | Set `worker.persistentVolumeClaim.logsPersistentVolume.enabled` to `true` to mount a logs volume for `worker` | `false` |
|
||||
| `worker.persistentVolumeClaim.logsPersistentVolume.accessModes` | `PersistentVolumeClaim` access modes | `[ReadWriteOnce]` |
|
||||
| `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` size | `20Gi` |
|
||||
| | | |
|
||||
| `alert.replicas` | Replicas is the desired number of replicas of the given Template | `1` |
|
||||
| `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` |
|
||||
| `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` |
|
||||
| `alert.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` |
|
||||
| `alert.annotations` | The `annotations` for alert server | `{}` |
|
||||
| `alert.affinity` | If specified, the pod's scheduling constraints | `{}` |
|
||||
| `alert.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
|
||||
| `alert.tolerations` | If specified, the pod's tolerations | `{}` |
|
||||
| `alert.resources` | The `resource` limit and request config for alert server | `{}` |
|
||||
| `alert.configmap.ALERT_SERVER_OPTS` | The jvm options for alert server | `-Xms512m -Xmx512m -Xmn256m` |
|
||||
| `alert.configmap.XLS_FILE_PATH` | XLS file path | `/tmp/xls` |
|
||||
| `alert.configmap.MAIL_SERVER_HOST` | Mail `SERVER HOST ` | `nil` |
|
||||
| `alert.configmap.MAIL_SERVER_PORT` | Mail `SERVER PORT` | `nil` |
|
||||
| `alert.configmap.MAIL_SENDER` | Mail `SENDER` | `nil` |
|
||||
| `alert.configmap.MAIL_USER` | Mail `USER` | `nil` |
|
||||
| `alert.configmap.MAIL_PASSWD` | Mail `PASSWORD` | `nil` |
|
||||
| `alert.configmap.MAIL_SMTP_STARTTLS_ENABLE` | Mail `SMTP STARTTLS` enable | `false` |
|
||||
| `alert.configmap.MAIL_SMTP_SSL_ENABLE` | Mail `SMTP SSL` enable | `false` |
|
||||
| `alert.configmap.MAIL_SMTP_SSL_TRUST` | Mail `SMTP SSL TRUST` | `nil` |
|
||||
| `alert.configmap.ENTERPRISE_WECHAT_ENABLE` | `Enterprise Wechat` enable | `false` |
|
||||
| `alert.configmap.ENTERPRISE_WECHAT_CORP_ID` | `Enterprise Wechat` corp id | `nil` |
|
||||
| `alert.configmap.ENTERPRISE_WECHAT_SECRET` | `Enterprise Wechat` secret | `nil` |
|
||||
| `alert.configmap.ENTERPRISE_WECHAT_AGENT_ID` | `Enterprise Wechat` agent id | `nil` |
|
||||
| `alert.configmap.ENTERPRISE_WECHAT_USERS` | `Enterprise Wechat` users | `nil` |
|
||||
| `alert.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
|
||||
| `alert.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
|
||||
| `alert.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `alert.livenessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `alert.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `alert.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `alert.readinessProbe.enabled` | Turn on and off readiness probe | `true` |
|
||||
| `alert.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` |
|
||||
| `alert.readinessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `alert.readinessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `alert.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `alert.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `alert.persistentVolumeClaim.enabled` | Set `alert.persistentVolumeClaim.enabled` to `true` to mount a new volume for `alert` | `false` |
|
||||
| `alert.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` access modes | `[ReadWriteOnce]` |
|
||||
| `alert.persistentVolumeClaim.storageClassName` | `Alert` logs data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `alert.persistentVolumeClaim.storage` | `PersistentVolumeClaim` size | `20Gi` |
|
||||
| | | |
|
||||
| `api.replicas` | Replicas is the desired number of replicas of the given Template | `1` |
|
||||
| `api.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` |
|
||||
| `api.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` |
|
||||
| `api.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` |
|
||||
| `api.annotations` | The `annotations` for api server | `{}` |
|
||||
| `api.affinity` | If specified, the pod's scheduling constraints | `{}` |
|
||||
| `api.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
|
||||
| `api.tolerations` | If specified, the pod's tolerations | `{}` |
|
||||
| `api.resources` | The `resource` limit and request config for api server | `{}` |
|
||||
| `api.configmap.API_SERVER_OPTS` | The jvm options for api server | `-Xms512m -Xmx512m -Xmn256m` |
|
||||
| `api.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
|
||||
| `api.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
|
||||
| `api.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `api.livenessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `api.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `api.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `api.readinessProbe.enabled` | Turn on and off readiness probe | `true` |
|
||||
| `api.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` |
|
||||
| `api.readinessProbe.periodSeconds` | How often to perform the probe | `30` |
|
||||
| `api.readinessProbe.timeoutSeconds` | When the probe times out | `5` |
|
||||
| `api.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
|
||||
| `api.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
|
||||
| `api.persistentVolumeClaim.enabled` | Set `api.persistentVolumeClaim.enabled` to `true` to mount a new volume for `api` | `false` |
|
||||
| `api.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` access modes | `[ReadWriteOnce]` |
|
||||
| `api.persistentVolumeClaim.storageClassName` | `api` logs data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
|
||||
| `api.persistentVolumeClaim.storage` | `PersistentVolumeClaim` size | `20Gi` |
|
||||
| `api.service.type` | `type` determines how the Service is exposed. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer | `ClusterIP` |
|
||||
| `api.service.clusterIP` | `clusterIP` is the IP address of the service and is usually assigned randomly by the master | `nil` |
|
||||
| `api.service.nodePort` | `nodePort` is the port on each node on which this service is exposed when type=NodePort | `nil` |
|
||||
| `api.service.externalIPs` | `externalIPs` is a list of IP addresses for which nodes in the cluster will also accept traffic for this service | `[]` |
|
||||
| `api.service.externalName` | `externalName` is the external reference that kubedns or equivalent will return as a CNAME record for this service | `nil` |
|
||||
| `api.service.loadBalancerIP` | `loadBalancerIP` when service.type is LoadBalancer. LoadBalancer will get created with the IP specified in this field | `nil` |
|
||||
| `api.service.annotations` | `annotations` may need to be set when service.type is LoadBalancer | `{}` |
|
||||
| | | |
|
||||
| `ingress.enabled` | Enable ingress | `false` |
|
||||
| `ingress.host` | Ingress host | `dolphinscheduler.org` |
|
||||
| `ingress.path` | Ingress path | `/dolphinscheduler` |
|
||||
| `ingress.tls.enabled` | Enable ingress tls | `false` |
|
||||
| `ingress.tls.secretName` | Ingress tls secret name | `dolphinscheduler-tls` |
|
||||
|
|
@ -1,236 +0,0 @@
|
|||
# Pseudo-Cluster Deployment
|
||||
|
||||
The purpose of the pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, API server, are all on the same machine.
|
||||
|
||||
If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md).
|
||||
|
||||
## Preparation
|
||||
|
||||
Pseudo-cluster deployment of DolphinScheduler requires external software support:
|
||||
|
||||
* JDK：Download [JDK][jdk] (1.8+), and configure `JAVA_HOME` to and `PATH` variable. You can skip this step, if it already exists in your environment.
|
||||
* Binary package: Download the DolphinScheduler binary package at [download page](https://dolphinscheduler.apache.org/en-us/download/download.html)
|
||||
* Database: [PostgreSQL](https://www.postgresql.org/download/) (8.2.15+) or [MySQL](https://dev.mysql.com/downloads/mysql/) (5.7+), you can choose one of the two, such as MySQL requires JDBC Driver 8.0.16
|
||||
* Registry Center: [ZooKeeper](https://zookeeper.apache.org/releases.html) (3.4.6+)，[download link][zookeeper]
|
||||
* Process tree analysis
|
||||
* `pstree` for macOS
|
||||
* `psmisc` for Fedora/Red/Hat/CentOS/Ubuntu/Debian
|
||||
|
||||
> **_Note:_** DolphinScheduler itself does not depend on Hadoop, Hive, Spark, but if you need to run tasks that depend on them, you need to have the corresponding environment support.
|
||||
|
||||
## DolphinScheduler Startup Environment
|
||||
|
||||
### Configure User Exemption and Permissions
|
||||
|
||||
Create a deployment user, and make sure to configure `sudo` without password. Here make an example to create user `dolphinscheduler`:
|
||||
|
||||
```shell
|
||||
# To create a user, login as root
|
||||
useradd dolphinscheduler
|
||||
|
||||
# Add password
|
||||
echo "dolphinscheduler" | passwd --stdin dolphinscheduler
|
||||
|
||||
# Configure sudo without password
|
||||
sed -i '$adolphinscheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
|
||||
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
|
||||
|
||||
# Modify directory permissions and grant permissions for user you created above
|
||||
chown -R dolphinscheduler:dolphinscheduler apache-dolphinscheduler-*-bin
|
||||
```
|
||||
|
||||
> **_NOTICE:_**
|
||||
>
|
||||
> * Due to DolphinScheduler's multi-tenant task switch user using command `sudo -u {linux-user}`, the deployment user needs to have `sudo` privileges and be password-free. If novice learners don’t understand, you can ignore this point for now.
|
||||
> * If you find the line "Defaults requirett" in the `/etc/sudoers` file, please comment the content.
|
||||
|
||||
### Configure Machine SSH Password-Free Login
|
||||
|
||||
Since resources need to be sent to different machines during installation, SSH password-free login is required between each machine. The following shows the steps to configure password-free login:
|
||||
|
||||
```shell
|
||||
su dolphinscheduler
|
||||
|
||||
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
|
||||
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
|
||||
chmod 600 ~/.ssh/authorized_keys
|
||||
```
|
||||
|
||||
> **_Notice:_** After the configuration is complete, you can run the command `ssh localhost` to test works or not. If you can login with ssh without password stands for successful.
|
||||
|
||||
### Start ZooKeeper
|
||||
|
||||
Go to the ZooKeeper installation directory, copy configure file `zoo_sample.cfg` to `conf/zoo.cfg`, and change value of dataDir in `conf/zoo.cfg` to `dataDir=./tmp/zookeeper`.
|
||||
|
||||
```shell
|
||||
# Start ZooKeeper
|
||||
./bin/zkServer.sh start
|
||||
```
|
||||
|
||||
<!--
|
||||
Modify the database configuration and initialize
|
||||
|
||||
```properties
|
||||
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
|
||||
spring.datasource.url=jdbc:mysql://localhost:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true
|
||||
# Modify it if you are not using dolphinscheduler/dolphinscheduler as your username and password
|
||||
spring.datasource.username=dolphinscheduler
|
||||
spring.datasource.password=dolphinscheduler
|
||||
```
|
||||
|
||||
After modifying and saving, execute the following command to create database tables and init basic data.
|
||||
|
||||
```shell
|
||||
sh script/create-dolphinscheduler.sh
|
||||
```
|
||||
-->
|
||||
|
||||
## Modify Configuration
|
||||
|
||||
After completing the preparation of the basic environment, you need to modify the configuration file according to the
|
||||
environment you used. The configuration files are both in directory `bin/env` and named `install_env.sh` and `dolphinscheduler_env.sh`.
|
||||
|
||||
### Modify `install_env.sh`
|
||||
|
||||
File `install_env.sh` describes which machines will be installed DolphinScheduler and what server will be installed on
|
||||
each machine. You could find this file in the path `bin/env/install_env.sh` and the detail of the configuration as below.
|
||||
|
||||
```shell
|
||||
# ---------------------------------------------------------
|
||||
# INSTALL MACHINE
|
||||
# ---------------------------------------------------------
|
||||
# Due to the master, worker, and API server being deployed on a single node, the IP of the server is the machine IP or localhost
|
||||
ips="localhost"
|
||||
masters="localhost"
|
||||
workers="localhost:default"
|
||||
alertServer="localhost"
|
||||
apiServers="localhost"
|
||||
|
||||
# DolphinScheduler installation path, it will auto-create if not exists
|
||||
installPath="~/dolphinscheduler"
|
||||
|
||||
# Deploy user, use the user you create in section **Configure machine SSH password-free login**
|
||||
deployUser="dolphinscheduler"
|
||||
```
|
||||
|
||||
### Modify `dolphinscheduler_env.sh`
|
||||
|
||||
File `dolphinscheduler_env.sh` describes the database configuration of DolphinScheduler, which in the path `bin/env/dolphinscheduler_env.sh`
|
||||
and some tasks which need external dependencies or libraries such as `JAVA_HOME` and `SPARK_HOME`. You could ignore the
|
||||
task external dependencies if you do not use those tasks, but you have to change `JAVA_HOME`, registry center and database
|
||||
related configurations based on your environment.
|
||||
|
||||
```sh
|
||||
# JAVA_HOME, will use it to start DolphinScheduler server
|
||||
export JAVA_HOME=${JAVA_HOME:-/custom/path}
|
||||
|
||||
# Database related configuration, set database type, username and password
|
||||
export DATABASE=${DATABASE:-postgresql}
|
||||
export SPRING_PROFILES_ACTIVE=${DATABASE}
|
||||
export SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
|
||||
export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler"
|
||||
export SPRING_DATASOURCE_USERNAME="username"
|
||||
export SPRING_DATASOURCE_PASSWORD="password"
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
|
||||
export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-localhost:2181}
|
||||
```
|
||||
|
||||
## Initialize the Database
|
||||
|
||||
DolphinScheduler metadata is stored in the relational database. Currently, supports PostgreSQL and MySQL. If you use MySQL, you need to manually download [mysql-connector-java driver][mysql] (8.0.16) and move it to the lib directory of DolphinScheduler, which is `tools/libs/`. Let's take MySQL as an example for how to initialize the database:
|
||||
|
||||
For mysql 5.6 / 5.7
|
||||
|
||||
```shell
|
||||
mysql -uroot -p
|
||||
|
||||
mysql> CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
|
||||
|
||||
# Replace {user} and {password} with your username and password
|
||||
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'%' IDENTIFIED BY '{password}';
|
||||
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost' IDENTIFIED BY '{password}';
|
||||
|
||||
mysql> flush privileges;
|
||||
```
|
||||
|
||||
For mysql 8:
|
||||
|
||||
```shell
|
||||
mysql -uroot -p
|
||||
|
||||
mysql> CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
|
||||
|
||||
# Replace {user} and {password} with your username and password
|
||||
mysql> CREATE USER '{user}'@'%' IDENTIFIED BY '{password}';
|
||||
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'%';
|
||||
mysql> CREATE USER '{user}'@'localhost' IDENTIFIED BY '{password}';
|
||||
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost';
|
||||
mysql> FLUSH PRIVILEGES;
|
||||
```
|
||||
|
||||
Change the username and password in `tools/conf/application.yaml` to {user} and {password} you set in the previous step.
|
||||
|
||||
Then, modify `tools/bin/dolphinscheduler_env.sh`, set mysql as default database `export DATABASE=${DATABASE:-mysql}`.
|
||||
|
||||
After the above steps done you would create a new database for DolphinScheduler, then run Shell scripts to init database:
|
||||
|
||||
```shell
|
||||
sh tools/bin/create-schema.sh
|
||||
```
|
||||
|
||||
## Start DolphinScheduler
|
||||
|
||||
Use **deployment user** you created above, running the following command to complete the deployment, and the server log will be stored in the logs folder.
|
||||
|
||||
```shell
|
||||
sh ./bin/install.sh
|
||||
```
|
||||
|
||||
> **_Note:_** For the first time deployment, there maybe occur five times of `sh: bin/dolphinscheduler-daemon.sh: No such file or directory` in the terminal,
|
||||
this is non-important information that you can ignore.
|
||||
|
||||
## Login DolphinScheduler
|
||||
|
||||
Access address `http://localhost:12345/dolphinscheduler` and login DolphinScheduler UI. The default username and password are **admin/dolphinscheduler123**
|
||||
|
||||
## Start or Stop Server
|
||||
|
||||
```shell
|
||||
# Stop all DolphinScheduler server
|
||||
sh ./bin/stop-all.sh
|
||||
|
||||
# Start all DolphinScheduler server
|
||||
sh ./bin/start-all.sh
|
||||
|
||||
# Start or stop DolphinScheduler Master
|
||||
sh ./bin/dolphinscheduler-daemon.sh stop master-server
|
||||
sh ./bin/dolphinscheduler-daemon.sh start master-server
|
||||
|
||||
# Start or stop DolphinScheduler Worker
|
||||
sh ./bin/dolphinscheduler-daemon.sh start worker-server
|
||||
sh ./bin/dolphinscheduler-daemon.sh stop worker-server
|
||||
|
||||
# Start or stop DolphinScheduler Api
|
||||
sh ./bin/dolphinscheduler-daemon.sh start api-server
|
||||
sh ./bin/dolphinscheduler-daemon.sh stop api-server
|
||||
|
||||
# Start or stop Alert
|
||||
sh ./bin/dolphinscheduler-daemon.sh start alert-server
|
||||
sh ./bin/dolphinscheduler-daemon.sh stop alert-server
|
||||
```
|
||||
|
||||
> **_Note1:_**: Each server have `dolphinscheduler_env.sh` file in path `<server-name>/conf/dolphinscheduler_env.sh` which
|
||||
> for micro-services need. It means that you could start all servers by command `<server-name>/bin/start.sh` with different
|
||||
> environment variable from `bin/env/dolphinscheduler_env.sh`. But it will use file `bin/env/dolphinscheduler_env.sh` overwrite
|
||||
> `<server-name>/conf/dolphinscheduler_env.sh` if you start server with command `/bin/dolphinscheduler-daemon.sh start <server-name>`.
|
||||
|
||||
> **_Note2:_**: Please refer to the section of "System Architecture Design" for service usage. Python gateway service is
|
||||
> started along with the api-server, and if you do not want to start Python gateway service please disabled it by changing
|
||||
> the yaml config `python-gateway.enabled : false` in api-server's configuration path `api-server/conf/application.yaml`
|
||||
|
||||
[jdk]: https://www.oracle.com/technetwork/java/javase/downloads/index.html
|
||||
[zookeeper]: https://zookeeper.apache.org/releases.html
|
||||
[mysql]: https://downloads.MySQL.com/archives/c-j/
|
||||
[issue]: https://github.com/apache/dolphinscheduler/issues/6597
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
# Standalone
|
||||
|
||||
Standalone only for quick experience for DolphinScheduler.
|
||||
|
||||
If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md).
|
||||
|
||||
> **_Note:_** Standalone only recommends the usage of fewer than 20 workflows, because it uses H2 Database, ZooKeeper Testing Server, too many tasks may cause instability.
|
||||
|
||||
## Preparation
|
||||
|
||||
* JDK：download [JDK][jdk] (1.8+), and configure `JAVA_HOME` to and `PATH` variable. You can skip this step if it already exists in your environment.
|
||||
* Binary package: download the DolphinScheduler binary package at [download page](https://dolphinscheduler.apache.org/en-us/download/download.html).
|
||||
|
||||
## Start DolphinScheduler Standalone Server
|
||||
|
||||
### Extract and Start DolphinScheduler
|
||||
|
||||
There is a standalone startup script in the binary compressed package, which can be quickly started after extraction. Switch to a user with sudo permission and run the script:
|
||||
|
||||
```shell
|
||||
# Extract and start Standalone Server
|
||||
tar -xvzf apache-dolphinscheduler-*-bin.tar.gz
|
||||
cd apache-dolphinscheduler-*-bin
|
||||
sh ./bin/dolphinscheduler-daemon.sh start standalone-server
|
||||
```
|
||||
|
||||
### Login DolphinScheduler
|
||||
|
||||
Access address `http://localhost:12345/dolphinscheduler` and login DolphinScheduler UI. The default username and password are **admin/dolphinscheduler123**
|
||||
|
||||
### Start or Stop Server
|
||||
|
||||
The script `./bin/dolphinscheduler-daemon.sh`can be used not only quickly start standalone, but also to stop the service operation. The following are all the commands:
|
||||
|
||||
```shell
|
||||
# Start Standalone Server
|
||||
sh ./bin/dolphinscheduler-daemon.sh start standalone-server
|
||||
# Stop Standalone Server
|
||||
sh ./bin/dolphinscheduler-daemon.sh stop standalone-server
|
||||
```
|
||||
|
||||
> Note: Python gateway service is started along with the api-server, and if you do not want to start Python gateway
|
||||
> service please disabled it by changing the yaml config `python-gateway.enabled : false` in api-server's configuration
|
||||
> path `api-server/conf/application.yaml`
|
||||
|
||||
[jdk]: https://www.oracle.com/technetwork/java/javase/downloads/index.html
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
# Monitor
|
||||
|
||||
## Service Management
|
||||
|
||||
- Service management is mainly to monitor and display the health status and basic information of each service in the system.
|
||||
|
||||
### Master Server
|
||||
|
||||
- Mainly related to master information.
|
||||
|
||||
|
||||
|
||||
### Worker Server
|
||||
|
||||
- Mainly related to worker information.
|
||||
|
||||
|
||||
|
||||
### Database
|
||||
|
||||
- Mainly the health status of the DB.
|
||||
|
||||
|
||||
|
||||
## Statistics Management
|
||||
|
||||
### Statistics
|
||||
|
||||
|
||||
|
||||
- Number of commands wait to be executed: statistics of the `t_ds_command` table data.
|
||||
- The number of failed commands: statistics of the `t_ds_error_command` table data.
|
||||
- Number of tasks wait to run: count the data of `task_queue` in the ZooKeeper.
|
||||
- Number of tasks wait to be killed: count the data of `task_kill` in the ZooKeeper.
|
||||
|
||||
### Audit Log
|
||||
|
||||
The audit log provides information about who accesses the system and the operations made to the system and record related
|
||||
time, which strengthen the security of the system and maintenance.
|
||||
|
||||
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
# Open API
|
||||
|
||||
## Background
|
||||
|
||||
Generally, projects and processes are created through pages, but considering the integration with third-party systems requires API calls to manage projects and workflows.
|
||||
|
||||
## The Operation Steps of DolphinScheduler API Calls
|
||||
|
||||
### Create a Token
|
||||
|
||||
1. Log in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/token-management-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
2. Select the "Expiration time" (Token validity time), select "User" (choose the specified user to perform the API operation), click "Generate token", copy the `Token` string, and click "Submit".
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/create-token-en1.png" width="80%" />
|
||||
</p>
|
||||
|
||||
### Token Usage
|
||||
|
||||
1. Open the API documentation page
|
||||
> Address：http://{API server ip}:12345/dolphinscheduler/doc.html?language=en_US&lang=en
|
||||
<p align="center">
|
||||
<img src="/img/api-documentation-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
2. select a test API, the API selected for this test is `queryAllProjectList`
|
||||
> projects/query-project-list
|
||||
3. Open `Postman`, fill in the API address, enter the `Token` in `Headers`, and then send the request to view the result:
|
||||
```
|
||||
token: The Token just generated
|
||||
```
|
||||
<p align="center">
|
||||
<img src="/img/test-api.png" width="80%" />
|
||||
</p>
|
||||
|
||||
### Create a Project
|
||||
|
||||
Here is an example of creating a project named "wudl-flink-test"：
|
||||
<p align="center">
|
||||
<img src="/img/api/create_project1.png" width="80%" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/create_project2.png" width="80%" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/create_project3.png" width="80%" />
|
||||
</p>
|
||||
The returned `msg` information is "success", indicating that we have successfully created the project through API.
|
||||
|
||||
If you are interested in the source code of creating a project, please continue to read the following：
|
||||
|
||||
### Appendix： The Source Code of Creating a Project
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/create_source1.png" width="80%" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/api/create_source2.png" width="80%" />
|
||||
</p>
|
||||
|
||||
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
# Built-in Parameter
|
||||
|
||||
## Basic Built-in Parameter
|
||||
|
||||
<table>
|
||||
<tr><th>Variable</th><th>Declaration Method</th><th>Meaning</th></tr>
|
||||
<tr>
|
||||
<td>system.biz.date</td>
|
||||
<td>${system.biz.date}</td>
|
||||
<td>The day before the schedule time of the daily scheduling instance, the format is yyyyMMdd</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>system.biz.curdate</td>
|
||||
<td>${system.biz.curdate}</td>
|
||||
<td>The schedule time of the daily scheduling instance, the format is yyyyMMdd</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>system.datetime</td>
|
||||
<td>${system.datetime}</td>
|
||||
<td>The schedule time of the daily scheduling instance, the format is yyyyMMddHHmmss</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
## Extended Built-in Parameter
|
||||
|
||||
- Support custom variables in the code, declaration way: `${variable name}`. Refers to "System Parameter".
|
||||
|
||||
- Benchmark variable defines as `$[...]` format, time format `$[yyyyMMddHHmmss]` can be decomposed and combined arbitrarily, such as: `$[yyyyMMdd]`, `$[HHmmss]`, `$[yyyy-MM-dd]`, etc.
|
||||
|
||||
- Or define by the 2 following ways:
|
||||
|
||||
1. Use add_month(yyyyMMdd, offset) function to add or minus number of months
|
||||
the first parameter of this function is [yyyyMMdd], represents the time format
|
||||
the second parameter is offset, represents the number of months the user wants to add or minus
|
||||
* Next N years：$[add_months(yyyyMMdd,12*N)]
|
||||
* N years before：$[add_months(yyyyMMdd,-12*N)]
|
||||
* Next N months：$[add_months(yyyyMMdd,N)]
|
||||
* N months before：$[add_months(yyyyMMdd,-N)]
|
||||
*********************************************************************************************************
|
||||
1. Add or minus numbers directly after the time format
|
||||
* Next N weeks：$[yyyyMMdd+7*N]
|
||||
* First N weeks：$[yyyyMMdd-7*N]
|
||||
* Next N days：$[yyyyMMdd+N]
|
||||
* N days before：$[yyyyMMdd-N]
|
||||
* Next N hours：$[HHmmss+N/24]
|
||||
* First N hours：$[HHmmss-N/24]
|
||||
* Next N minutes：$[HHmmss+N/24/60]
|
||||
* First N minutes：$[HHmmss-N/24/60]
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
# Refer to Parameter Context
|
||||
|
||||
DolphinScheduler provides the ability to refer to each other between parameters, including local parameters refer to global parameters and upstream and downstream parameter transfer. Due to the existence of references, it involves the priority of parameters when the parameter names are the same. see also [Parameter Priority](priority.md)
|
||||
|
||||
## Local Task Refers to Global Parameter
|
||||
|
||||
The premise of local tasks referring global parameters is that you have already defined [Global Parameter](global.md). The usage is similar to the usage in [local parameters](local.md), but the value of the parameter needs to be configured as the key of the global parameter.
|
||||
|
||||
|
||||
|
||||
As the figure above shows, `${biz_date}` and `${curdate}` are examples of local parameters that refer to global parameters. Observe the last line of the above figure, `local_param_bizdate` uses `${global_bizdate}` to refer to the global parameter. In the shell script, you can use `${local_param_bizdate}` to refer to the value of the global variable `global_bizdate`, or set the value of `local_param_bizdate` directly through JDBC. Similarly, `local_param` refers to the global parameters defined in the previous section through `${local_param}`. `biz_date`, `biz_curdate`, `system.datetime` are all user-defined parameters, which are assigned value via `${global parameters}`.
|
||||
|
||||
## Pass Parameter From Upstream Task to Downstream
|
||||
|
||||
DolphinScheduler allows parameter transfer between tasks. Currently, transfer direction only supports one-way transfer from upstream to downstream. The task types that support this feature are：
|
||||
|
||||
* [Shell](../task/shell.md)
|
||||
* [SQL](../task/sql.md)
|
||||
* [Procedure](../task/stored-procedure.md)
|
||||
|
||||
When defining an upstream node, if there is a need to transmit the result of that node to a dependency related downstream node. You need to set an `OUT` direction parameter to [Custom Parameters] of the [Current Node Settings]. At present, we mainly focus on the SQL and shell nodes to pass parameters downstream.
|
||||
|
||||
### SQL
|
||||
|
||||
`prop` is user-specified; the direction selects `OUT`, and will define as an export parameter only when the direction is `OUT`. Choose data structures for data type according to the scenario, and the leave the value part blank.
|
||||
|
||||
If the result of the SQL node has only one row, one or multiple fields, the name of the `prop` needs to be the same as the field name. The data type can choose structure except `LIST`. The parameter assigns the value according to the same column name in the SQL query result.
|
||||
|
||||
If the result of the SQL node has multiple rows, one or more fields, the name of the `prop` needs to be the same as the field name. Choose the data type structure as `LIST`, and the SQL query result will be converted to `LIST<VARCHAR>`, and forward to convert to JSON as the parameter value.
|
||||
|
||||
Let's make an example of the SQL node process in the above picture:
|
||||
|
||||
The following defines the [createParam1] node in the above figure:
|
||||
|
||||
|
||||
|
||||
The following defines the [createParam2] node:
|
||||
|
||||
|
||||
|
||||
Find the value of the variable in the [Workflow Instance] page corresponding to the node instance.
|
||||
|
||||
The following shows the Node instance [createparam1]:
|
||||
|
||||
|
||||
|
||||
Here, the value of "id" is 12.
|
||||
|
||||
Let's see the case of the node instance [createparam2].
|
||||
|
||||
|
||||
|
||||
There is only the "id" value. Although the user-defined SQL query both "id" and "database_name" field, only set the `OUT` parameter `id` due to only one parameter "id" is defined for output. The length of the result list is 10 due to display reasons.
|
||||
|
||||
### SHELL
|
||||
|
||||
`prop` is user-specified and the direction is `OUT`. The output is defined as an export parameter only when the direction is `OUT`. Choose data structures for data type according to the scenario, and leave the value part blank.
|
||||
The user needs to pass the parameter when creating the shell script, the output statement format is `${setValue(key=value)}`, the key is the `prop` of the corresponding parameter, and value is the value of the parameter.
|
||||
|
||||
For example, in the figure below:
|
||||
|
||||
|
||||
|
||||
When the log detects the `${setValue(key=value1)}` format in the shell node definition, it will assign value1 to the key, and downstream nodes can use the variable key directly. Similarly, you can find the corresponding node instance on the [Workflow Instance] page to see the value of the variable.
|
||||
|
||||
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
# Global Parameter
|
||||
|
||||
## Scope
|
||||
|
||||
The parameters defined on the process definition page can apply to all the scope of the process tasks.
|
||||
|
||||
## Usage
|
||||
|
||||
Usage of global parameters is: at the process define page, click the '+' beside the 'Set global' and fill in the key and value to save:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/supplement_global_parameter_en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/local_parameter_en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
The `global_bizdate` parameter defined here can be referenced by local parameters of any other task node, and set the value of `global_bizdate` by referencing the system parameter `system.biz.date`.
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
# Local Parameter
|
||||
|
||||
## Scope
|
||||
|
||||
Parameters configured on the task definition page, the scope of this parameter is inside this task only. But if you configure according to [Refer to Parameter Context](context.md), it could pass to downstream tasks.
|
||||
|
||||
## Usage
|
||||
|
||||
Usage of local parameters is: at the task define page, click the '+' beside the 'Custom Parameters' and fill in the key and value to save:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/supplement_local_parameter_en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/global_parameter_en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
If you want to call the [built-in parameter](built-in.md) in the local parameters, fill in thevalue of built-in parameters in `value`. As in the above figure, `${biz_date}` and `${curdate}`.
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
# Parameter Priority
|
||||
|
||||
DolphinScheduler has three parameter types:
|
||||
|
||||
* [Global Parameter](global.md): parameters defined at the workflow define page
|
||||
* [Parameter Context](context.md): parameters passed by upstream task nodes
|
||||
* [Local Parameter](local.md): parameters belong to its node, which is the parameters defined by the user in [Custom Parameters]. The user can define part of the parameters when creating workflow definitions.
|
||||
|
||||
Due to there are multiple sources of the parameter value, there will raise parameter priority issues when the parameter name is the same. The priority of DolphinScheduler parameters from high to low is: `Local Parameter > Parameter Context > Global Parameter`
|
||||
|
||||
In the case of upstream tasks can pass parameters to the downstream, there may be multiple tasks upstream that pass the same parameter name:
|
||||
|
||||
* Downstream nodes prefer to use parameters with non-empty values
|
||||
* If there are multiple parameters with non-empty values, select the value from the upstream task with the earliest completion time
|
||||
|
||||
## Example
|
||||
|
||||
Followings are examples shows task parameters priority problems:#############
|
||||
|
||||
1: Use shell nodes to explain the first case.
|
||||
|
||||
|
||||
|
||||
The [useParam] node can use the parameters which are set in the [createParam] node. The [useParam] node cannot obtain the parameters from the [noUseParam] node due to there is no dependency between them. Other task node types have the same usage rules with the Shell example here.
|
||||
|
||||
|
||||
|
||||
The [createParam] node can use parameters directly. In addition, the node creates two parameters named "key" and "key1", and "key1" has the same name as the one passed by the upstream node and assign value "12". However, due to the priority rules, the value assignment will assign "12" and the value from the upstream node is discarded.
|
||||
|
||||
2: Use SQL nodes to explain another case.
|
||||
|
||||
|
||||
|
||||
The following shows the definition of the [use_create] node:
|
||||
|
||||
|
||||
|
||||
"status" is the own parameters of the node set by the current node. However, the user also sets the "status" parameter (global parameter) when saving the process definition and assign its value to -1. Then the value of status will be 2 with higher priority when the SQL executes. The global parameter value is discarded.
|
||||
|
||||
The "ID" here is the parameter set by the upstream node. The user sets the parameters of the same parameter name "ID" for the [createparam1] node and [createparam2] node. And the [use_create] node uses the value of [createParam1] which is finished first.
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Project Management
|
||||
|
||||
## Create Project
|
||||
|
||||
- Click "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Submit" to create a new project.
|
||||
|
||||
|
||||
|
||||
## Project Home
|
||||
|
||||
- Click the project name link on the project management page to enter the project home page, as shown in the figure below, the project home page contains the task status statistics, process status statistics, and workflow definition statistics of the project. The introduction for those metrics:
|
||||
|
||||
- Task status statistics: within the specified time range, count the number of task instances status as successful submission, running, ready to pause, pause, ready to stop, stop, failure, success, need fault tolerance, kill and waiting threads
|
||||
- Process status statistics: within the specified time range, count the number of workflow instances status as submission success, running, ready to pause, pause, ready to stop, stop, failure, success, need fault tolerance, kill and waiting threads
|
||||
- Workflow definition statistics: count the workflow definitions created by this user and granted by the administrator
|
||||
|
||||
|
||||
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
# Task Definition
|
||||
|
||||
Task definition allows to modify or operate tasks at the task level rather than modifying them in the workflow definition.
|
||||
We already have workflow level task editor in [workflow definition](workflow-definition.md) which you can click the specific
|
||||
workflow and then edit its task definition. It is depressing when you want to edit the task definition but do not remember
|
||||
which workflow it belongs to. So we decide to add `Task Definition` view under `Task` menu.
|
||||
|
||||
|
||||
|
||||
In this view, you can create, query, update, delete task definition by click the related button in `operation` column. The
|
||||
most exciting thing is you could query task by task name in the wildcard, and it is useful when you only remember the task
|
||||
name but forget which workflow it belongs to. It is also supported query by the task name alone with `Task Type` or
|
||||
`Workflow Name`
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
## Task Instance
|
||||
|
||||
- Click Project Management -> Workflow -> Task Instance. Enter the task instance page, as shown in the figure below, click workflow instance name, you can jump to the workflow instance DAG chart to view the task status.
|
||||
<p align="center">
|
||||
<img src="/img/task-list-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
- View log：Click the "view log" button in the operation column to view task execution log.
|
||||
<p align="center">
|
||||
<img src="/img/task-log2-en.png" width="80%" />
|
||||
</p>
|
||||
|
|
@ -1,114 +0,0 @@
|
|||
# Workflow Definition
|
||||
|
||||
## Create workflow definition
|
||||
|
||||
- Click Project Management -> Workflow -> Workflow Definition, enter the workflow definition page, and click the "Create Workflow" button to enter the **workflow DAG edit** page, as shown in the following figure:
|
||||
<p align="center">
|
||||
<img src="/img/dag5.png" width="80%" />
|
||||
</p>
|
||||
- Drag from the toolbar <img src="/img/tasks/icons/shell.png" width="15"/> to the canvas, to add a shell task to the canvas, as shown in the figure below:
|
||||
|
||||
|
||||
|
||||
- **Add parameter settings for shell task:**
|
||||
|
||||
1. Fill in the "Node Name", "Description" and "Script" fields;
|
||||
2. Check “Normal” for “Run Flag”. If “Prohibit Execution” is checked, the task will not execute when the workflow runs;
|
||||
3. Select "Task Priority": when the number of worker threads is insufficient, high priority tasks will execute first in the execution queue, and tasks with the same priority will execute in the order of first in, first out;
|
||||
4. Timeout alarm (optional): check the timeout alarm, timeout failure, and fill in the "timeout period". When the task execution time exceeds **timeout period**, an alert email will send and the task timeout fails;
|
||||
5. Resources (optional). Resources are files create or upload in the Resource Center -> File Management page. For example, the file name is `test.sh`, and the command to call the resource in the script is `sh test.sh`;
|
||||
6. Customize parameters (optional);
|
||||
7. Click the "Confirm Add" button to save the task settings.
|
||||
|
||||
- **Set dependencies between tasks:** Click the icon in the upper right corner <img src="/img/line.png" width="35"/> to connect the task; as shown in the figure below, task 2 and task 3 execute in parallel, When task 1 finished execution, tasks 2 and 3 will execute simultaneously.
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/dag6.png" width="80%" />
|
||||
</p>
|
||||
|
||||
- **Delete dependencies:** Click the "arrow" icon in the upper right corner <img src="/img/arrow.png" width="35"/>, select the connection line, and click the "Delete" icon in the upper right corner <img src= "/img/delete.png" width="35"/>, delete dependencies between tasks.
|
||||
<p align="center">
|
||||
<img src="/img/dag7.png" width="80%" />
|
||||
</p>
|
||||
|
||||
- **Save workflow definition:** Click the "Save" button, and the "Set DAG chart name" window pops up, as shown in the figure below. Enter the workflow definition name, workflow definition description, and set global parameters (optional, refer to [global parameters](../parameter/global.md)), click the "Add" button to finish workflow definition creation.
|
||||
<p align="center">
|
||||
<img src="/img/dag8.png" width="80%" />
|
||||
</p>
|
||||
> For other types of tasks, please refer to [Task Node Type and Parameter Settings](#TaskParamers). <!-- markdown-link-check-disable-line -->
|
||||
|
||||
## Workflow Definition Operation Function
|
||||
|
||||
Click Project Management -> Workflow -> Workflow Definition to enter the workflow definition page, as shown below:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/work_list_en.png" width="80%" />
|
||||
</p>
|
||||
The following are the operation functions of the workflow definition list:
|
||||
|
||||
- **Edit:** Only "Offline" workflow definitions can be edited. Workflow DAG editing is the same as [Create Workflow Definition](#creatDag) <!-- markdown-link-check-disable-line -->
|
||||
- **Online:** When the workflow status is "Offline", used to make workflow online. Only the workflow in the "Online" state can run, but cannot edit
|
||||
- **Offline:** When the workflow status is "Online", used to make workflow offline. Only the workflow in the "Offline" state can be edited, but cannot run
|
||||
- **Run:** Only workflow in the online state can run. See [2.3.3 Run Workflow](#run-the-workflow) for the operation steps.
|
||||
- **Timing:** Timing can only set to online workflows, and the system automatically schedules to run the workflow on time. The status after creating a timing setting is "offline", and the timing must set online on the timing management page to make effect. See [2.3.4 Workflow Timing](#workflow-timing) for timing operation steps
|
||||
- **Timing Management:** The timing management page can edit, online or offline and delete timing
|
||||
- **Delete:** Delete the workflow definition
|
||||
- **Download:** Download workflow definition to local
|
||||
- **Tree Diagram:** Display the task node type and task status in a tree structure, as shown in the figure below:
|
||||
<p align="center">
|
||||
<img src="/img/tree_en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## Run the Workflow
|
||||
|
||||
- Click Project Management -> Workflow -> Workflow Definition to enter the workflow definition page, as shown in the figure below, click the "Go Online" button <img src="/img/online.png" width="35"/>to make workflow online.
|
||||
<p align="center">
|
||||
<img src="/img/work_list_en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
- Click the "Run" button to pop up the startup parameter setting window, as shown in the figure below, set the startup parameters, click the "Run" button in the pop-up box, the workflow starts running, and the workflow instance page generates a workflow instance.
|
||||
<p align="center">
|
||||
<img src="/img/run_work_en.png" width="80%" />
|
||||
</p>
|
||||
Description of workflow operating parameters:
|
||||
|
||||
* Failure strategy: When a task node fails to execute, other parallel task nodes need to execute this strategy. "Continue" means: after a certain task fails, other task nodes execute normally; "End" means: terminate all tasks execution, and terminate the entire process
|
||||
* Notification strategy: When the process is over, send the process execution result notification email according to the process status, options including no send, send if sucess, send of failure, send whatever result
|
||||
* Process priority: The priority of process operation, divide into five levels: highest (HIGHEST), high (HIGH), medium (MEDIUM), low (LOW), and lowest (LOWEST). When the number of master threads is insufficient, high priority processes will execute first in the execution queue, and processes with the same priority will execute in the order of first in, first out;
|
||||
* Worker group: The process can only be executed in the specified worker machine group. The default is `Default`, which can execute on any worker
|
||||
* Notification group: select notification strategy||timeout alarm||when fault tolerance occurs, process result information or email will send to all members in the notification group
|
||||
* Recipient: select notification policy||timeout alarm||when fault tolerance occurs, process result information or alarm email will be sent to the recipient list
|
||||
* Cc: select notification policy||timeout alarm||when fault tolerance occurs, the process result information or warning email will be copied to the CC list
|
||||
* Startup parameter: Set or overwrite global parameter values when starting a new process instance
|
||||
* Complement: two modes including serial complement and parallel complement. Serial complement: within the specified time range, the complements are executed from the start date to the end date and N process instances are generated in turn; parallel complement: within the specified time range, multiple days are complemented at the same time to generate N process instances.
|
||||
* You can select complement time range (only support continuous date) when executing a timing workflow definition. For example, need to fill in the data from 1st May to 10th May, as shown in the figure below:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/complement_en1.png" width="80%" />
|
||||
</p>
|
||||
|
||||
> Serial mode: the complement execute sequentially from 1st May to 10th May, and the process instance page generates 10 process instances;
|
||||
|
||||
> Parallel mode: The tasks from 1st May to 10th May execute simultaneously, and the process instance page generates 10 process instances;
|
||||
|
||||
## Workflow Timing
|
||||
|
||||
- Create timing: Click Project Management->Workflow->Workflow Definition, enter the workflow definition page, make the workflow online, click the "timing" button <img src="/img/timing.png" width="35"/> , the timing parameter setting dialog box pops up, as shown in the figure below:
|
||||
<p align="center">
|
||||
<img src="/img/time_schedule_en.png" width="80%" />
|
||||
</p>
|
||||
- Choose the start and end time. In the time range, the workflow runs at regular intervals; If not in the time range, no regular workflow instances generate.
|
||||
- Add a timing that execute once a day at 5 AM, as shown in the following figure:
|
||||
<p align="center">
|
||||
<img src="/img/timer-en.png" width="80%" />
|
||||
</p>
|
||||
- Failure strategy, notification strategy, process priority, worker group, notification group, recipient, and CC are the same as workflow running parameters.
|
||||
- Click the "Create" button to create the timing. Now the timing status is "**Offline**" and the timing needs to be **Online** to make effect.
|
||||
- Timing online: Click the "Timing Management" button <img src="/img/timeManagement.png" width="35"/>, enter the timing management page, click the "online" button, the timing status will change to "online", as shown in the below figure, the workflow makes effect regularly.
|
||||
<p align="center">
|
||||
<img src="/img/time-manage-list-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## Import Workflow
|
||||
|
||||
Click Project Management -> Workflow -> Workflow Definition to enter the workflow definition page, click the "Import Workflow" button to import the local workflow file, the workflow definition list displays the imported workflow and the status is offline.
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
# Workflow Instance
|
||||
|
||||
## View Workflow Instance
|
||||
|
||||
- Click Project Management -> Workflow -> Workflow Instance, enter the Workflow Instance page, as shown in the figure below:
|
||||
<p align="center">
|
||||
<img src="/img/instance-list-en.png" width="80%" />
|
||||
</p>
|
||||
- Click the workflow name to enter the DAG view page, and check the task execution status, as shown in the figure below:
|
||||
<p align="center">
|
||||
<img src="/img/instance-runs-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## View Task Log
|
||||
|
||||
- Enter the workflow instance page, click the workflow name, enter the DAG view page, double-click the task node, as shown in the figure below:
|
||||
<p align="center">
|
||||
<img src="/img/instanceViewLog-en.png" width="80%" />
|
||||
</p>
|
||||
- Click "View Log", a log window pops up, as shown in the figure below, you can also view the task log on the task instance page, refer to [Task View Log](./task-instance.md)
|
||||
<p align="center">
|
||||
<img src="/img/task-log-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## View Task History
|
||||
|
||||
- Click Project Management -> Workflow -> Workflow Instance, enter the workflow instance page, and click the workflow name to enter the workflow DAG page;
|
||||
- Double-click the task node, as shown in the figure below, click "View History" to jump to the task instance page, and display a list of task instances running by the workflow instance
|
||||
<p align="center">
|
||||
<img src="/img/task_history_en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## View Operation Parameters
|
||||
|
||||
- Click Project Management -> Workflow -> Workflow Instance, enter the workflow instance page, and click the workflow name to enter the workflow DAG page;
|
||||
- Click the icon in the upper left corner <img src="/img/run_params_button.png" width="35"/>，View the startup parameters of the workflow instance; click the icon <img src="/img/global_param.png" width="35"/>，View the global and local parameters of the workflow instance, as shown in the following figure:
|
||||
<p align="center">
|
||||
<img src="/img/run_params_en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## Workflow Instance Operation Function
|
||||
|
||||
Click Project Management -> Workflow -> Workflow Instance, enter the workflow instance page, as shown in the figure below:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/instance-list-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
- **Edit：** only can edit terminated processes. Click the "Edit" button or the workflow instance name to enter the DAG edit page. After the edit, click the "Save" button to confirm, as shown in the figure below. In the pop-up box, check "Whether to update to workflow definition" to update the workflow definition; if not checked, no effect on the workflow definition
|
||||
<p align="center">
|
||||
<img src="/img/editDag-en.png" width="80%" />
|
||||
</p>
|
||||
- **Rerun：** re-execute the terminated process
|
||||
- **Recovery failed：** for failed processes, you can perform failure recovery operations, starting from the failed node
|
||||
- **Stop：** to **stop** the running process, the background code will first `kill` the worker process, and then execute `kill -9` operation
|
||||
- **Pause:** Perform a **pause** operation on the running process, the system status will change to **waiting for execution**, it will wait for the task to finish, and pause the next sequence task.
|
||||
- **Resume pause:** to resume the paused process, start running directly from the **paused node**
|
||||
- **Delete:** delete the workflow instance and the task instance under the workflow instance
|
||||
- **Gantt chart:** the vertical axis of the Gantt chart is the topological sorting of task instances of the workflow instance, and the horizontal axis is the running time of the task instances, as shown in the figure below:
|
||||
<p align="center">
|
||||
<img src="/img/gantt-en.png" width="80%" />
|
||||
</p>
|
||||
|
|
@ -1,165 +0,0 @@
|
|||
# Resource Center
|
||||
|
||||
If you want to use the resource upload function, you can appoint the local file directory as the upload directory for a single machine (this operation does not need to deploy Hadoop). Or you can also upload to a Hadoop or MinIO cluster, at this time, you need to have Hadoop (2.6+) or MinIO or other related environments.
|
||||
|
||||
> **_Note:_**
|
||||
>
|
||||
> * If you want to use the resource upload function, the deployment user in [installation and deployment](installation/standalone.md) must have relevant operation authority.
|
||||
> * If you using a Hadoop cluster with HA, you need to enable HDFS resource upload, and you need to copy the `core-site.xml` and `hdfs-site.xml` under the Hadoop cluster to `/opt/dolphinscheduler/conf`, otherwise skip this copy step.
|
||||
|
||||
## HDFS Resource Configuration
|
||||
|
||||
- Upload resource files and UDF functions, all uploaded files and resources will be stored on HDFS, so require the following configurations:
|
||||
|
||||
```
|
||||
conf/common.properties
|
||||
# Users who have permission to create directories under the HDFS root path
|
||||
hdfs.root.user=hdfs
|
||||
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
|
||||
resource.upload.path=/dolphinscheduler
|
||||
# resource storage type : HDFS,S3,NONE
|
||||
resource.storage.type=HDFS
|
||||
# whether kerberos starts
|
||||
hadoop.security.authentication.startup.state=false
|
||||
# java.security.krb5.conf path
|
||||
java.security.krb5.conf.path=/opt/krb5.conf
|
||||
# loginUserFromKeytab user
|
||||
login.user.keytab.username=hdfs-mycluster@ESZ.COM
|
||||
# loginUserFromKeytab path
|
||||
login.user.keytab.path=/opt/hdfs.headless.keytab
|
||||
# if resource.storage.type is HDFS，and your Hadoop Cluster NameNode has HA enabled, you need to put core-site.xml and hdfs-site.xml in the installPath/conf directory. In this example, it is placed under /opt/soft/dolphinscheduler/conf, and configure the namenode cluster name; if the NameNode is not HA, modify it to a specific IP or host name.
|
||||
# if resource.storage.type is S3，write S3 address，HA，for example ：s3a://dolphinscheduler，
|
||||
# Note，s3 be sure to create the root directory /dolphinscheduler
|
||||
fs.defaultFS=hdfs://mycluster:8020
|
||||
#resourcemanager ha note this need ips , this empty if single
|
||||
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
|
||||
# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine
|
||||
yarn.application.status.address=http://xxxx:8088/ws/v1/cluster/apps/%s
|
||||
|
||||
```
|
||||
|
||||
## File Management
|
||||
|
||||
> It is the management of various resource files, including creating basic `txt/log/sh/conf/py/java` and jar packages and other type files, and can do edit, rename, download, delete and other operations to the files.
|
||||
|
||||
|
||||
|
||||
- Create a file
|
||||
> The file format supports the following types: txt, log, sh, conf, cfg, py, java, sql, xml, hql, properties.
|
||||
|
||||
|
||||
|
||||
- upload files
|
||||
|
||||
> Upload file: Click the "Upload File" button to upload, drag the file to the upload area, the file name will be automatically completed with the uploaded file name.
|
||||
|
||||
|
||||
|
||||
- File View
|
||||
|
||||
> For the files that can be viewed, click the file name to view the file details.
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/file_detail_en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
- Download file
|
||||
|
||||
> Click the "Download" button in the file list to download the file or click the "Download" button in the upper right corner of the file details to download the file.
|
||||
|
||||
- File rename
|
||||
|
||||
|
||||
|
||||
- delete
|
||||
> File list -> Click the "Delete" button to delete the specified file.
|
||||
|
||||
- Re-upload file
|
||||
|
||||
> Re-upload file: Click the "Re-upload File" button to upload a new file to replace the old file, drag the file to the re-upload area, the file name will be automatically completed with the new uploaded file name.
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/reupload_file_en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
|
||||
## UDF Management
|
||||
|
||||
### Resource Management
|
||||
|
||||
> The resource management and file management functions are similar. The difference is that the resource management is the UDF upload function, and the file management uploads the user programs, scripts and configuration files.
|
||||
> Operation function: rename, download, delete.
|
||||
|
||||
- Upload UDF resources
|
||||
> Same as uploading files.
|
||||
|
||||
### Function Management
|
||||
|
||||
- Create UDF function
|
||||
> Click "Create UDF Function", enter the UDF function parameters, select the UDF resource, and click "Submit" to create the UDF function.
|
||||
|
||||
> Currently, only supports temporary UDF functions of Hive.
|
||||
|
||||
- UDF function name: enter the name of the UDF function.
|
||||
- Package name Class name: enter the full path of the UDF function.
|
||||
- UDF resource: set the resource file corresponding to the created UDF function.
|
||||
|
||||
|
||||
|
||||
## Task Group Settings
|
||||
|
||||
The task group is mainly used to control the concurrency of task instances and is designed to control the pressure of other resources (it can also control the pressure of the Hadoop cluster, the cluster will have queue control it). When creating a new task definition, you can configure the corresponding task group and configure the priority of the task running in the task group.
|
||||
|
||||
### Task Group Configuration
|
||||
|
||||
#### Create Task Group
|
||||
|
||||
|
||||
|
||||
The user clicks [Resources] - [Task Group Management] - [Task Group option] - [Create Task Group]
|
||||
|
||||
|
||||
|
||||
You need to enter the information inside the picture:
|
||||
|
||||
- Task group name: the name displayed of the task group
|
||||
|
||||
- Project name: the project range that the task group functions, this item is optional, if not selected, all the projects in the whole system can use this task group.
|
||||
|
||||
- Resource pool size: The maximum number of concurrent task instances allowed.
|
||||
|
||||
#### View Task Group Queue
|
||||
|
||||
|
||||
|
||||
Click the button to view task group usage information:
|
||||
|
||||
|
||||
|
||||
#### Use of Task Groups
|
||||
|
||||
**Note**: The usage of task groups is applicable to tasks executed by workers, such as [switch] nodes, [condition] nodes, [sub_process] and other node types executed by the master are not controlled by the task group. Let's take the shell node as an example:
|
||||
|
||||
|
||||
|
||||
Regarding the configuration of the task group, all you need to do is to configure these parts in the red box:
|
||||
|
||||
- Task group name: The task group name is displayed on the task group configuration page. Here you can only see the task group that the project has permission to access (the project is selected when creating a task group) or the task group that scope globally (no project is selected when creating a task group).
|
||||
|
||||
- Priority: When there is a waiting resource, the task with high priority will be distributed to the worker by the master first. The larger the value of this part, the higher the priority.
|
||||
|
||||
### Implementation Logic of Task Group
|
||||
|
||||
#### Get Task Group Resources:
|
||||
|
||||
The master judges whether the task is configured with a task group when distributing the task. If the task is not configured, it is normally thrown to the worker to run; if a task group is configured, it checks whether the remaining size of the task group resource pool meets the current task operation before throwing it to the worker for execution. , if the resource pool -1 is satisfied, continue to run; if not, exit the task distribution and wait for other tasks to wake up.
|
||||
|
||||
#### Release and Wake Up:
|
||||
|
||||
When the task that has occupied the task group resource is finished, the task group resource will be released. After the release, it will check whether there is a task waiting in the current task group. If there is, mark the task with the best priority to run, and create a new executable event. The event stores the task ID that is marked to acquire the resource, and then the task obtains the task group resource and run.
|
||||
|
||||
#### Task Group Flowchart
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/task_group_process.png" width="80%" />
|
||||
</p>
|
||||
|
|
@ -1,151 +0,0 @@
|
|||
# Security (Authorization System)
|
||||
|
||||
* Only the administrator account in the security center has the authority to operate. It has functions such as queue management, tenant management, user management, alarm group management, worker group management, token management, etc. In the user management module, can authorize to the resources, data sources, projects, etc.
|
||||
* Administrator login, the default username and password is `admin/dolphinscheduler123`
|
||||
|
||||
## Create Queue
|
||||
|
||||
- Configure `queue` parameter to execute programs such as Spark and MapReduce.
|
||||
- The administrator enters the `Security Center->Queue Management` page and clicks the "Create Queue" button to create a new queue.
|
||||
|
||||
|
||||
|
||||
## Add Tenant
|
||||
|
||||
- The tenant corresponds to the Linux user, which is used by the worker to submit the job. The task will fail if Linux does not have this user exists. You can set the parameter `worker.tenant.auto.create` as `true` in configuration file `worker.properties`. After that DolphinScheduler will create a user if not exists, The property `worker.tenant.auto.create=true` requests worker run `sudo` command without password.
|
||||
- Tenant Code: **Tenant Code is the only user on Linux and cannot be repeated**
|
||||
- The administrator enters the `Security Center->Tenant Management` page and clicks the `Create Tenant` button to create a tenant.
|
||||
|
||||
|
||||
|
||||
## Create Normal User
|
||||
|
||||
- Users are divided into **administrator users** and **normal users**
|
||||
|
||||
- The administrator has authorization to authorize and user management authorities but does not have the authority to create project and workflow definition operations.
|
||||
- Normal users can create projects and create, edit and execute workflow definitions.
|
||||
- **Note**: If the user switches tenants, all resources under the tenant to which the user belongs will be copied to the new tenant that is switched.
|
||||
|
||||
- The administrator enters the `Security Center -> User Management` page and clicks the `Create User` button to create a user.
|
||||
|
||||
|
||||
|
||||
> **Edit user information**
|
||||
|
||||
- The administrator enters the `Security Center->User Management` page and clicks the `Edit` button to edit user information.
|
||||
- After a normal user logs in, click the user information in the username drop-down box to enter the user information page, and click the `Edit` button to edit the user information.
|
||||
|
||||
> **Modify user password**
|
||||
|
||||
- The administrator enters the `Security Center->User Management` page and clicks the `Edit` button. When editing user information, enter the new password to modify the user password.
|
||||
- After a normal user logs in, click the user information in the username drop-down box to enter the password modification page, enter the password and confirm the password and click the `Edit` button, then the password modification is a success.
|
||||
|
||||
## Create Alarm Group
|
||||
|
||||
- The alarm group is a parameter set at startup. After the process ends, the status of the process and other information will be sent to the alarm group by email.
|
||||
|
||||
* The administrator enters the `Security Center -> Alarm Group Management` page and clicks the `Create Alarm Group` button to create an alarm group.
|
||||
|
||||
|
||||
|
||||
## Token Management
|
||||
|
||||
> Since the back-end interface has login check, token management provides a way to execute various operations on the system by calling interfaces.
|
||||
|
||||
- The administrator enters the `Security Center -> Token Management page`, clicks the `Create Token` button, selects the expiration time and user, clicks the `Generate Token` button, and clicks the `Submit` button, then create the selected user's token successfully.
|
||||
|
||||
|
||||
|
||||
- After a normal user logs in, click the user information in the username drop-down box, enter the token management page, select the expiration time, click the `Generate Token` button, and click the `Submit` button, then the user creates a token successfully.
|
||||
- Call example:
|
||||
|
||||
```java
|
||||
/**
|
||||
* test token
|
||||
*/
|
||||
public void doPOSTParam()throws Exception{
|
||||
// create HttpClient
|
||||
CloseableHttpClient httpclient = HttpClients.createDefault();
|
||||
|
||||
// create http post request
|
||||
HttpPost httpPost = new HttpPost("http://127.0.0.1:12345/escheduler/projects/create");
|
||||
httpPost.setHeader("token", "123");
|
||||
// set parameters
|
||||
List<NameValuePair> parameters = new ArrayList<NameValuePair>();
|
||||
parameters.add(new BasicNameValuePair("projectName", "qzw"));
|
||||
parameters.add(new BasicNameValuePair("desc", "qzw"));
|
||||
UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(parameters);
|
||||
httpPost.setEntity(formEntity);
|
||||
CloseableHttpResponse response = null;
|
||||
try {
|
||||
// execute
|
||||
response = httpclient.execute(httpPost);
|
||||
// response status code 200
|
||||
if (response.getStatusLine().getStatusCode() == 200) {
|
||||
String content = EntityUtils.toString(response.getEntity(), "UTF-8");
|
||||
System.out.println(content);
|
||||
}
|
||||
} finally {
|
||||
if (response != null) {
|
||||
response.close();
|
||||
}
|
||||
httpclient.close();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Granted Permissions
|
||||
|
||||
* Granted permissions include project permissions, resource permissions, data source permissions, UDF function permissions.
|
||||
* The administrator can authorize the projects, resources, data sources and UDF functions to normal users which not created by them. Because the way to authorize projects, resources, data sources and UDF functions to users is the same, we take project authorization as an example.
|
||||
* Note: The user has all permissions to the projects created by them. Projects will not be displayed in the project list and the selected project list.
|
||||
|
||||
- The administrator enters the `Security Center -> User Management` page and clicks the `Authorize` button of the user who needs to be authorized, as shown in the figure below:
|
||||
<p align="center">
|
||||
<img src="/img/auth-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
- Select the project and authorize the project.
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/auth-project-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
- Resources, data sources, and UDF function authorization are the same as project authorization.
|
||||
|
||||
## Worker Grouping
|
||||
|
||||
Each worker node will belong to its own worker group, and the default group is "default".
|
||||
|
||||
When executing a task, the task can be assigned to the specified worker group, and the task will be executed by the worker node in the group.
|
||||
|
||||
> Add or update worker group
|
||||
|
||||
- Open the `conf/worker.properties` configuration file on the worker node where you want to configure the groups and modify the `worker.groups` parameter.
|
||||
- The `worker.groups` parameter is followed by the name of the group corresponding to the worker node, which is `default`.
|
||||
- If the worker node corresponds to more than one group, they are separated by commas.
|
||||
|
||||
```conf
|
||||
worker.groups=default,test
|
||||
```
|
||||
- You can also change the worker group for the worker during execution, and if the modification is successful, the worker will use the new group and ignore the configuration in `worker.properties`. The step to modify work group as below: `Security Center -> Worker Group Management -> click 'New Worker Group' -> click 'New Worker Group' -> enter 'Group Name' -> Select Exists Worker -> Click Submit`.
|
||||
|
||||
## Environmental Management
|
||||
|
||||
* Configure the Worker operating environment online. A Worker can specify multiple environments, and each environment is equivalent to the `dolphinscheduler_env.sh` file.
|
||||
|
||||
* The default environment is the `dolphinscheduler_env.sh` file.
|
||||
|
||||
* When executing a task, the task can be assigned to the specified worker group, and select the corresponding environment according to the worker group. Finally, the worker node executes the environment first and then executes the task.
|
||||
|
||||
> Add or update environment
|
||||
|
||||
- The environment configuration is equivalent to the configuration in the `dolphinscheduler_env.sh` file.
|
||||
|
||||
|
||||
|
||||
> Usage environment
|
||||
|
||||
- Create a task node in the workflow definition, select the worker group and the environment corresponding to the worker group. When executing the task, the Worker will execute the environment first before executing the task.
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,62 +0,0 @@
|
|||
# Quick Start
|
||||
|
||||
* Watch Apache DolphinScheduler Quick Start Tutorile here:
|
||||
[](https://www.youtube.com/watch?v=nrF20hpCkug)
|
||||
|
||||
|
||||
* Administrator user login
|
||||
|
||||
> Address：http://localhost:12345/dolphinscheduler Username and password: `admin/dolphinscheduler123`
|
||||
|
||||
|
||||
|
||||
* Create a queue
|
||||
|
||||
|
||||
|
||||
* Create a tenant
|
||||
|
||||
|
||||
|
||||
* Create Ordinary Users
|
||||
|
||||
|
||||
|
||||
* Create an alarm instance
|
||||
|
||||
|
||||
|
||||
* Create an alarm group
|
||||
|
||||
|
||||
|
||||
* Create a worker group
|
||||
|
||||
|
||||
|
||||
* Create environment
|
||||
|
||||
|
||||
|
||||
* Create a token
|
||||
|
||||
|
||||
|
||||
* Login with regular users
|
||||
> Click on the user name in the upper right corner to "exit" and re-use the normal user login.
|
||||
|
||||
* `Project Management - > Create Project - > Click on Project Name`
|
||||
|
||||
|
||||
|
||||
* `Click Workflow Definition - > Create Workflow Definition - > Online Process Definition`
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/process_definition_en.png" width="60%" />
|
||||
</p>
|
||||
|
||||
* `Running Process Definition - > Click Workflow Instance - > Click Process Instance Name - > Double-click Task Node - > View Task Execution Log`
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/log_en.png" width="60%" />
|
||||
</p>
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
# Conditions
|
||||
|
||||
Condition is a conditional node, that determines which downstream task should run based on the condition of the upstream task. Currently, the Conditions support multiple upstream tasks, but only two downstream tasks. When the number of upstream tasks exceeds one, achieve complex upstream dependencies by through `and` and `or` operators.
|
||||
|
||||
## Create Task
|
||||
|
||||
Drag from the toolbar <img src="/img/conditions.png" width="20"/> task node to canvas to create a new Conditions task, as shown in the figure below:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/condition_dag_en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/condition_task_en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
## Parameter
|
||||
|
||||
- Node name: The node name in a workflow definition is unique.
|
||||
- Run flag: Identifies whether this node schedules normally, if it does not need to execute, select the `prohibition execution`.
|
||||
- Descriptive information: Describe the function of the node.
|
||||
- Task priority: When the number of worker threads is insufficient, execute in the order of priority from high to low, and tasks with the same priority will execute in a first-in first-out order.
|
||||
- Worker grouping: Assign tasks to the machines of the worker group to execute. If `Default` is selected, randomly select a worker machine for execution.
|
||||
- Times of failed retry attempts: The number of times the task failed to resubmit. You can select from drop-down or fill-in a number.
|
||||
- Failed retry interval: The time interval for resubmitting the task after a failed task. You can select from drop-down or fill-in a number.
|
||||
- Timeout alarm: Check the timeout alarm and timeout failure. When the task runs exceed the "timeout", an alarm email will send and the task execution will fail.
|
||||
- Downstream tasks selection: supports two branches success and failure.
|
||||
- Success: When the upstream task runs successfully, run the success branch.
|
||||
- Failure: When the upstream task runs failed, run the failure branch.
|
||||
- Upstream condition selection: can select one or more upstream tasks for conditions.
|
||||
- Add an upstream dependency: the first parameter is to choose a specified task name, and the second parameter is to choose the upstream task status to trigger conditions.
|
||||
- Select upstream task relationship: use `and` and `or` operators to handle the complex relationship of upstream when there are multiple upstream tasks for conditions.
|
||||
|
||||
## Related Task
|
||||
|
||||
[switch](switch.md): Conditions task mainly executes the corresponding branch based on the execution status (success, failure) of the upstream nodes. The [Switch](switch.md) task node mainly executes the corresponding branch based on the value of the [global parameter](../parameter/global.md) and the result of user written expression.
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
# DataX
|
||||
|
||||
## Overview
|
||||
|
||||
DataX task type for executing DataX programs. For DataX nodes, the worker will execute `${DATAX_HOME}/bin/datax.py` to analyze the input json file.
|
||||
|
||||
## Create Task
|
||||
|
||||
- Click `Project -> Management-Project -> Name-Workflow Definition`, and click the `Create Workflow` button to enter the DAG editing page.
|
||||
- Drag from the toolbar <img src="/img/tasks/icons/datax.png" width="15"/> task node to canvas.
|
||||
|
||||
## Task Parameter
|
||||
|
||||
- **Node name**: The node name in a workflow definition is unique.
|
||||
- **Run flag**: Identifies whether this node schedules normally, if it does not need to execute, select the `prohibition execution`.
|
||||
- **Descriptive information**: Describe the function of the node.
|
||||
- **Task priority**: When the number of worker threads is insufficient, execute in the order of priority from high to low, and tasks with the same priority will execute in a first-in first-out order.
|
||||
- **Worker grouping**: Assign tasks to the machines of the worker group to execute. If `Default` is selected, randomly select a worker machine for execution.
|
||||
- **Environment Name**: Configure the environment name in which run the script.
|
||||
- **Times of failed retry attempts**: The number of times the task failed to resubmit.
|
||||
- **Failed retry interval**: The time interval (unit minute) for resubmitting the task after a failed task.
|
||||
- **Delayed execution time**: The time (unit minute) that a task delays in execution.
|
||||
- **Timeout alarm**: Check the timeout alarm and timeout failure. When the task runs exceed the "timeout", an alarm email will send and the task execution will fail.
|
||||
- **Custom template**: Customize the content of the DataX node's JSON profile when the default DataSource provided does not meet the requirements.
|
||||
- **JSON**: JSON configuration file for DataX synchronization.
|
||||
- **Custom parameters**: SQL task type, and stored procedure is a custom parameter order, to set customized parameter type and data type for the method is the same as the stored procedure task type. The difference is that the custom parameter of the SQL task type replaces the `${variable}` in the SQL statement.
|
||||
- **Data source**: Select the data source to extract data.
|
||||
- **SQL statement**: The SQL statement used to extract data from the target database, the SQL query column name is automatically parsed when execute the node, and mapped to the target table to synchronize column name. When the column names of the source table and the target table are inconsistent, they can be converted by column alias (as)
|
||||
- **Target library**: Select the target library for data synchronization.
|
||||
- **Pre-SQL**: Pre-SQL executes before the SQL statement (executed by the target database).
|
||||
- **Post-SQL**: Post-SQL executes after the SQL statement (executed by the target database).
|
||||
- **Stream limit (number of bytes)**: Limit the number of bytes for a query.
|
||||
- **Limit flow (number of records)**: Limit the number of records for a query.
|
||||
- **Running memory**: Set the minimum and maximum memory required, which can be set according to the actual production environment.
|
||||
- **Predecessor task**: Selecting a predecessor task for the current task, will set the selected predecessor task as upstream of the current task.
|
||||
|
||||
## Task Example
|
||||
|
||||
This example demonstrates how to import data from Hive into MySQL.
|
||||
|
||||
### Configure the DataX environment in DolphinScheduler
|
||||
|
||||
If you are using the DataX task type in a production environment, it is necessary to configure the required environment first. The following is the configuration file: `bin/env/dolphinscheduler_env.sh`.
|
||||
|
||||
|
||||
|
||||
After finish the environment configuration, need to restart DolphinScheduler.
|
||||
|
||||
### Configure DataX Task Node
|
||||
|
||||
As the default DataSource does not contain data read from Hive, require a custom JSON, refer to: [HDFS Writer](https://github.com/alibaba/DataX/blob/master/hdfswriter/doc/hdfswriter.md). Note: Partition directories exist on the HDFS path, when importing data in real world situations, partitioning is recommended to be passed as a parameter, using custom parameters.
|
||||
|
||||
After finish the required JSON file, you can configure the node by following the steps in the diagram below:
|
||||
|
||||
|
||||
|
||||
### View Execution Result
|
||||
|
||||
|
||||
|
||||
### Notice
|
||||
|
||||
If the default DataSource provided does not meet your needs, you can configure the writer and reader of the DataX according to the actual usage environment in the custom template options, available at [DataX](https://github.com/alibaba/DataX).
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
# Dependent Node
|
||||
|
||||
- Dependent nodes are **dependency check nodes**. For example, process A depends on the successful execution of process B from yesterday, and the dependent node will check whether process B run successful yesterday.
|
||||
|
||||
> Drag from the toolbar  task node to the canvas, as shown in the figure below:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/dependent-nodes-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
> The dependent node provides a logical judgment function, such as checking whether the B process was successful yesterday, or whether the C process was executed successfully.
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/depend-node-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
> For example, process A is a weekly report task, processes B and C are daily tasks, and task A requires tasks B and C to be successfully executed every day of the last week, as shown in the figure:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/depend-node1-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
> If the weekly report A also needs to be executed successfully last Tuesday:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/depend-node3-en.png" width="80%" />
|
||||
</p>
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
# Amazon EMR
|
||||
|
||||
## Overview
|
||||
|
||||
Amazon EMR task type, for creating EMR clusters on AWS and running computing tasks. Using [aws-java-sdk](https://aws.amazon.com/cn/sdk-for-java/) in the background code, to transfer JSON parameters to [RunJobFlowRequest](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/elasticmapreduce/model/RunJobFlowRequest.html) object and submit to AWS.
|
||||
|
||||
## Parameter
|
||||
|
||||
- Node name: The node name in a workflow definition is unique.
|
||||
- Run flag: Identifies whether this node schedules normally, if it does not need to execute, select the `prohibition execution`.
|
||||
- Descriptive information: Describe the function of the node.
|
||||
- Task priority: When the number of worker threads is insufficient, execute in the order of priority from high to low, and tasks with the same priority will execute in a first-in first-out order.
|
||||
- Worker grouping: Assign tasks to the machines of the worker group to execute. If `Default` is selected, randomly select a worker machine for execution.
|
||||
- Times of failed retry attempts: The number of times the task failed to resubmit. You can select from drop-down or fill-in a number.
|
||||
- Failed retry interval: The time interval for resubmitting the task after a failed task. You can select from drop-down or fill-in a number.
|
||||
- Timeout alarm: Check the timeout alarm and timeout failure. When the task runs exceed the "timeout", an alarm email will send and the task execution will fail.
|
||||
- JSON: JSON corresponding to the [RunJobFlowRequest](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/elasticmapreduce/model/RunJobFlowRequest.html) object, for details refer to [API_RunJobFlow_Examples](https://docs.aws.amazon.com/emr/latest/APIReference/API_RunJobFlow.html#API_RunJobFlow_Examples).
|
||||
|
||||
## JSON example
|
||||
|
||||
```json
|
||||
{
|
||||
"Name": "SparkPi",
|
||||
"ReleaseLabel": "emr-5.34.0",
|
||||
"Applications": [
|
||||
{
|
||||
"Name": "Spark"
|
||||
}
|
||||
],
|
||||
"Instances": {
|
||||
"InstanceGroups": [
|
||||
{
|
||||
"Name": "Primary node",
|
||||
"InstanceRole": "MASTER",
|
||||
"InstanceType": "m4.xlarge",
|
||||
"InstanceCount": 1
|
||||
}
|
||||
],
|
||||
"KeepJobFlowAliveWhenNoSteps": false,
|
||||
"TerminationProtected": false
|
||||
},
|
||||
"Steps": [
|
||||
{
|
||||
"Name": "calculate_pi",
|
||||
"ActionOnFailure": "CONTINUE",
|
||||
"HadoopJarStep": {
|
||||
"Jar": "command-runner.jar",
|
||||
"Args": [
|
||||
"/usr/lib/spark/bin/run-example",
|
||||
"SparkPi",
|
||||
"15"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"JobFlowRole": "EMR_EC2_DefaultRole",
|
||||
"ServiceRole": "EMR_DefaultRole"
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
# Flink Node
|
||||
|
||||
## Overview
|
||||
|
||||
Flink task type for executing Flink programs. For Flink nodes, the worker submits the task by using the Flink command `flink run`. See [flink cli](https://nightlies.apache.org/flink/flink-docs-release-1.14/docs/deployment/cli/) for more details.
|
||||
|
||||
## Create Task
|
||||
|
||||
- Click `Project -> Management-Project -> Name-Workflow Definition`, and click the "Create Workflow" button to enter the DAG editing page.
|
||||
- Drag from the toolbar <img src="/img/tasks/icons/flink.png" width="15"/>task node to canvas.
|
||||
|
||||
## Task Parameter
|
||||
|
||||
- **Node name**: The node name in a workflow definition is unique.
|
||||
- **Run flag**: Identifies whether this node schedules normally, if it does not need to execute, select the `prohibition execution`.
|
||||
- **Descriptive information**: Describe the function of the node.
|
||||
- **Task priority**: When the number of worker threads is insufficient, execute in the order of priority from high to low, and tasks with the same priority will execute in a first-in first-out order.
|
||||
- **Worker grouping**: Assign tasks to the machines of the worker group to execute. If `Default` is selected, randomly select a worker machine for execution.
|
||||
- **Environment Name**: Configure the environment name in which run the script.
|
||||
- **Times of failed retry attempts**: The number of times the task failed to resubmit.
|
||||
- **Failed retry interval**: The time interval (unit minute) for resubmitting the task after a failed task.
|
||||
- **Delayed execution time**: The time (unit minute) that a task delays in execution.
|
||||
- **Timeout alarm**: Check the timeout alarm and timeout failure. When the task runs exceed the "timeout", an alarm email will send and the task execution will fail.
|
||||
- **Program type**: Supports Java, Scala and Python.
|
||||
- **The class of main function**: The **full path** of Main Class, the entry point of the Flink program.
|
||||
- **Main jar package**: The jar package of the Flink program (upload by Resource Center).
|
||||
- **Deployment mode**: Support 2 deployment modes: cluster and local.
|
||||
- **Flink version**: Select version according to the execution env.
|
||||
- **Task name** (optional): Flink task name.
|
||||
- **JobManager memory size**: Used to set the size of jobManager memories, which can be set according to the actual production environment.
|
||||
- **Number of slots**: Used to set the number of slots, which can be set according to the actual production environment.
|
||||
- **TaskManager memory size**: Used to set the size of taskManager memories, which can be set according to the actual production environment.
|
||||
- **Number of TaskManager**: Used to set the number of taskManagers, which can be set according to the actual production environment.
|
||||
- **Parallelism**: Used to set the degree of parallelism for executing Flink tasks.
|
||||
- **Main program parameters**: Set the input parameters for the Flink program and support the substitution of custom parameter variables.
|
||||
- **Optional parameters**: Support `--jar`, `--files`,` --archives`, `--conf` format.
|
||||
- **Resource**: Appoint resource files in the `Resource` if parameters refer to them.
|
||||
- **Custom parameter**: It is a local user-defined parameter for Flink, and will replace the content with `${variable}` in the script.
|
||||
- **Predecessor task**: Selecting a predecessor task for the current task, will set the selected predecessor task as upstream of the current task.
|
||||
|
||||
## Task Example
|
||||
|
||||
### Execute the WordCount Program
|
||||
|
||||
This is a common introductory case in the big data ecosystem, which often apply to computational frameworks such as MapReduce, Flink and Spark. The main purpose is to count the number of identical words in the input text. (Flink's releases attach this example job)
|
||||
|
||||
#### Configure the flink environment in DolphinScheduler
|
||||
|
||||
If you are using the flink task type in a production environment, it is necessary to configure the required environment first. The following is the configuration file: `bin/env/dolphinscheduler_env.sh`.
|
||||
|
||||
|
||||
|
||||
#### Upload the Main Package
|
||||
|
||||
When using the Flink task node, you need to upload the jar package to the Resource Center for the execution, refer to the [resource center](../resource.md).
|
||||
|
||||
After finish the Resource Centre configuration, upload the required target files directly by dragging and dropping.
|
||||
|
||||
|
||||
|
||||
#### Configure Flink Nodes
|
||||
|
||||
Configure the required content according to the parameter descriptions above.
|
||||
|
||||
|
||||
|
||||
## Notice
|
||||
|
||||
JAVA and Scala only used for identification, there is no difference. If use Python to develop Flink, there is no class of the main function and the rest is the same.
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
# HTTP Node
|
||||
|
||||
## Overview
|
||||
|
||||
This node is used to perform http type tasks such as the common POST and GET request types, and also supports http request validation and other functions.
|
||||
|
||||
## Create Task
|
||||
|
||||
- Click `Project -> Management-Project -> Name-Workflow Definition`, and click the "Create Workflow" button to enter the DAG editing page.
|
||||
- Drag the <img src="/img/tasks/icons/http.png" width="15"/> from the toolbar to the drawing board.
|
||||
|
||||
## Task Parameter
|
||||
|
||||
- **Node name**: The node name in a workflow definition is unique.
|
||||
- **Run flag**: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.
|
||||
- **Descriptive information**: describe the function of the node.
|
||||
- **Task priority**: When the number of worker threads is insufficient, they are executed in order from high to low, and when the priority is the same, they are executed according to the first-in first-out principle.
|
||||
- **Worker grouping**: Tasks are assigned to the machines of the worker group to execute. If Default is selected, a worker machine will be randomly selected for execution.
|
||||
- **Environment Name**: Configure the environment name in which to run the script.
|
||||
- **Number of failed retry attempts**: The number of times the task failed to be resubmitted.
|
||||
- **Failed retry interval**: The time, in cents, interval for resubmitting the task after a failed task.
|
||||
- **Delayed execution time**: the time, in cents, that a task is delayed in execution.
|
||||
- **Timeout alarm**: Check the timeout alarm and timeout failure. When the task exceeds the "timeout period", an alarm email will be sent and the task execution will fail.
|
||||
- **Request address**: HTTP request URL.
|
||||
- **Request type**: Support GET, POSt, HEAD, PUT, DELETE.
|
||||
- **Request parameters**: Support Parameter, Body, Headers.
|
||||
- **Verification conditions**: support default response code, custom response code, content included, content not included.
|
||||
- **Verification content**: When the verification condition selects a custom response code, the content contains, and the content does not contain, the verification content is required.
|
||||
- **Custom parameter**: It is a user-defined parameter of http part, which will replace the content with `${variable}` in the script.
|
||||
- **Predecessor task**: Selecting a predecessor task for the current task will set the selected predecessor task as upstream of the current task.
|
||||
|
||||
## Example
|
||||
|
||||
HTTP defines the different methods of interacting with the server, the most basic methods are GET, POST, PUT and DELETE. Here we use the http task node to demonstrate the use of POST to send a request to the system's login page to submit data.
|
||||
|
||||
The main configuration parameters are as follows:
|
||||
|
||||
- URL: Address to access the target resource. Here is the system's login page.
|
||||
- HTTP Parameters:
|
||||
- userName: Username
|
||||
- userPassword: User login password
|
||||
|
||||
|
||||
|
||||
## Notice
|
||||
|
||||
None
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
# MapReduce Node
|
||||
|
||||
## Overview
|
||||
|
||||
MapReduce(MR) task type used for executing MapReduce programs. For MapReduce nodes, the worker submits the task by using the Hadoop command `hadoop jar`. See [Hadoop Command Manual](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CommandsManual.html#jar) for more details.
|
||||
|
||||
## Create Task
|
||||
|
||||
- Click `Project -> Management-Project -> Name-Workflow Definition`, and click the `Create Workflow` button to enter the DAG editing page.
|
||||
- Drag from the toolbar <img src="/img/tasks/icons/mr.png" width="15"/> to the canvas.
|
||||
|
||||
## Task Parameter
|
||||
|
||||
- **Node name**: The node name in a workflow definition is unique.
|
||||
- **Run flag**: Identifies whether this node schedules normally, if it does not need to execute, select the `prohibition execution`.
|
||||
- **Descriptive information**: Describe the function of the node.
|
||||
- **Task priority**: When the number of worker threads is insufficient, execute in the order of priority from high to low, and tasks with the same priority will execute in a first-in first-out order.
|
||||
- **Worker grouping**: Assign tasks to the machines of the worker group to execute. If `Default` is selected, randomly select a worker machine for execution.
|
||||
- **Environment Name**: Configure the environment name in which run the script.
|
||||
- **Times of failed retry attempts**: The number of times the task failed to resubmit.
|
||||
- **Failed retry interval**: The time interval (unit minute) for resubmitting the task after a failed task.
|
||||
- **Delayed execution time**: The time (unit minute) that a task delays in execution.
|
||||
- **Timeout alarm**: Check the timeout alarm and timeout failure. When the task runs exceed the "timeout", an alarm email will send and the task execution will fail.
|
||||
- **Resource**: Refers to the list of resource files that called in the script, and upload or create files by the Resource Center file management.
|
||||
- **Custom parameters**: It is a local user-defined parameter for MapReduce, and will replace the content with `${variable}` in the script.
|
||||
- **Predecessor task**: Selecting a predecessor task for the current task, will set the selected predecessor task as upstream of the current task.
|
||||
|
||||
### JAVA or SCALA Program
|
||||
|
||||
- **Program type**: Select JAVA or SCALA program.
|
||||
- **The class of the main function**: The **full path** of Main Class, the entry point of the MapReduce program.
|
||||
- **Main jar package**: The jar package of the MapReduce program.
|
||||
- **Task name** (optional): MapReduce task name.
|
||||
- **Command line parameters**: Set the input parameters of the MapReduce program and support the substitution of custom parameter variables.
|
||||
- **Other parameters**: support `-D`, `-files`, `-libjars`, `-archives` format.
|
||||
- **Resource**: Appoint resource files in the `Resource` if parameters refer to them.
|
||||
- **User-defined parameter**: It is a local user-defined parameter for MapReduce, and will replace the content with `${variable}` in the script.
|
||||
|
||||
## Python Program
|
||||
|
||||
- **Program type**: Select Python language.
|
||||
- **Main jar package**: The Python jar package for running MapReduce.
|
||||
- **Other parameters**: support `-D`, `-mapper`, `-reducer,` `-input` `-output` format, and you can set the input of user-defined parameters, such as:
|
||||
- `-mapper "mapper.py 1"` `-file mapper.py` `-reducer reducer.py` `-file reducer.py` `–input /journey/words.txt` `-output /journey/out/mr/\${currentTimeMillis}`
|
||||
- The `mapper.py 1` after `-mapper` is two parameters, the first parameter is `mapper.py`, and the second parameter is `1`.
|
||||
- **Resource**: Appoint resource files in the `Resource` if parameters refer to them.
|
||||
- **User-defined parameter**: It is a local user-defined parameter for MapReduce, and will replace the content with `${variable}` in the script.
|
||||
|
||||
## Task Example
|
||||
|
||||
### Execute the WordCount Program
|
||||
|
||||
This example is a common introductory type of MapReduce application, which used to count the number of identical words in the input text.
|
||||
|
||||
#### Configure the MapReduce Environment in DolphinScheduler
|
||||
|
||||
If you are using the MapReduce task type in a production environment, it is necessary to configure the required environment first. The following is the configuration file: `bin/env/dolphinscheduler_env.sh`.
|
||||
|
||||
|
||||
|
||||
#### Upload the Main Package
|
||||
|
||||
When using the MapReduce task node, you need to use the Resource Centre to upload the jar package for the execution. Refer to the [resource centre](../resource.md).
|
||||
|
||||
After finish the Resource Centre configuration, upload the required target files directly by dragging and dropping.
|
||||
|
||||
|
||||
|
||||
#### Configure MapReduce Nodes
|
||||
|
||||
Configure the required content according to the parameter descriptions above.
|
||||
|
||||
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
# Pigeon
|
||||
|
||||
Pigeon is a task used to trigger remote tasks, acquire logs or status by calling remote WebSocket service. It is DolphinScheduler uses a remote WebSocket service to call tasks.
|
||||
|
||||
## Create
|
||||
|
||||
Drag from the toolbar <img src="/img/pigeon.png" width="20"/> to the canvas to create a new Pigeon task.
|
||||
|
||||
## Parameter
|
||||
|
||||
- Node name: The node name in a workflow definition is unique.
|
||||
- Run flag: Identifies whether this node schedules normally, if it does not need to execute, select the `prohibition execution`.
|
||||
- Descriptive information: Describe the function of the node.
|
||||
- Task priority: When the number of worker threads is insufficient, execute in the order of priority from high to low, and tasks with the same priority will execute in a first-in first-out order.
|
||||
- Worker grouping: Assign tasks to the machines of the worker group to execute. If `Default` is selected, randomly select a worker machine for execution.
|
||||
- Times of failed retry attempts: The number of times the task failed to resubmit. You can select from drop-down or fill-in a number.
|
||||
- Failed retry interval: The time interval for resubmitting the task after a failed task. You can select from drop-down or fill-in a number.
|
||||
- Timeout alarm: Check the timeout alarm and timeout failure. When the task runs exceed the "timeout", an alarm email will send and the task execution will fail.
|
||||
- Target task name: Target task name of this Pigeon node.
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
# Python Node
|
||||
|
||||
## Overview
|
||||
|
||||
Use `Python Task` to create a python-type task and execute python scripts. When the worker executes `Python Task`,
|
||||
it will generate a temporary python script, and executes the script by the Linux user with the same name as the tenant.
|
||||
|
||||
## Create Task
|
||||
|
||||
- Click Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.
|
||||
- Drag <img src="/img/tasks/icons/python.png" width="15"/> from the toolbar to the canvas.
|
||||
|
||||
## Task Parameter
|
||||
|
||||
- Node name: The node name in a workflow definition is unique.
|
||||
- Run flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.
|
||||
- Descriptive information: Describe the function of the node.
|
||||
- Task priority: When the number of worker threads is insufficient, execute in the order of priority from high to low, and tasks with the same priority will execute in a first-in first-out order.
|
||||
- Worker grouping: Assign tasks to the machines of the worker group to execute. If `Default` is selected, randomly select a worker machine for execution.
|
||||
- Environment Name: Configure the environment name in which to run the script.
|
||||
- Number of failed retry attempts: The failure task resubmitting times. It supports drop-down and hand-filling.
|
||||
- Failed retry interval: The time interval for resubmitting the task after a failed task. It supports drop-down and hand-filling.
|
||||
- Timeout alarm: Check the timeout alarm and timeout failure. When the task exceeds the "timeout period", an alarm email will send and the task execution will fail.
|
||||
- Script: Python program developed by the user.
|
||||
- Resource: Refers to the list of resource files that need to be called in the script, and the files uploaded or created by the resource center-file management.
|
||||
- Custom parameters: It is the user-defined parameters of Python, which will replace the content with \${variable} in the script.
|
||||
|
||||
## Task Example
|
||||
|
||||
### Simply Print
|
||||
|
||||
This example simulates a common task that runs by a simple command. The example is to print one line in the log file, as shown in the following figure:
|
||||
"This is a demo of python task".
|
||||
|
||||
|
||||
|
||||
```python
|
||||
print("This is a demo of python task")
|
||||
```
|
||||
|
||||
### Custom Parameters
|
||||
|
||||
This example simulates a custom parameter task. We use parameters for reusing existing tasks as template or coping with the dynamic task. In this case,
|
||||
we declare a custom parameter named "param_key", with the value "param_val". Then we use echo to print the parameter "${param_key}" we just declared.
|
||||
After running this example, we would see "param_val" print in the log.
|
||||
|
||||
|
||||
|
||||
```python
|
||||
print("${param_key}")
|
||||
```
|
||||
|
||||
## Notice
|
||||
|
||||
None
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
# Shell
|
||||
|
||||
## Overview
|
||||
|
||||
Shell task used to create a shell task type and execute a series of shell scripts. When the worker run the shell task, a temporary shell script is generated, and use the Linux user with the same name as the tenant executes the script.
|
||||
|
||||
## Create Task
|
||||
|
||||
- Click `Project -> Management-Project -> Name-Workflow Definition`, and click the `Create Workflow` button to enter the DAG editing page.
|
||||
- Drag from the toolbar <img src="/img/tasks/icons/shell.png" width="15"/> to the canvas.
|
||||
|
||||
## Task Parameter
|
||||
|
||||
- Node name: The node name in a workflow definition is unique.
|
||||
- Run flag: Identifies whether this node schedules normally, if it does not need to execute, select the `prohibition execution`.
|
||||
- Descriptive information: Describe the function of the node.
|
||||
- Task priority: When the number of worker threads is insufficient, execute in the order of priority from high to low, and tasks with the same priority will execute in a first-in first-out order.
|
||||
- Worker grouping: Assign tasks to the machines of the worker group to execute. If `Default` is selected, randomly select a worker machine for execution.
|
||||
- Environment Name: Configure the environment name in which run the script.
|
||||
- Times of failed retry attempts: The number of times the task failed to resubmit. You can select from drop-down or fill-in a number.
|
||||
- Failed retry interval: The time interval for resubmitting the task after a failed task. You can select from drop-down or fill-in a number.
|
||||
- Timeout alarm: Check the timeout alarm and timeout failure. When the task runs exceed the "timeout", an alarm email will send and the task execution will fail.
|
||||
- Script: Shell program developed by users.
|
||||
- Resource: Refers to the list of resource files that called in the script, and upload or create files by the Resource Center file management.
|
||||
- Custom parameters: It is a user-defined local parameter of Shell, and will replace the content with `${variable}` in the script.
|
||||
- Predecessor task: Selecting a predecessor task for the current task, will set the selected predecessor task as upstream of the current task.
|
||||
|
||||
## Task Example
|
||||
|
||||
### Simply Print
|
||||
|
||||
We make an example simulate from a common task which runs by one command. The example is to print one line in the log file, as shown in the following figure:
|
||||
"This is a demo of shell task".
|
||||
|
||||
|
||||
|
||||
### Custom Parameters
|
||||
|
||||
This example simulates a custom parameter task. We use parameters for reusing existing tasks as template or coping with the dynamic task. In this case,
|
||||
we declare a custom parameter named "param_key", with the value "param_val". Then we use `echo` to print the parameter "${param_key}" we just declared.
|
||||
After running this example, we would see "param_val" print in the log.
|
||||
|
||||
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
# Spark Node
|
||||
|
||||
## Overview
|
||||
|
||||
Spark task type used to execute Spark program. For Spark nodes, the worker submits the task by using the spark command `spark submit`. See [spark-submit](https://spark.apache.org/docs/3.2.1/submitting-applications.html#launching-applications-with-spark-submit) for more details.
|
||||
|
||||
## Create Task
|
||||
|
||||
- Click `Project -> Management-Project -> Name-Workflow Definition`, and click the `Create Workflow` button to enter the DAG editing page.
|
||||
- Drag from the toolbar <img src="/img/tasks/icons/spark.png" width="15"/> to the canvas.
|
||||
|
||||
## Task Parameter
|
||||
|
||||
- **Node name**: The node name in a workflow definition is unique.
|
||||
- **Run flag**: Identifies whether this node schedules normally, if it does not need to execute, select the `prohibition execution`.
|
||||
- **Descriptive information**: Describe the function of the node.
|
||||
- **Task priority**: When the number of worker threads is insufficient, execute in the order of priority from high to low, and tasks with the same priority will execute in a first-in first-out order.
|
||||
- **Worker grouping**: Assign tasks to the machines of the worker group to execute. If `Default` is selected, randomly select a worker machine for execution.
|
||||
- **Environment Name**: Configure the environment name in which run the script.
|
||||
- **Times of failed retry attempts**: The number of times the task failed to resubmit.
|
||||
- **Failed retry interval**: The time interval (unit minute) for resubmitting the task after a failed task.
|
||||
- **Delayed execution time**: The time (unit minute) that a task delays in execution.
|
||||
- **Timeout alarm**: Check the timeout alarm and timeout failure. When the task runs exceed the "timeout", an alarm email will send and the task execution will fail.
|
||||
- **Program type**: Supports Java, Scala and Python.
|
||||
- **Spark version**: Support Spark1 and Spark2.
|
||||
- **The class of main function**: The **full path** of Main Class, the entry point of the Spark program.
|
||||
- **Main jar package**: The Spark jar package (upload by Resource Center).
|
||||
- **Deployment mode**: Support 3 deployment modes: yarn-cluster, yarn-client and local.
|
||||
- **Task name** (optional): Spark task name.
|
||||
- **Driver core number**: Set the number of Driver core, which can be set according to the actual production environment.
|
||||
- **Driver memory size**: Set the size of Driver memories, which can be set according to the actual production environment.
|
||||
- **Number of Executor**: Set the number of Executor, which can be set according to the actual production environment.
|
||||
- **Executor memory size**: Set the size of Executor memories, which can be set according to the actual production environment.
|
||||
- **Main program parameters**: Set the input parameters of the Spark program and support the substitution of custom parameter variables.
|
||||
- **Optional parameters**: support `--jars`, `--files`,` --archives`, `--conf` format.
|
||||
- **Resource**: Appoint resource files in the `Resource` if parameters refer to them.
|
||||
- **Custom parameter**: It is a local user-defined parameter for Spark, and will replace the content with `${variable}` in the script.
|
||||
- **Predecessor task**: Selecting a predecessor task for the current task, will set the selected predecessor task as upstream of the current task.
|
||||
|
||||
## Task Example
|
||||
|
||||
### Execute the WordCount Program
|
||||
|
||||
This is a common introductory case in the big data ecosystem, which often apply to computational frameworks such as MapReduce, Flink and Spark. The main purpose is to count the number of identical words in the input text. (Flink's releases attach this example job)
|
||||
|
||||
#### Configure the Spark Environment in DolphinScheduler
|
||||
|
||||
If you are using the Spark task type in a production environment, it is necessary to configure the required environment first. The following is the configuration file: `bin/env/dolphinscheduler_env.sh`.
|
||||
|
||||
|
||||
|
||||
#### Upload the Main Package
|
||||
|
||||
When using the Spark task node, you need to upload the jar package to the Resource Centre for the execution, refer to the [resource center](../resource.md).
|
||||
|
||||
After finish the Resource Centre configuration, upload the required target files directly by dragging and dropping.
|
||||
|
||||
|
||||
|
||||
#### Configure Spark Nodes
|
||||
|
||||
Configure the required content according to the parameter descriptions above.
|
||||
|
||||
|
||||
|
||||
## Notice
|
||||
|
||||
JAVA and Scala only used for identification, there is no difference. If you use Python to develop Spark application, there is no class of the main function and the rest is the same.
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
# SQL
|
||||
|
||||
## Overview
|
||||
|
||||
SQL task type used to connect to databases and execute SQL.
|
||||
|
||||
## Create DataSource
|
||||
|
||||
Refer to [DataSource](../datasource/introduction.md)
|
||||
|
||||
## Create Task
|
||||
|
||||
- Click `Project -> Management-Project -> Name-Workflow Definition`, and click the "Create Workflow" button to enter the DAG editing page.
|
||||
- Drag from the toolbar <img src="/img/tasks/icons/sql.png" width="25"/> to the canvas.
|
||||
|
||||
## Task Parameter
|
||||
|
||||
- Data source: Select the corresponding DataSource.
|
||||
- SQL type: Supports query and non-query. The query is a `select` type query, which is returned with a result set. You can specify three templates for email notification: form, attachment or form attachment. Non-queries return without a result set, three types of operations are: update, delete and insert.
|
||||
- SQL parameter: The input parameter format is `key1=value1;key2=value2...`.
|
||||
- SQL statement: SQL statement.
|
||||
- UDF function: For Hive DataSources, you can refer to UDF functions created in the resource center, but other DataSource do not support UDF functions.
|
||||
- Custom parameters: SQL task type, and stored procedure is a custom parameter order, to set customized parameter type and data type for the method is the same as the stored procedure task type. The difference is that the custom parameter of the SQL task type replaces the `${variable}` in the SQL statement.
|
||||
- Pre-SQL: Pre-SQL executes before the SQL statement.
|
||||
- Post-SQL: Post-SQL executes after the SQL statement.
|
||||
|
||||
## Task Example
|
||||
|
||||
### Create a Temporary Table in Hive and Write Data
|
||||
|
||||
This example creates a temporary table `tmp_hello_world` in Hive and writes a row of data. Before creating a temporary table, we need to ensure that the table does not exist. So we use custom parameters to obtain the time of the day as the suffix of the table name every time we run, this task can run every different day. The format of the created table name is: `tmp_hello_world_{yyyyMMdd}`.
|
||||
|
||||
|
||||
|
||||
### After Running the Task Successfully, Query the Results in Hive
|
||||
|
||||
Log in to the bigdata cluster and use 'hive' command or 'beeline' or 'JDBC' and other methods to connect to the 'Apache Hive' for the query. The query SQL is `select * from tmp_hello_world_{yyyyMMdd}`, please replace `{yyyyMMdd}` with the date of the running day. The following shows the query screenshot:
|
||||
|
||||
|
||||
|
||||
## Notice
|
||||
|
||||
Pay attention to the selection of SQL type. If it is an insert operation, need to change to "Non-Query" type.
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
# Stored Procedure
|
||||
|
||||
- Execute the stored procedure according to the selected DataSource.
|
||||
|
||||
> Drag from the toolbar  task node into the canvas, as shown in the figure below:
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/procedure-en.png" width="80%" />
|
||||
</p>
|
||||
|
||||
- DataSource: The DataSource type of the stored procedure supports MySQL and POSTGRESQL, select the corresponding DataSource.
|
||||
- Method: The method name of the stored procedure.
|
||||
- Custom parameters: The custom parameter types of the stored procedure support `IN` and `OUT`, and the data types support: VARCHAR, INTEGER, LONG, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP and BOOLEAN.
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
# SubProcess Node
|
||||
|
||||
## Overview
|
||||
|
||||
The sub-process node is to execute an external workflow definition as a task node.
|
||||
|
||||
## Create Task
|
||||
|
||||
- Click `Project -> Management-Project -> Name-Workflow Definition`, and click the `Create Workflow` button to enter the DAG editing page.
|
||||
- Drag from the toolbar <img src="/img/tasks/icons/sub_process.png" width="15"/> task node to canvas to create a new SubProcess task.
|
||||
|
||||
## Task Parameter
|
||||
|
||||
- Node name: The node name in a workflow definition is unique.
|
||||
- Run flag: Identifies whether this node schedules normally.
|
||||
- Descriptive information: Describe the function of the node.
|
||||
- Task priority: When the number of worker threads is insufficient, execute in the order of priority from high to low, and tasks with the same priority will execute in a first-in first-out order.
|
||||
- Worker grouping: Assign tasks to the machines of the worker group to execute. If `Default` is selected, randomly select a worker machine for execution.
|
||||
- Environment Name: Configure the environment name in which run the script.
|
||||
- Timeout alarm: Check the timeout alarm and timeout failure. When the task runs exceed the "timeout", an alarm email will send and the task execution will fail.
|
||||
- Sub-node: It is the workflow definition of the selected sub-process. Enter the sub-node in the upper right corner to jump to the workflow definition of the selected sub-process.
|
||||
- Predecessor task: Selecting a predecessor task for the current task, will set the selected predecessor task as upstream of the current task.
|
||||
|
||||
## Task Example
|
||||
|
||||
This example simulates a common task type, here we use a child node task to recall the [Shell](shell.md) to print out "hello". This means executing a shell task as a child node.
|
||||
|
||||
### Create a Shell task
|
||||
|
||||
Create a shell task to print "hello" and define the workflow as `test_dag01`.
|
||||
|
||||
|
||||
|
||||
## Create the Sub_process task
|
||||
|
||||
To use the sub_process, you need to create the sub-node task, which is the shell task we created in the first step. After that, as shown in the diagram below, select the corresponding sub-node in position ⑤.
|
||||
|
||||
|
||||
|
||||
After creating the sub_process, create a corresponding shell task for printing "world" and link both together. Save the current workflow and run it to get the expected result.
|
||||
|
||||
|
||||
|
||||
## Notice
|
||||
|
||||
When using `sub_process` to recall a sub-node task, you need to ensure that the defined sub-node is online status, otherwise, the sub_process workflow will not work properly.
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
# Switch
|
||||
|
||||
The switch is a conditional judgment node, decide the branch executes according to the value of [global variable](../parameter/global.md) and the expression result written by the user.
|
||||
|
||||
## Create
|
||||
|
||||
Drag from the toolbar <img src="/img/switch.png" width="20"/> task node to canvas to create a task.
|
||||
**Note** After created a switch task, you must first configure the upstream and downstream, then configure the parameter of task branches.
|
||||
|
||||
## Parameter
|
||||
|
||||
- Node name: The node name in a workflow definition is unique.
|
||||
- Run flag: Identifies whether this node schedules normally, if it does not need to execute, select the `prohibition execution`.
|
||||
- Descriptive information: Describe the function of the node.
|
||||
- Task priority: When the number of worker threads is insufficient, execute in the order of priority from high to low, and tasks with the same priority will execute in a first-in first-out order.
|
||||
- Worker grouping: Assign tasks to the machines of the worker group to execute. If `Default` is selected, randomly select a worker machine for execution.
|
||||
- Times of failed retry attempts: The number of times the task failed to resubmit. You can select from drop-down or fill-in a number.
|
||||
- Failed retry interval: The time interval for resubmitting the task after a failed task. You can select from drop-down or fill-in a number.
|
||||
- Timeout alarm: Check the timeout alarm and timeout failure. When the task runs exceed the "timeout", an alarm email will send and the task execution will fail.
|
||||
- Condition: You can configure multiple conditions for the switch task. When the conditions are satisfied, execute the configured branch. You can configure multiple different conditions to satisfy different businesses.
|
||||
- Branch flow: The default branch flow, when all the conditions are not satisfied, execute this branch flow.
|
||||
|
||||
## Detail
|
||||
|
||||
Here we have three tasks, the dependencies are `A -> B -> [C, D]`, and `task_a` is a shell task and `task_b` is a switch task
|
||||
|
||||
- In task A, a global variable named `id` is defined through [global variable](../parameter/global.md), and the declaration method is `${setValue(id=1)}`
|
||||
- Task B adds conditions and uses global variables declared upstream to achieve conditional judgment (Note: switch can get the global variables value, as long as its direct or indirect upstream have already assigned the global variables before switch acquires). We want to execute task C when `id = 1`, otherwise run task D
|
||||
- Configure task C to run when the global variable `id=1`. Then edit `${id} == 1` in the condition of task B, and select `C` as branch flow
|
||||
- For other tasks, select `D` as branch flow
|
||||
|
||||
The following shows the switch task configuration:
|
||||
|
||||
|
||||
|
||||
## Related Task
|
||||
|
||||
[condition](conditions.md)：[Condition](conditions.md)task mainly executes the corresponding branch based on the execution result status (success, failure) of the upstream node.
|
||||
The [Switch](switch.md) task mainly executes the corresponding branch based on the value of the [global parameter](../parameter/global.md) and the judgment expression result written by the user.
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
# DolphinScheduler Upgrade Documentation
|
||||
|
||||
## Back-Up Previous Version's Files and Database
|
||||
|
||||
## Stop All Services of DolphinScheduler
|
||||
|
||||
`sh ./script/stop-all.sh`
|
||||
|
||||
## Download the Latest Version Installation Package
|
||||
|
||||
- [download](/en-us/download/download.html) the latest version of the installation packages.
|
||||
- The following upgrade operations need to be performed in the new version's directory.
|
||||
|
||||
## Database Upgrade
|
||||
|
||||
- Change `username` and `password` in `./tools/conf/application.yaml` to yours.
|
||||
|
||||
- If using MySQL as the database to run DolphinScheduler, please config it in `./tools/bin/dolphinscheduler_env.sh`, and add MYSQL connector jar into lib dir `./tools/lib`, here we download `mysql-connector-java-8.0.16.jar`, and then correctly configure database connection information. You can download MYSQL connector jar from [here](https://downloads.MySQL.com/archives/c-j/). Otherwise, PostgreSQL is the default database.
|
||||
|
||||
```shell
|
||||
export DATABASE=${DATABASE:-mysql}
|
||||
```
|
||||
|
||||
- Execute database upgrade script:
|
||||
|
||||
`sh ./tools/bin/upgrade-schema.sh`
|
||||
|
||||
## Backend Service Upgrade
|
||||
|
||||
### Modify the Content in `conf/config/install_config.conf` File
|
||||
|
||||
- Standalone Deployment please refer to the [Standalone-Deployment](./installation/standalone.md).
|
||||
- Cluster Deployment please refer to the [Cluster-Deployment](./installation/cluster.md).
|
||||
|
||||
#### Masters Need Attentions
|
||||
|
||||
Create worker group in 1.3.1 version has a different design:
|
||||
|
||||
- Before version 1.3.1 worker group can be created through UI interface.
|
||||
- Since version 1.3.1 worker group can be created by modifying the worker configuration.
|
||||
|
||||
#### When Upgrade from Version Before 1.3.1 to 1.3.2, the Below Operations are What We Need to Do to Keep Worker Group Configuration Consist with Previous
|
||||
|
||||
1. Go to the backup database, search records in `t_ds_worker_group table`, mainly focus `id, name and IP` three columns.
|
||||
|
||||
| id | name | ip_list |
|
||||
| :--- | :---: | ---: |
|
||||
| 1 | service1 | 192.168.xx.10 |
|
||||
| 2 | service2 | 192.168.xx.11,192.168.xx.12 |
|
||||
|
||||
2. Modify the worker configuration in `conf/config/install_config.conf` file.
|
||||
|
||||
Assume bellow are the machine worker service to be deployed:
|
||||
| hostname | ip |
|
||||
| :--- | :---: |
|
||||
| ds1 | 192.168.xx.10 |
|
||||
| ds2 | 192.168.xx.11 |
|
||||
| ds3 | 192.168.xx.12 |
|
||||
|
||||
To keep worker group config consistent with the previous version, we need to modify workers configuration as below:
|
||||
|
||||
```shell
|
||||
#worker service is deployed on which machine, and also specify which worker group this worker belongs to.
|
||||
workers="ds1:service1,ds2:service2,ds3:service2"
|
||||
```
|
||||
|
||||
#### The Worker Group has Been Enhanced in Version 1.3.2
|
||||
|
||||
Workers in 1.3.1 can't belong to more than one worker group, but in 1.3.2 it's supported. So in 1.3.1 it's not supported when `workers="ds1:service1,ds1:service2"`, and in 1.3.2 it's supported.
|
||||
|
||||
### Execute Deploy Script
|
||||
|
||||
```shell
|
||||
`sh install.sh`
|
||||
```
|
||||
|
||||
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
<!-- markdown-link-check-disable -->
|
||||
|
||||
# Older Versions：
|
||||
|
||||
#### Setup instructions, are available for each stable version of Apache DolphinScheduler below:
|
||||
|
||||
### Versions: 3.0.0-alpha
|
||||
|
||||
#### Links： [3.0.0-alpha Document](../3.0.0/user_doc/about/introduction.md)
|
||||
|
||||
### Versions: 2.0.5
|
||||
|
||||
#### Links： [2.0.5 Document](../2.0.5/user_doc/guide/quick-start.md)
|
||||
|
||||
### Versions: 2.0.3
|
||||
|
||||
#### Links： [2.0.3 Document](../2.0.3/user_doc/guide/quick-start.md)
|
||||
|
||||
### Versions: 2.0.2
|
||||
|
||||
#### Links：[2.0.2 Document](../2.0.2/user_doc/guide/quick-start.md)
|
||||
|
||||
### Versions: 2.0.1
|
||||
|
||||
#### Links：[2.0.1 Document](../2.0.1/user_doc/guide/quick-start.md)
|
||||
|
||||
### Versions: 2.0.0
|
||||
|
||||
#### Links：[2.0.0 Document](../2.0.0/user_doc/guide/quick-start.md)
|
||||
|
||||
### Versions：1.3.9
|
||||
|
||||
#### Links：[1.3.9 Document](../1.3.9/user_doc/quick-start.md)
|
||||
|
||||
### Versions：1.3.8
|
||||
|
||||
#### Links：[1.3.8 Document](../1.3.8/user_doc/quick-start.md)
|
||||
|
||||
### Versions：1.3.6
|
||||
|
||||
#### Links：[1.3.6 Document](../1.3.6/user_doc/quick-start.md)
|
||||
|
||||
### Versions：1.3.5
|
||||
|
||||
#### Links：[1.3.5 Document](../1.3.5/user_doc/quick-start.md)
|
||||
|
||||
### Versions：1.3.4
|
||||
|
||||
##### Links：[1.3.4 Document](../1.3.4/user_doc/quick-start.md)
|
||||
|
||||
### Versions：1.3.3
|
||||
|
||||
#### Links：[1.3.3 Document](../1.3.4/user_doc/quick-start.md)
|
||||
|
||||
### Versions：1.3.2
|
||||
|
||||
#### Links：[1.3.2 Document](../1.3.2/user_doc/quick-start.md)
|
||||
|
||||
### Versions：1.3.1
|
||||
|
||||
#### Links：[1.3.1 Document](../1.3.1/user_doc/quick-start.md)
|
||||
|
||||
### Versions：1.2.1
|
||||
|
||||
#### Links：[1.2.1 Document](../1.2.1/user_doc/quick-start.md)
|
||||
|
||||
### Versions：1.2.0
|
||||
|
||||
#### Links：[1.2.0 Document](../1.2.0/user_doc/quick-start.md)
|
||||
|
||||
### Versions：1.1.0
|
||||
|
||||
#### Links：[1.1.0 Document](../1.2.0/user_doc/quick-start.md)
|
||||
|
||||
### Versions：Dev
|
||||
|
||||
#### Links：[Dev Document](../dev/user_doc/about/introduction.md)
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
## 名词解释
|
||||
|
||||
在对Apache DolphinScheduler了解之前，我们先来认识一下调度系统常用的名词
|
||||
|
||||
### 1.名词解释
|
||||
|
||||
**DAG：** 全称Directed Acyclic Graph，简称DAG。工作流中的Task任务以有向无环图的形式组装起来，从入度为零的节点进行拓扑遍历，直到无后继节点为止。举例如下图：
|
||||
|
||||
<p align="center">
|
||||
<img src="/img/dag_examples_cn.jpg" alt="dag示例" width="60%" />
|
||||
<p align="center">
|
||||
<em>dag示例</em>
|
||||
</p>
|
||||
</p>
|
||||
|
||||
**流程定义**：通过拖拽任务节点并建立任务节点的关联所形成的可视化**DAG**
|
||||
|
||||
**流程实例**：流程实例是流程定义的实例化，可以通过手动启动或定时调度生成,流程定义每运行一次，产生一个流程实例
|
||||
|
||||
**任务实例**：任务实例是流程定义中任务节点的实例化，标识着具体的任务执行状态
|
||||
|
||||
**任务类型**：目前支持有SHELL、SQL、SUB_PROCESS(子流程)、PROCEDURE、MR、SPARK、PYTHON、DEPENDENT(依赖)、，同时计划支持动态插件扩展，注意：其中子 **SUB_PROCESS**
|
||||
也是一个单独的流程定义，是可以单独启动执行的
|
||||
|
||||
**调度方式**：系统支持基于cron表达式的定时调度和手动调度。命令类型支持：启动工作流、从当前节点开始执行、恢复被容错的工作流、恢复暂停流程、从失败节点开始执行、补数、定时、重跑、暂停、停止、恢复等待线程。
|
||||
其中 **恢复被容错的工作流** 和 **恢复等待线程** 两种命令类型是由调度内部控制使用，外部无法调用
|
||||
|
||||
**定时调度**：系统采用 **quartz** 分布式调度器，并同时支持cron表达式可视化的生成
|
||||
|
||||
**依赖**：系统不单单支持 **DAG** 简单的前驱和后继节点之间的依赖，同时还提供**任务依赖**节点，支持**流程间的自定义任务依赖**
|
||||
|
||||
**优先级** ：支持流程实例和任务实例的优先级，如果流程实例和任务实例的优先级不设置，则默认是先进先出
|
||||
|
||||
**邮件告警**：支持 **SQL任务** 查询结果邮件发送，流程实例运行结果邮件告警及容错告警通知
|
||||
|
||||
**失败策略**：对于并行运行的任务，如果有任务失败，提供两种失败策略处理方式，**继续**是指不管并行运行任务的状态，直到流程失败结束。**结束**是指一旦发现失败任务，则同时Kill掉正在运行的并行任务，流程失败结束
|
||||
|
||||
**补数**：补历史数据，支持**区间并行和串行**两种补数方式
|
||||
|
||||
### 2.模块介绍
|
||||
|
||||
- dolphinscheduler-alert 告警模块，提供 AlertServer 服务。
|
||||
|
||||
- dolphinscheduler-api web应用模块，提供 ApiServer 服务。
|
||||
|
||||
- dolphinscheduler-common 通用的常量枚举、工具类、数据结构或者基类
|
||||
|
||||
- dolphinscheduler-dao 提供数据库访问等操作。
|
||||
|
||||
- dolphinscheduler-remote 基于 netty 的客户端、服务端
|
||||
|
||||
- dolphinscheduler-server MasterServer 和 WorkerServer 服务
|
||||
|
||||
- dolphinscheduler-service service模块，包含Quartz、Zookeeper、日志客户端访问服务，便于server模块和api模块调用
|
||||
|
||||
- dolphinscheduler-ui 前端模块
|
||||
|
||||
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
# 软硬件环境建议配置
|
||||
|
||||
DolphinScheduler 作为一款开源分布式工作流任务调度系统，可以很好地部署和运行在 Intel 架构服务器及主流虚拟化环境下，并支持主流的Linux操作系统环境
|
||||
|
||||
## 1. Linux 操作系统版本要求
|
||||
|
||||
| 操作系统 | 版本 |
|
||||
| :----------------------- | :----------: |
|
||||
| Red Hat Enterprise Linux | 7.0 及以上 |
|
||||
| CentOS | 7.0 及以上 |
|
||||
| Oracle Enterprise Linux | 7.0 及以上 |
|
||||
| Ubuntu LTS | 16.04 及以上 |
|
||||
|
||||
> **注意：**
|
||||
>以上 Linux 操作系统可运行在物理服务器以及 VMware、KVM、XEN 主流虚拟化环境上
|
||||
|
||||
## 2. 服务器建议配置
|
||||
DolphinScheduler 支持运行在 Intel x86-64 架构的 64 位通用硬件服务器平台。对生产环境的服务器硬件配置有以下建议：
|
||||
### 生产环境
|
||||
|
||||
| **CPU** | **内存** | **硬盘类型** | **网络** | **实例数量** |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 4核+ | 8 GB+ | SAS | 千兆网卡 | 1+ |
|
||||
|
||||
> **注意：**
|
||||
> - 以上建议配置为部署 DolphinScheduler 的最低配置，生产环境强烈推荐使用更高的配置
|
||||
> - 硬盘大小配置建议 50GB+ ，系统盘和数据盘分开
|
||||
|
||||
|
||||
## 3. 网络要求
|
||||
|
||||
DolphinScheduler正常运行提供如下的网络端口配置：
|
||||
|
||||
| 组件 | 默认端口 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| MasterServer | 5678 | 非通信端口，只需本机端口不冲突即可 |
|
||||
| WorkerServer | 1234 | 非通信端口，只需本机端口不冲突即可 |
|
||||
| ApiApplicationServer | 12345 | 提供后端通信端口 |
|
||||
|
||||
|
||||
> **注意：**
|
||||
> - MasterServer 和 WorkerServer 不需要开启网络间通信，只需本机端口不冲突即可
|
||||
> - 管理员可根据实际环境中 DolphinScheduler 组件部署方案，在网络侧和主机侧开放相关端口
|
||||
|
||||
## 4. 客户端 Web 浏览器要求
|
||||
|
||||
DolphinScheduler 推荐 Chrome 以及使用 Chromium 内核的较新版本浏览器访问前端可视化操作界面
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
# 关于DolphinScheduler
|
||||
|
||||
Apache DolphinScheduler是一个分布式易扩展的可视化DAG工作流任务调度开源系统。解决数据研发ETL 错综复杂的依赖关系，不能直观监控任务健康状态等问题。DolphinScheduler以DAG流式的方式将Task组装起来，可实时监控任务的运行状态，同时支持重试、从指定节点恢复失败、暂停及Kill任务等操作
|
||||
|
||||
# 简单易用
|
||||
DAG监控界面，所有流程定义都是可视化，通过拖拽任务定制DAG，通过API方式与第三方系统对接, 一键部署
|
||||
# 高可靠性
|
||||
去中心化的多Master和多Worker, 自身支持HA功能, 采用任务队列来避免过载，不会造成机器卡死
|
||||
# 丰富的使用场景
|
||||
支持暂停恢复操作.支持多租户，更好的应对大数据的使用场景. 支持更多的任务类型，如 spark, hive, mr, python, sub_process, shell
|
||||
# 高扩展性
|
||||
支持自定义任务类型，调度器使用分布式调度，调度能力随集群线性增长，Master和Worker支持动态上下线
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue