seg_yinzy
/
DolphinScheduler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[Improvement-5405][Docker/K8s] Adapt to registry SPI (#5681)

dev-master-refactor
Shiwen Cheng 2021-06-23 11:30:15 +08:00 committed by GitHub
parent 664206cb81
commit 93f1d4df7d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 36 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docker/build/conf/dolphinscheduler/registry.properties.tpl
View File

@ -15,9 +15,13 @@
# limitations under the License.
#
registry.plugin.name=${REGISTRY_PLUGIN_NAME}
#registry.plugin.dir config the Registry Plugin dir.
registry.plugin.dir=${REGISTRY_PLUGIN_DIR}
#registry.plugin.binding=registry
registry.plugin.name=${REGISTRY_PLUGIN_NAME}
registry.servers=${REGISTRY_SERVERS}
#maven.local.repository=/usr/local/localRepository
#registry.plugin.binding config the Registry Plugin need be load when development and run in IDE
#registry.plugin.binding=./dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml

2
docker/build/startup-init-conf.sh
View File

@ -37,7 +37,7 @@ export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
#============================================================================
# Registry
#============================================================================
export REGISTRY_PLUGIN_DIR=${REGISTRY_PLUGIN_DIR:-"lib/plugin/registry/zookeeper"}
export REGISTRY_PLUGIN_DIR=${REGISTRY_PLUGIN_DIR:-"lib/plugin/registry"}
export REGISTRY_PLUGIN_NAME=${REGISTRY_PLUGIN_NAME:-"zookeeper"}
export REGISTRY_SERVERS=${REGISTRY_SERVERS:-"127.0.0.1:2181"}

3
docker/docker-swarm/config.env.sh
View File

@ -37,9 +37,8 @@ DATABASE_PARAMS=characterEncoding=utf8
# DATABASE_PARAMS=useUnicode=true&characterEncoding=UTF-8
#============================================================================
# ZooKeeper
# Registry
#============================================================================
REGISTRY_PLUGIN_DIR=lib/plugin/registry
REGISTRY_PLUGIN_NAME=zookeeper
REGISTRY_SERVERS=dolphinscheduler-zookeeper:2181

28
docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
View File

@ -162,21 +162,27 @@ Create a database environment variables.
{{- end }}
{{- end -}}
{{/* todo
Create a rregistry environment variables.
{{/*
Create a registry environment variables.
*/}}
{{- define "dolphinscheduler.zookeeper.env_vars" -}}
- name: ZOOKEEPER_QUORUM
{{- define "dolphinscheduler.registry.env_vars" -}}
- name: REGISTRY_PLUGIN_DIR
{{- if .Values.zookeeper.enabled }}
value: "lib/plugin/registry"
{{- else }}
value: {{ .Values.externalRegistry.registryPluginDir }}
{{- end }}
- name: REGISTRY_PLUGIN_NAME
{{- if .Values.zookeeper.enabled }}
value: "zookeeper"
{{- else }}
value: {{ .Values.externalRegistry.registryPluginName }}
{{- end }}
- name: REGISTRY_SERVERS
{{- if .Values.zookeeper.enabled }}
value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- name: ZOOKEEPER_ROOT
{{- if .Values.zookeeper.enabled }}
value: {{ .Values.zookeeper.zookeeperRoot }}
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperRoot }}
value: {{ .Values.externalRegistry.registryServers }}
{{- end }}
{{- end -}}

2
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
View File

@ -68,7 +68,7 @@ spec:
- name: TZ
value: {{ .Values.timezone }}
{{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
{{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }}
{{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }}
{{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
envFrom:
- configMapRef:

2
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
View File

@ -65,7 +65,7 @@ spec:
- name: TZ
value: {{ .Values.timezone }}
{{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
{{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }}
{{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }}
{{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
envFrom:
- configMapRef:

2
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
View File

@ -69,7 +69,7 @@ spec:
- name: ALERT_LISTEN_HOST
value: {{ include "dolphinscheduler.fullname" . }}-alert
{{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
{{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }}
{{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }}
{{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
envFrom:
- configMapRef:

18
docker/kubernetes/dolphinscheduler/values.yaml
View File

@ -27,7 +27,7 @@ image:
pullPolicy: "IfNotPresent"
pullSecret: ""
## If not exists external database, by default, Dolphinscheduler's database will use it.
## If not exists external database, the postgresql database will be used by default.
postgresql:
enabled: true
postgresqlUsername: "root"
@ -38,8 +38,7 @@ postgresql:
size: "20Gi"
storageClass: "-"
## If exists external database, and set postgresql.enable value to false.
## external database will be used, otherwise Dolphinscheduler's database will be used.
## If exists external database and set postgresql.enable value to false, the external database will be used.
externalDatabase:
type: "postgresql"
driver: "org.postgresql.Driver"
@ -50,7 +49,7 @@ externalDatabase:
database: "dolphinscheduler"
params: "characterEncoding=utf8"
## If not exists external zookeeper, by default, Dolphinscheduler's zookeeper will use it.
## If not exists external registry, the zookeeper registry will be used by default.
zookeeper:
enabled: true
fourlwCommandsWhitelist: "srvr,ruok,wchs,cons"
@ -58,13 +57,12 @@ zookeeper:
enabled: false
size: "20Gi"
storageClass: "-"
zookeeperRoot: "/dolphinscheduler"
## If exists external zookeeper, and set zookeeper.enable value to false.
## If zookeeper.enable is false, Dolphinscheduler's zookeeper will use it.
externalZookeeper:
zookeeperQuorum: "127.0.0.1:2181"
zookeeperRoot: "/dolphinscheduler"
## If exists external registry and set zookeeper.enable value to false, the external registry will be used.
externalRegistry:
registryPluginDir: "lib/plugin/registry"
registryPluginName: "zookeeper"
registryServers: "127.0.0.1:2181"
common:
## Configmap

1
dolphinscheduler-alert/src/main/resources/logback-alert.xml
View File

@ -46,7 +46,6 @@
</appender>
<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="ALERTLOGFILE"/>
</root>

2
dolphinscheduler-server/src/main/resources/logback-master.xml
View File

@ -30,8 +30,6 @@
</encoder>
</appender>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<!-- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>

2
dolphinscheduler-service/src/main/resources/registry.properties
View File

@ -25,5 +25,3 @@ registry.servers=127.0.0.1:2181
#registry.plugin.binding config the Registry Plugin need be load when development and run in IDE
#registry.plugin.binding=./dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml