re-written Magda postgresql dockfile image & add wal-g for point-in-t…

…ime recovery & backup

deploy/helm/internal-charts/postgresql-wrapper/templates/cronjob-backup.yaml

@@ -0,0 +1,34 @@
+{{- if .Values.backupRestore.backup.enabled -}}
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: "{{ .Values.dbModuleName }}-backup-jobs"
+spec:
+  concurrencyPolicy: Forbid
+  schedule: {{ .Values.wal-g.backup.schedule | quote }}
+  failedJobsHistoryLimit: 1
+  successfulJobsHistoryLimit: 1
+  startingDeadlineSeconds: 120
+  jobTemplate:
+    spec:
+      backoffLimit: 3
+      completions: 1
+      parallelism: 1
+      template:
+        metadata:
+          name: "{{ .Values.dbModuleName }}-backup-job"
+          labels:
+            cron: "{{ .Values.dbModuleName }}-backup-jobs"
+        spec:
+{{- include "magda.pullSecrets" . | indent 10 }}
+          containers:
+          - image: "{{ .Values.backupRestore.image.repository }}:{{ .Values.backupRestore.image.tag }}"
+            imagePullPolicy: "{{ .Values.backupRestore.image.repository }}"
+            name: db-backup
+            command:
+              - "/bin/sh"
+              - "-c"
+              - "apk add --no-cache ca-certificates curl && curl -i -X POST http://indexer/v0/reindex"
+              - "{{ typeof "sdsd" }}"
+          restartPolicy: OnFailure
+{{- end }}

deploy/helm/internal-charts/postgresql-wrapper/values.yaml

@@ -8,6 +8,11 @@ global:
     postgresqlDatabase: "postgres"
     # -- Whether auto create secret for db superuser account password
     autoCreateSecret: true
+
+# -- a name used to naming common k8s objects (e.g. backup jobs) of this chart to avoid name conflicts.
+# e.g. combined-db chart might want to set this field to "combined-db" while session-db will want to set this field to "session-db".
+dbModuleName: "default-db"
+
 postgresql:
   image: 
     tag: "13.3.0-debian-10-r79"
@@ -21,3 +26,14 @@ postgresql:
       cpu: 200m
       memory: 500Mi
 
+backupRestore:
+  image:
+    # -- wal-g docker image tag
+    tag: "1.1.0-debian-10-r3"
+    # -- wal-g docker image repo
+    repository: "bitnami/wal-g"
+     # -- wal-g docker image pull policy
+    pullPolicy: IfNotPresent
+  backup:
+    enabled: true
+

deploy/helm/magda-core/templates/_helpers.tpl

@@ -15,6 +15,30 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{- define "magda.pullSecrets" -}}
+  {{- $pullSecrets := list }}
+  {{- if eq (typeOf .Values.image.pullSecret) "string" }}
+    {{- $pullSecrets = append $pullSecrets .Values.image.pullSecret }}
+  {{- end }}
+  {{- if eq (typeOf .Values.image.pullSecrets) "[]interface {}" }}
+    {{- $pullSecrets = concat $pullSecrets .Values.image.pullSecrets }}
+  {{- end }}
+  {{- if empty $pullSecrets }}
+    {{- if eq (typeOf .Values.global.image.pullSecret) "string" }}
+      {{- $pullSecrets = append $pullSecrets .Values.global.image.pullSecret }}
+    {{- end }}
+    {{- if eq (typeOf .Values.global.image.pullSecrets) "[]interface {}" }}
+      {{- $pullSecrets = concat $pullSecrets .Values.global.image.pullSecrets }}
+    {{- end }}
+  {{- end }}
+  {{- if (not (empty $pullSecrets)) }}
+imagePullSecrets:
+    {{- range $pullSecrets }}
+  - name: {{ . }}
+    {{- end }}
+  {{- end }}
+{{- end -}}
+
 {{- define "dockerimage" -}}
 "{{ .Values.image.repository | default .Values.global.image.repository }}/magda-{{ .Chart.Name }}:{{ .Values.image.tag | default .Values.global.image.tag | default .Chart.Version }}"
 {{- end -}}

magda-postgres/Dockerfile

@@ -1,28 +1,4 @@
-FROM postgres:9.6.15
+FROM bitnami/postgresql:13.3.0-debian-10-r79
 
-# Install wall-e
-RUN apt-get update && \
-    apt-get install -y patch && \
-    apt-get install -y python3-pip lzop pv && \
-    python3 -m pip install --upgrade pip && \
-    python3 -m pip install wal-e[aws,google,swift] && \
-# Install postgresql pl debugger
-# The debugger module won't be loaded until you modify the postgresql.conf with:
-# shared_preload_libraries = ‘$libdir/other_libraries/plugin_debugger’
-# We will do it in start.sh depends on env var switch
-# After that, you need to run `CREATE EXTENSION pldbgapi;` 
-# in the database that you want to debug to turn it on database level
-    apt-get install -y postgresql-9.6-pldebugger
-
-COPY component/start.sh /usr/local/bin/
-
-# docker-entrypoint.sh (from the parent image) frustratingly only creates the database
-# if the "next" command it runs is "postgres".  But we want to do a migration after
-# the normal startup, so we can't simply launch postgres.  So here we monkey patch
-# the startup script to allow chaining to any command.  This is arguably better than
-# the alternative of forking the upstream docker-entrypoint.sh entirely.
-COPY component/docker-entrypoint.sh.patch /usr/local/bin
-RUN patch -i /usr/local/bin/docker-entrypoint.sh.patch /usr/local/bin/docker-entrypoint.sh
-
-ENTRYPOINT ["docker-entrypoint.sh"]
-CMD ["start.sh", "postgres"]
+# copy wal-g from pre-built image
+COPY --from=bitnami/wal-g:1.1.0-debian-10-r3 /opt/bitnami/wal-g/bin/wal-g /usr/local/bin/wal-g

magda-postgres/package.json

@@ -13,7 +13,7 @@
   "config": {
     "docker": {
       "name": "data61/magda-postgres",
-      "include": "Dockerfile start.sh docker-entrypoint.sh.patch"
+      "include": "Dockerfile"
     }
   },
   "devDependencies": {