Dev-631: backup minio

group_vars/stage_prodwork01/backup.yml

@@ -0,0 +1,16 @@
+backup_lvm_hcloudvol_size: 30
+backup_lvm_hcloudvol_count: 8
+
+minio_nsodev_accesskey: "{{ minio_nsodev_accesskey_vault }}"
+minio_nsodev_secretkey: "{{ minio_nsodev_secretkey_vault }}"
+
+minio_cusqa_accesskey: "{{ minio_cusqa_accesskey_vault }}"
+minio_cusqa_secretkey: "{{ minio_cusqa_secretkey_vault }}"
+
+minio_cusprod_accesskey: "{{ minio_cusprod_accesskey_vault }}"
+minio_cusprod_secretkey: "{{ minio_cusprod_secretkey_vault }}"
+
+minio_stage_dicts:
+  - { stage: "nsodev", url: "https://s3storage-nsodev-prodwork01.smardigo.digital", minio_accesskey: "{{ minio_nsodev_accesskey }}", minio_secretkey: "{{ minio_nsodev_secretkey }}", hour: "2", minute: "30"}
+  - { stage: "cusqa", url: "https://s3storage-cusqa-prodwork01.smardigo.digital", minio_accesskey: "{{ minio_cusqa_accesskey }}", minio_secretkey: "{{ minio_cusqa_secretkey }}", hour: "2", minute: "30"}
+  - { stage: "cusprod", url: "https://s3storage-cusprod-prodwork01.smardigo.digital", minio_accesskey: "{{ minio_cusprod_accesskey }}", minio_secretkey: "{{ minio_cusprod_secretkey }}", hour: "2", minute: "30"}

roles/backup_minio/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+system_user: backuphamster
+
+backup_lvm_hcloudvol_size: 10
+backup_lvm_hcloudvol_count: 1
+backup_lvm_hcloudvol_mountpath: '/home/{{ system_user }}/backups'

roles/backup_minio/files/pull_from_minio_server.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+#
+#
+
+MINIO_URL=$1
+STAGE=$2
+ACCESSKEY=$3
+SECRETKEY=$4
+
+
+LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}"
+POSTGRES_BACKUP_DIR="${LOCAL_BACKUP_DIR}/postgres"
+WORDPRESS_BACKUP_DIR="${LOCAL_BACKUP_DIR}/wordpress"
+METRICS_FILE=${HOME}/metrics_${STAGE}.prom
+
+mkdir -p ${POSTGRES_BACKUP_DIR}
+mkdir -p ${WORDPRESS_BACKUP_DIR}
+
+rm -rf ${POSTGRES_BACKUP_DIR}/*
+rm -rf ${WORDPRESS_BACKUP_DIR}/*
+
+mc alias set ${STAGE} ${MINIO_URL} ${ACCESSKEY} ${SECRETKEY}
+
+mc mirror ${STAGE}/postgres ${POSTGRES_BACKUP_DIR} --overwrite --newer-than 1d --json | jq -e '.transferred != 0'
+if [ "$?" -eq "0" ] 
+then
+  NIGHTLY_BACKUP_SUCCESSFUL_POSTGRES="0"
+else
+  NIGHTLY_BACKUP_SUCCESSFUL_POSTGRES="1"
+fi
+
+mc mirror ${STAGE}/wordpress ${WORDPRESS_BACKUP_DIR} --overwrite --newer-than 1d --json | jq -e '.transferred != 0'
+if [ "$?" -eq "0" ] 
+then
+  NIGHTLY_BACKUP_SUCCESSFUL_WORDPRESS="0"
+else
+  NIGHTLY_BACKUP_SUCCESSFUL_WORDPRESS="1"
+fi
+
+mc alias rm ${STAGE}
+
+if [[ ${NIGHTLY_BACKUP_SUCCESSFUL_POSTGRES} -eq "0" ]] && [[ ${NIGHTLY_BACKUP_SUCCESSFUL_WORDPRESS} -eq "0" ]]
+then
+  echo "Nightly Backup Successful - writing METRICS_FILE"
+  cat <<EOF > $METRICS_FILE
+# HELP nightly_backup_successful_$STAGE
+# TYPE nightly_backup_successful_$STAGE gauge
+nightly_backup_successful_$STAGE{stage="$STAGE"} 0
+EOF
+else
+  echo "Nightly Backup Failed - writing METRICS_FILE"
+  cat <<EOF > $METRICS_FILE
+# HELP nightly_backup_successful_$STAGE
+# TYPE nightly_backup_successful_$STAGE gauge
+nightly_backup_successful_$STAGE{stage="$STAGE"} 1
+EOF
+fi

roles/backup_minio/tasks/main.yml

@@ -0,0 +1,96 @@
+---
+
+- name: "Backup storage server | create system user"
+  become: yes
+  ansible.builtin.user:
+    name: '{{ system_user }}'
+    comment: "user for backup"
+    shell: /bin/bash
+  register: create_user
+
+- name: "Create .ssh dir and backups dir"
+  become: yes
+  file:
+    path: '/home/{{ system_user }}/{{ item.name }}/'
+    mode: '{{ item.mode }}'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+    state: directory
+  loop:
+    - name: '.ssh'
+      mode: '0700'
+    - name: 'backups'
+      mode: '0775'
+
+- name: "Create/Resize LVM for datadir"
+  include_role:
+    name: lvm_with_hetzner_volumes
+  vars:
+    lvm_with_hetzner_volumes__volprefix: backup_datadir
+    lvm_with_hetzner_volumes__volsize: "{{ backup_lvm_hcloudvol_size }}"
+    lvm_with_hetzner_volumes__volcount: "{{ backup_lvm_hcloudvol_count }}"
+    lvm_with_hetzner_volumes__mountpath: "{{ backup_lvm_hcloudvol_mountpath }}"
+
+- name: Recursively change ownership of backups directory
+  ansible.builtin.file:
+    path: /home/{{ system_user }}/backups
+    state: directory
+    recurse: yes
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+
+- name: Download minio client
+  become: yes
+  ansible.builtin.get_url:
+    url: https://dl.min.io/client/mc/release/linux-amd64/mc
+    dest: /usr/bin/mc
+    mode: '0755'
+
+# - name: "Providing SSH priv.key"
+#   no_log: true
+#   become: yes
+#   copy:
+#     dest: '/home/{{ system_user }}/.ssh/id_rsa'
+#     mode: '0400'
+#     owner: '{{ system_user }}'
+#     group: '{{ system_user }}'
+#     content: '{{ backup_user_ssh_privkey_vault }}'
+
+- name: "Providing mc client script"
+  become: yes
+  copy:
+    src: '{{ item }}'
+    dest: '/home/{{ system_user }}/{{ item }}'
+    mode: '0755'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+  with_items:
+    - pull_from_minio_server.sh
+
+- name: Create Cron Job for pull_from_minio_server.sh script
+  ansible.builtin.cron:
+    name: "pull minio backups for {{ item.stage }}"
+    hour: "{{ item.hour }}"
+    minute: "{{ item.minute }}"
+    user: '{{ system_user }}'
+    job: "/home/{{ system_user }}/pull_from_minio_server.sh {{ item.url }} {{ item.stage }} {{ item.minio_accesskey }} {{ item.minio_secretkey }}"
+  loop: "{{ minio_stage_dicts }}"
+
+
+- name: Touch metrics_nsodev.prom if not exists
+  file:
+    path: "/home/{{ system_user }}/metrics_{{ item.stage }}.prom"
+    state: touch
+    mode: '0744'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+  loop: "{{ minio_stage_dicts }}"
+
+
+- name: Create symbolic link for node_exporter text nsodev metrics
+  file:
+    src: "/home/{{ system_user }}/metrics_{{ item.stage }}.prom"
+    dest: "/var/lib/prometheus/node-exporter/metrics_{{ item.stage }}.prom"
+    state: link
+  loop: "{{ minio_stage_dicts }}"
+

smardigo.yml

@@ -75,6 +75,9 @@
     - role: backup
       when: "'backup' in group_names"
 
+    - role: backup_minio
+      when: "'backup_minio' in group_names"
+
     - role: keycloak_compact
       when: "'keycloak_compact' in group_names"
 

stage-prodwork01

@@ -1,3 +1,6 @@
+[backup_minio]
+prodwork01-backup-01
+
 [keycloak_compact]
 prodwork01-keycloak-01
 
@@ -29,6 +32,7 @@ kube_node
 [stage_prodwork01:children]
 k8s_cluster
 keycloak_compact
+backup_minio
 
 [all:children]
 stage_prodwork01