Différences

Ci-dessous, les différences entre deux révisions de la page.

--- admin:infrastructure:backup [2019/05/31 09:02] – cpm
+++ admin:infrastructure:backup [2021/01/02 17:53] (Version actuelle) – [Configuration du monitoring] fpoulain
@@ Ligne 7: / Ligne 7: @@
 ==== Script de backup ====
-<code bash src/backup.sh>
+C'est fournis par [[https://torsion.org/borgmatic/ | borgmatic]].
-#! /bin/bash
-sleep $[$RANDOM % 30]m
+On y adjoint une configuration dans ''/etc'' :
-logger="/var/log/backup.log"
+<code yaml /etc/borgmatic.d/root.yaml>
-borg_bin="/usr/bin/borg"
+location:
-backup_name=`date +%Y-%m-%d`
+  source_directories:
-backup_dest="backup@backup.chapril.org:/srv/backups/`hostname --fqdn`"
+    - /
+  exclude_patterns:
+    - '/dev'
+    - '/media/*'
+    - '/mnt/*'
+    - '/proc'
+    - '/run/*'
+    - '/srv/backups/*.chapril.org'
+    - '/sys'
+    - '/var/cache/*'
+    - '/var/lib/backuppc/*'
+    - '/var/lib/libvirt/coon'
+    - '/var/lib/libvirt/images/'
+    - '/var/lib/libvirt/maine'
+  repositories:
+    - 'backup@backup.chapril.org:/srv/backups/{fqdn}'
-export BORG_RSH="ssh -p 2242 -A"
+storage:
+  ssh_command: ssh -p 2242 -A
+  archive_name_format: '{now:%Y-%m-%dT%H:%M:%S}'
+  # pour bullseye : borg_cache_directory: /var/cache/borg
-echo ======================================================================== 	>> $logger
+consistency:
-echo "                              New backup" 				>> $logger
+  check_last: 2
-echo ======================================================================== 	>> $logger
+  prefix: '20'
-date 						>> $logger
-echo "" 					>> $logger
-echo "== Executing package selection: " 	>> $logger
+retention:
-date 						>> $logger
+  keep_daily: 7
-echo "" 					>> $logger
+  keep_weekly: 4
-dpkg --get-selections > /root/package-selections
+  prefix: '20'
-echo "== Backup pre-hook"			>> $logger
+hooks:
-date 						>> $logger
+  before_backup:
-echo "" 					>> $logger
+    - echo "Launching root backup at $(date -Iseconds)"
+    - for file in /etc/borg/scripts/pre-hooks/*  ; do test -e "$file" || continue; echo "Executing $file..."; $file; done
+  after_backup:
+    - for file in /etc/borg/scripts/post-hooks/* ; do test -e "$file" || continue; echo "Executing $file..."; $file; done
+    - echo "Succeeded root backup at $(date -Iseconds)"
+  on_error:
+    - echo "Failed root backup at $(date -Iseconds)"
+# pour bullseye :
+#  after_check:
+#    - echo "Succeeded root checks at $(date -Iseconds)"
+#  after_prune:
+#    - echo "Succeeded root prune at $(date -Iseconds)"
+</code>
-for file in /etc/borg/scripts/pre-hooks/*; do
+==== Entrée Systemd ====
-	echo "Executing $file...			" >> $logger
-	$file
-done
-echo "== Backup launch"				>> $logger
+On déclenche avec un timer systemd qui retarde le démarrage avec un timing aléatoire pour éviter le ddos de [[admin:machines_virtuelles:felicette|Félicette]].
-date 						>> $logger
-echo "" 					>> $logger
-$borg_bin create --stats $backup_dest::$backup_name / \
+<code conf /etc/systemd/system/borgmatic.timer>
-	--exclude /proc \
+[Unit]
-	--exclude /dev  \
+Description=Run borgmatic backup
-	--exclude /sys  \
-	--exclude /var/lib/libvirt/coon \
-	--exclude /var/lib/libvirt/maine \
-	--exclude /var/lib/libvirt/images/  \
-        --exclude '/srv/backups/*.chapril.org' \
->&1 >> $logger
-rc=$?
+[Timer]
-if [[ $rc != 0 ]]; then exit $rc; fi
+# Will trigger at 01:00 each day
+# + 0-60 random minutes
+# + 30 minutes delay from borgmatic.service
+OnCalendar=*-*-* 01:00:00
+Persistent=true
+RandomizedDelaySec=60 minutes
-echo "== Backup info"				>> $logger
+[Install]
-date 						>> $logger
+WantedBy=timers.target
-echo "" 					>> $logger
+</code>
-$borg_bin info $backup_dest::$backup_name 2>&1 >> $logger
+<code conf /etc/systemd/system/borgmatic.service>
+[Unit]
+Description=borgmatic backup
+Wants=network-online.target
+After=network-online.target
+ConditionACPower=true
-echo "== Backup pruning"			>> $logger
+[Service]
-date 						>> $logger
+Type=oneshot
-echo "" 					>> $logger
-$borg_bin prune -v --keep-daily=7 --keep-weekly=5 $backup_dest 2>&1 >> $logger
+## Lower CPU and I/O priority.
+Nice=19
+CPUSchedulingPolicy=batch
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+IOWeight=100
-echo "== Backup post-hook"			>> $logger
+## Logs
-date 						>> $logger
+StandardOutput=syslog
-echo "" 					>> $logger
+StandardError=syslog
+SyslogIdentifier=borgmatic
-for file in /etc/borg/scripts/post-hooks/*; do
+# Prevent rate limiting of borgmatic log events.
-    [[ -f "$file" ]] || continue
+LogRateLimitIntervalSec=0
-	echo "Executing $file...			" >> $logger
-	$file
-done
-echo "" 					>> $logger
-date 						>> $logger
-echo Returned $rc 				>> $logger
-echo ======================================================================== >> $logger
-exit $rc
-</code>
-==== Entrée Cron ====
-<code cron src/backup>
+## Launcher
-2 * * * root bash /srv/bin/backup.sh
+# Delay start to prevent backups immediately upon system startup
+ExecStartPre=sleep 30m
+ExecStart=borgmatic -v1
+Restart=no
 </code>
@@ Ligne 126: / Ligne 146: @@
     su - postgres -c "pg_dump $db" | bzip2 - > $backup_dir/$db.sql.bz2
 done
+</code>
+<code bash scripts/pre-hooks/dump-influxdb>
+#!/bin/bash
+if test -x /usr/bin/influxd ; then
+    backup_dir=/var/backups/influxdb
+    db=icinga2
+    # Prepare.
+    mkdir -p $backup_dir
+    chmod 700 $backup_dir
+    # Backup.
+    influxd backup -portable -database $db -host localhost:8088 $backup_dir/$db
+    # Prune.
+    find $backup_dir/$db -type f -mtime +2 -delete
+fi
 </code>
@@ Ligne 223: / Ligne 262: @@
 exit 0
+</code>
+==== Rsyslog ====
+<code conf /etc/rsyslog.d/borgmatic.conf>
+if $programname == 'borgmatic' then /var/log/borgmatic.log
+& stop
 </code>
 ==== Log rotate ====
-<code conf debian/logrotate>
+<code conf debian/borgmatic>
-/var/log/backup.log {
+/var/log/borgmatic.log
-        weekly
+{
-        rotate 52
+  rotate 6
-        compress
+  weekly
-        delaycompress
+  compress
-        missingok
+  missingok
-        notifempty
+  notifempty
-        create 644 root root
 }
 </code>
@@ Ligne 242: / Ligne 287: @@
 C'est surtout du ssh.
-<code ssh __icinga2__/etc/ssh/authorized_keys/backup>
+<code ssh __felicette__/etc/ssh/authorized_keys/backup>
 command="borg serve --restrict-to-path /srv/backups/dns.cluster.chapril.org",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc ssh-ed25519 ... root@dns.cluster.chapril.org
 command="borg serve --restrict-to-path /srv/backups/admin.cluster.chapril.org",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc ssh-ed25519 ... root@admin.cluster.chapril.org
@@ Ligne 252: / Ligne 297: @@
 On a un script qui parse sur chaque machine le log de backup et qui est déployé par le paquet monitoring-plugins-chapril :
-<code python /usr/lib/nagios/plugins/check_backup>
+<code python /usr/lib/nagios/plugins/check_borgmatic>
-#!/usr/bin/env python
+#!/usr/bin/env python3
-# -*- encoding:utf8 -*-
-import datetime, os, re, locale
+import datetime, itertools, os, re
-today= datetime.datetime.now ()
+now = datetime.datetime.now(datetime.timezone.utc)
-max_backup_delay = datetime.timedelta (1, 7200)
+max_backup_delay = datetime.timedelta(1, 7200)
-def last_backup (log_file):
+def get_name(match):
-    with open(log_file) as s:
+    return match.group('name')
-        logs_ok = re.findall (r'^([ a-zéûA-Z:,0-9]*)( \(UTC\+0[12]00\))?\nReturned 0\n={30}', s.read (), re.MULTILINE)[-1][0]
-        print "Last backup : " + logs_ok
-        try:
-            return datetime.datetime.strptime (logs_ok, '%a %b %d %X %Z %Y')
-        except:
-            locale.setlocale(locale.LC_ALL, 'fr_FR.UTF-8')
-            return datetime.datetime.strptime (logs_ok, '%A %d %B %Y, %X')
+def check_backup(filename):
+    with open(filename) as f:
+        logs = f.read()
+        mixed_statuses = list(re.finditer(r'(?P<status>Succeeded|Failed) (?P<name>\w+) backup at (?P<date>\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\+\d\d:\d\d)$', logs, re.MULTILINE))
+        for name, statuses in itertools.groupby(sorted(mixed_statuses, key=get_name), key=get_name):
+            last = sorted(statuses, key=lambda x: x.group('date'))[-1]
+            print('{name}: {status} at {date}'.format(**last.groupdict()))
+            last_date = datetime.datetime.fromisoformat(last.group('date'))
+            last_status = last.group('status')
+            if last_status != 'Succeeded' or now - last_date > max_backup_delay:
+                failure.append(name)
+failure = []
 try:
-    last_backup_date= last_backup ("/var/log/backup.log")
+    check_backup ("/var/log/borgmatic.log")
-except:
+except Exception:
-    last_backup_date= last_backup ("/var/log/backup.log.1")
+    check_backup ("/var/log/borgmatic.log.1")
-if today - last_backup_date < max_backup_delay:
+if failure:
+    exit (1)
+else:
     exit (0)
-else:
-    exit (1)
 </code>
@@ Ligne 285: / Ligne 335: @@
 <code conf __admin__/etc/icinga2/zones.d/global-templates/services/backups.conf>
 object CheckCommand "backup" {
-	command = [ PluginDir + "/check_backup" ]
+	command = [ "sudo", PluginDir + "/check_borgmatic" ]
 }
@@ Ligne 297: / Ligne 347: @@
 }
 </code>
 ===== Aspects contrôle d'intégrité =====
-On contrôle directement chaque nuit sur la machine où les backups sont stockés ([[admin:machines_virtuelles:icinga2|Icinga2]]).
+On contrôle directement chaque nuit sur la machine où les backups sont stockés ([[admin:machines_virtuelles:felicette|Félicette]]).
 ==== Script de contrôle ====
-<code bash __icinga2__/srv/bin/check_backup.sh>
+<code bash __felicette__/srv/bin/check_backup.sh>
 #! /bin/bash
@@ Ligne 342: / Ligne 391: @@
 ==== Entrée Cron ====
-<code cron __icinga2__/etc/cron.d/check_backup>
+<code cron __felicette__/etc/cron.d/check_backup>
 4 * * * root bash /srv/bin/check_backup.sh
 </code>
@@ Ligne 348: / Ligne 397: @@
 ==== Log rotate ====
-<code conf __icinga2__/etc/logrotate.d/check-backup>
+<code conf __felicette__/etc/logrotate.d/check-backup>
 /var/log/check_backup.log {
         weekly
@@ Ligne 363: / Ligne 412: @@
 On a un script qui parse sur la machine le log de check_backup :
-<code python __icinga2__/usr/local/lib/nagios/plugins/check_check_backup>
+<code python __felicette__/usr/local/lib/nagios/plugins/check_check_backup>
 #!/usr/bin/env python
 # -*- encoding:utf8 -*-
@@ Ligne 407: / Ligne 456: @@
   command_endpoint = host.vars.client_endpoint
-  assign where host.name == "icinga2.cluster.chapril.org"
+  assign where host.name == "felicette.cluster.chapril.org"
 }
 </code>