Différences

Ci-dessous, les différences entre deux révisions de la page.

--- admin:infrastructure:backup [2018/11/26 10:54] – créée fpoulain
+++ admin:infrastructure:backup [2024/12/29 09:19] (Version actuelle) – [Script de backup] correction d'une typo dans l'URL du remote (suppression ":") pilou
@@ Ligne 1: / Ligne 1: @@
-Le backup de chapril est déployé via un paquet Debian sur un repo privé. On décrit ici les points essentiels.
 ====== Paquet Debian ======
-===== Script de backup =====
+Le backup de Chapril est déployé via un paquet Debian sur un repo privé. On décrit ici les points essentiels du paquet à défaut de publier le repo, ainsi que la configuration du contrôle d'intégrité des archives.
-<code bash src/backup.sh>
+===== Aspects backup =====
-#! /bin/bash
-sleep $[$RANDOM % 30]m
+==== Script de backup ====
-logger="/var/log/backup.log"
+C'est fournis par [[https://torsion.org/borgmatic/ | borgmatic]].
-borg_bin="/usr/bin/borg"
-backup_name=`date +%Y-%m-%d`
-backup_dest="backup@backup.chapril.org:/srv/backups/`hostname --fqdn`"
-export BORG_RSH="ssh -p 2242 -A"
+On y adjoint une configuration dans ''/etc'' :
-echo ======================================================================== 	>> $logger
+<code yaml /etc/borgmatic.d/root.yaml>
-echo "                              New backup" 				>> $logger
+location:
-echo ======================================================================== 	>> $logger
+  source_directories:
-date 						>> $logger
+    - /
-echo "" 					>> $logger
+  exclude_patterns:
+    - '/dev'
+    - '/media/*'
+    - '/mnt/*'
+    - '/proc'
+    - '/run/*'
+    - '/srv/backups/*.chapril.org'
+    - '/sys'
+    - '/var/cache/*'
+    - '/var/lib/backuppc/*'
+    - '/var/lib/libvirt/images/'
+  repositories:
+    - 'ssh://backup@backup.chapril.org/srv/backups/{fqdn}'
-echo "== Executing package selection: " 	>> $logger
+storage:
-date 						>> $logger
+  ssh_command: ssh -p 2242 -A
-echo "" 					>> $logger
+  archive_name_format: '{now:%Y-%m-%dT%H:%M:%S}'
-dpkg --get-selections > /root/package-selections
+  # pour bullseye : borg_cache_directory: /var/cache/borg
-echo "== Backup pre-hook"			>> $logger
+consistency:
-date 						>> $logger
+  check_last: 2
-echo "" 					>> $logger
+  prefix: '20'
-for file in /etc/borg/scripts/pre-hooks/*; do
+retention:
-	echo "Executing $file...			" >> $logger
+  keep_daily: 7
-	$file
+  keep_weekly: 4
-done
+  prefix: '20'
-echo "== Backup launch"				>> $logger
+hooks:
-date 						>> $logger
+  before_backup:
-echo "" 					>> $logger
+    - echo "Launching root backup at $(date -Iseconds)"
+    - for file in /etc/borg/scripts/pre-hooks/*  ; do test -e "$file" || continue; echo "Executing $file..."; $file; done
+  after_backup:
+    - for file in /etc/borg/scripts/post-hooks/* ; do test -e "$file" || continue; echo "Executing $file..."; $file; done
+    - echo "Succeeded root backup at $(date -Iseconds)"
+    - borgmatic info --archive latest --json
+  on_error:
+    - echo "Failed root backup at $(date -Iseconds)"
+# pour bullseye :
+#  after_check:
+#    - echo "Succeeded root checks at $(date -Iseconds)"
+#  after_prune:
+#    - echo "Succeeded root prune at $(date -Iseconds)"
+</code>
-$borg_bin create --stats $backup_dest::$backup_name / \
+==== Entrée Systemd ====
-	--exclude /proc \
-	--exclude /dev  \
-	--exclude /sys  \
-	--exclude /var/lib/libvirt/coon \
-	--exclude /var/lib/libvirt/maine \
-	--exclude /var/lib/libvirt/images/  \
-        --exclude '/srv/backups/*.chapril.org' \
->&1 >> $logger
-rc=$?
+On déclenche avec un timer systemd qui retarde le démarrage avec un timing aléatoire pour éviter le ddos de [[admin:machines_virtuelles:felicette|Félicette]].
-if [[ $rc != 0 ]]; then exit $rc; fi
-echo "== Backup info"				>> $logger
+<code conf /etc/systemd/system/borgmatic.timer>
-date 						>> $logger
+[Unit]
-echo "" 					>> $logger
+Description=Run borgmatic backup
-$borg_bin info $backup_dest::$backup_name 2>&1 >> $logger
+[Timer]
+# Will trigger at 01:00 each day
+# + 0-60 random minutes
+# + 30 minutes delay from borgmatic.service
+OnCalendar=*-*-* 01:00:00
+Persistent=true
+RandomizedDelaySec=60 minutes
-echo "== Backup pruning"			>> $logger
+[Install]
-date 						>> $logger
+WantedBy=timers.target
-echo "" 					>> $logger
+</code>
-$borg_bin prune -v --keep-daily=7 --keep-weekly=5 $backup_dest 2>&1 >> $logger
+<code conf /etc/systemd/system/borgmatic.service>
+[Unit]
+Description=borgmatic backup
+Wants=network-online.target
+After=network-online.target
+ConditionACPower=true
-echo "== Backup post-hook"			>> $logger
+[Service]
-date 						>> $logger
+Type=oneshot
-echo "" 					>> $logger
-for file in /etc/borg/scripts/post-hooks/*; do
+## Lower CPU and I/O priority.
-    [[ -f "$file" ]] || continue
+Nice=19
-	echo "Executing $file...			" >> $logger
+CPUSchedulingPolicy=batch
-	$file
+IOSchedulingClass=best-effort
-done
+IOSchedulingPriority=7
+IOWeight=100
-echo "" 					>> $logger
-date 						>> $logger
-echo Returned $rc 				>> $logger
-echo ======================================================================== >> $logger
-exit $rc
-</code>
-===== Entrée Cron =====
+## Logs
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=borgmatic
+# Prevent rate limiting of borgmatic log events.
+LogRateLimitIntervalSec=0
-<code cron src/backup>
+## Launcher
-2 * * * root bash /srv/bin/backup.sh
+# Delay start to prevent backups immediately upon system startup
+ExecStartPre=sleep 30m
+ExecStart=borgmatic -v1
+Restart=no
 </code>
-===== Scripts de pre hooks =====
+==== Scripts de pre hooks ====
 <code bash scripts/pre-hooks/dump-mysql>
@@ Ligne 124: / Ligne 145: @@
     su - postgres -c "pg_dump $db" | bzip2 - > $backup_dir/$db.sql.bz2
 done
+</code>
+<code bash scripts/pre-hooks/dump-influxdb>
+#!/bin/bash
+if test -x /usr/bin/influxd ; then
+    backup_dir=/var/backups/influxdb
+    db=icinga2
+    # Prepare.
+    mkdir -p $backup_dir
+    chmod 700 $backup_dir
+    # Backup.
+    influxd backup -portable -database $db -host localhost:8088 $backup_dir/$db
+    # Prune.
+    find $backup_dir/$db -type f -mtime +2 -delete
+fi
 </code>
@@ Ligne 134: / Ligne 174: @@
 </code>
-===== Script de post install =====
+==== Script de post install ====
 <code bash debian/postinst>
@@ Ligne 223: / Ligne 263: @@
 </code>
-===== Log rotate =====
+==== Rsyslog ====
-<code conf debian/logrotate>
+<code conf /etc/rsyslog.d/borgmatic.conf>
-/var/log/backup.log {
+if $programname == 'borgmatic' then /var/log/borgmatic.log
-        weekly
+& stop
-        rotate 52
+</code>
-        compress
-        delaycompress
+==== Log rotate ====
-        missingok
-        notifempty
+<code conf debian/borgmatic>
-        create 644 root root
+/var/log/borgmatic.log
+{
+  rotate 6
+  weekly
+  compress
+  missingok
+  notifempty
 }
 </code>
-====== Configuration de l'hote ======
+===== Configuration de l'hote =====
 C'est surtout du ssh.
-<code ssh /etc/ssh/authorized_keys/backup>
+<code ssh __felicette__/etc/ssh/authorized_keys/backup>
 command="borg serve --restrict-to-path /srv/backups/dns.cluster.chapril.org",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc ssh-ed25519 ... root@dns.cluster.chapril.org
 command="borg serve --restrict-to-path /srv/backups/admin.cluster.chapril.org",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc ssh-ed25519 ... root@admin.cluster.chapril.org
@@ Ligne 247: / Ligne 293: @@
 </code>
-====== Configuration du monitoring ======
+===== Configuration du monitoring =====
 On a un script qui parse sur chaque machine le log de backup et qui est déployé par le paquet monitoring-plugins-chapril :
-<code python /usr/lib/nagios/plugins/check_backup>
+<code python /usr/lib/nagios/plugins/check_borgmatic>
+#!/usr/bin/env python3
+import datetime, itertools, os, re
+now = datetime.datetime.now(datetime.timezone.utc)
+max_backup_delay = datetime.timedelta(1, 7200)
+def get_name(match):
+    return match.group('name')
+def check_backup(filename):
+    with open(filename) as f:
+        logs = f.read()
+        mixed_statuses = list(re.finditer(r'(?P<status>Succeeded|Failed) (?P<name>\w+) backup at (?P<date>\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\+\d\d:\d\d)$', logs, re.MULTILINE))
+        for name, statuses in itertools.groupby(sorted(mixed_statuses, key=get_name), key=get_name):
+            last = sorted(statuses, key=lambda x: x.group('date'))[-1]
+            print('{name}: {status} at {date}'.format(**last.groupdict()))
+            last_date = datetime.datetime.fromisoformat(last.group('date'))
+            last_status = last.group('status')
+            if last_status != 'Succeeded' or now - last_date > max_backup_delay:
+                failure.append(name)
+failure = []
+try:
+    check_backup ("/var/log/borgmatic.log")
+except Exception:
+    check_backup ("/var/log/borgmatic.log.1")
+if failure:
+    exit (1)
+else:
+    exit (0)
+</code>
+Et la conf icinga2 :
+<code conf __admin__/etc/icinga2/zones.d/global-templates/services/backups.conf>
+object CheckCommand "backup" {
+	command = [ "sudo", PluginDir + "/check_borgmatic" ]
+}
+apply Service "Backup " {
+  import "generic-service"
+  check_command = "backup"
+  command_endpoint = host.vars.client_endpoint
+  assign where host.address && !host.vars.external
+}
+</code>
+===== Aspects contrôle d'intégrité =====
+On contrôle directement chaque nuit sur la machine où les backups sont stockés ([[admin:machines_virtuelles:felicette|Félicette]]).
+==== Script de contrôle ====
+<code bash __felicette__/srv/bin/check_backup.sh>
+#! /bin/bash
+logger="/var/log/check_backup.log"
+borg_bin="/usr/bin/borg"
+backup_dest="/srv/backups/"
+echo ======================================================================== 	>> $logger
+echo "                           New backup check" 				                >> $logger
+echo ======================================================================== 	>> $logger
+date 						>> $logger
+echo "" 					>> $logger
+cd $backup_dest
+for repository in $(ls -d $backup_dest/*$(hostname -d))
+do
+    echo "==  Checking $repository"		>> $logger
+    date 						        >> $logger
+    echo "" 					        >> $logger
+    $borg_bin check $repository    2>&1 >> $logger
+    rc=$?
+    if [[ $rc != 0 ]]; then exit $rc; fi
+done
+echo "" 					>> $logger
+date 						>> $logger
+echo Returned $rc 				>> $logger
+echo ========================================================================   >> $logger
+exit $rc
+</code>
+==== Entrée Cron ====
+<code cron __felicette__/etc/cron.d/check_backup>
+4 * * * root bash /srv/bin/check_backup.sh
+</code>
+==== Log rotate ====
+<code conf __felicette__/etc/logrotate.d/check-backup>
+/var/log/check_backup.log {
+        weekly
+        rotate 52
+        compress
+        delaycompress
+        missingok
+        notifempty
+        create 644 backup backup
+}
+</code>
+==== Configuration du monitoring ====
+On a un script qui parse sur la machine le log de check_backup :
+<code python __felicette__/usr/local/lib/nagios/plugins/check_check_backup>
 #!/usr/bin/env python
 # -*- encoding:utf8 -*-
@@ Ligne 262: / Ligne 423: @@
     with open(log_file) as s:
         logs_ok = re.findall (r'^([ a-zéûA-Z:,0-9]*)( \(UTC\+0[12]00\))?\nReturned 0\n={30}', s.read (), re.MULTILINE)[-1][0]
-        print "Last backup : " + logs_ok
+        print "Last backup check : " + logs_ok
         try:
             return datetime.datetime.strptime (logs_ok, '%a %b %d %X %Z %Y')
@@ Ligne 270: / Ligne 431: @@
 try:
-    last_backup_date= last_backup ("/var/log/backup.log")
+    last_backup_date= last_backup ("/var/log/check_backup.log")
 except:
-    last_backup_date= last_backup ("/var/log/backup.log.1")
+    last_backup_date= last_backup ("/var/log/check_backup.log.1")
 if today - last_backup_date < max_backup_delay:
@@ Ligne 278: / Ligne 439: @@
 else:
     exit (1)
+</code>
+Et la conf icinga2 :
+<code conf __admin__/etc/icinga2/zones.d/global-templates/services/backups.conf>
+object CheckCommand "check_backup" {
+    command = [ "/usr/local/lib/nagios/plugins/check_check_backup" ]
+}
+</code>
+<code conf __admin__/etc/icinga2/zones.d/master/galanga/icinga2.conf>
+/* Backup checks */
+apply Service "Check Backup " {
+  import "generic-service"
+  check_command = "check_backup"
+  command_endpoint = host.vars.client_endpoint
+  assign where host.name == "felicette.cluster.chapril.org"
+}
 </code>