feat: new admin command with ZFS without restic nor encryption, native ZFS encryption

This commit is contained in:
Hordé Nicolas 2025-06-04 01:27:00 +02:00
parent 25cbbddebf
commit c20da7b9ea

585
newadmin.sh Executable file
View File

@ -0,0 +1,585 @@
#!/bin/bash
set -e
# Configuration
LOCAL_TEMP_DIR="/tmp/zfs_backup"
KEEP_FULL_SNAP_MONTHS=10
KEEP_INCR_SNAP_MONTHS=3
DATE=$(date +"%Y-%m-%d")
DAY=$(date +"%d")
DEBUG=0 # Passer à 1 pour activer le mode verbeux (-vv) pour rclone
PCLOUD_CREDENTIALS_FILE="/root/pcloud"
TANG_URL="https://tang.ia86.cc"
mkdir -p "$LOCAL_TEMP_DIR"
suid()
{
if [ "$EUID" -ne 0 ]; then
echo "*** Ce script nécessite des privilèges administrateur. Relance avec sudo..."
exec sudo "$0" "$@"
fi
trap cleanup EXIT
}
cleanup() {
echo "*** Effacement credentials"
umount /mnt/usb 2>/dev/null || true
rm -rf /mnt/usb
rm -rf "$LOCAL_TEMP_DIR"
rm -rf /tmp/credentials-vel
}
mount_tmpfs() {
echo "*** Montage tmpfs pour credentials"
mkdir -p /mnt/usb
mount -t tmpfs tmpfs /mnt/usb
mkdir -p /mnt/usb/rclone
}
encrypt_tang() {
echo -n "$1" | clevis encrypt tang "{\"url\": \"$TANG_URL\"}" | base64 -w0
}
decrypt_tang() {
echo -n "$1" | base64 -d | clevis decrypt tang
}
# Définit RCLONE_OPTS en fonction de DEBUG
if [ "$DEBUG" -eq 1 ]; then
RCLONE_OPTS="--checksum -vv"
else
RCLONE_OPTS="--checksum"
fi
config() {
read -ps "Pcloud Token : " token
echo
read -p "Remote Directory : " remote_dir
cat <<EOF > "$PCLOUD_CREDENTIALS_FILE"
PCLOUD_TOKEN="$(encrypt_tang "$token")"
REMOTE_DIR="$remote_dir"
EOF
chmod 600 "$PCLOUD_CREDENTIALS_FILE"
echo "*** Configuration sauvegardée dans $PCLOUD_CREDENTIALS_FILE"
}
mount_rclone_conf() {
mount_tmpfs
if [ ! -f "$PCLOUD_CREDENTIALS_FILE" ]; then
echo "*** ERREUR : fichier de credentials $PCLOUD_CREDENTIALS_FILE introuvable"
exit 1
fi
source "$PCLOUD_CREDENTIALS_FILE"
PCLOUD_TOKEN=$(decrypt_tang "$PCLOUD_TOKEN")
cat <<EOF > /mnt/usb/rclone/rclone.conf
[pcloud]
type = pcloud
hostname = eapi.pcloud.com
token = {"access_token":"${PCLOUD_TOKEN}","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}
EOF
echo "*** rclone.conf généré"
}
backup() {
local dataset="$1"
echo "*** Dataset: ${dataset:-TOUS}"
mount_rclone_conf
if [ -z "$dataset" ]; then
datasets_to_backup=$(zfs list -H -o name)
else
datasets_to_backup=("$dataset")
fi
for ds in ${datasets_to_backup[@]}; do
echo "*** Dataset: $ds"
DATASET_SAFE_NAME=$(echo "$ds" | tr '/' '_')
REMOTE_PATH="pcloud:${REMOTE_DIR}/$DATASET_SAFE_NAME/"
echo "- Détermination du type de snapshot"
if ! rclone --config /mnt/usb/rclone/rclone.conf ls $RCLONE_OPTS "$REMOTE_PATH" | grep -q "\.zfs\.xz$"; then
SNAPSHOT_TYPE="full"
elif [ "$DAY" -eq "01" ]; then
SNAPSHOT_TYPE="full"
else
SNAPSHOT_TYPE="incr"
fi
SNAPSHOT_NAME="$ds@$DATE-$SNAPSHOT_TYPE"
echo "- Gestion des anciens snapshots"
if zfs list -t snapshot -H -o name | grep -q "^${SNAPSHOT_NAME}$"; then
echo "- Suppression ancien snapshot: $SNAPSHOT_NAME"
zfs destroy "$SNAPSHOT_NAME"
fi
echo "- Création snapshot: $SNAPSHOT_NAME"
zfs snapshot "$SNAPSHOT_NAME"
# Si incrémental, déterminer le snapshot précédent
if [ "$SNAPSHOT_TYPE" = "incr" ]; then
prev_snap=$(zfs list -t snapshot -H -o name -s creation | grep "^${ds}@" | tail -n2 | head -n1)
echo "- Snapshot précédent pour incrémental: $prev_snap"
fi
# Nom de fichier : si full => sans '-full', si incr => avec '-incr'
if [ "$SNAPSHOT_TYPE" = "full" ]; then
FILENAME="$DATASET_SAFE_NAME-$DATE.zfs.xz"
else
FILENAME="$DATASET_SAFE_NAME-$DATE-incr.zfs.xz"
fi
echo "- Compression et envoi streaming"
SHA_FILE="$LOCAL_TEMP_DIR/$FILENAME.sha256"
if [ "$SNAPSHOT_TYPE" = "full" ]; then
send_cmd="zfs send --raw $SNAPSHOT_NAME"
else
send_cmd="zfs send --raw -i $prev_snap $SNAPSHOT_NAME"
fi
echo "- $send_cmd | xz -9e | tee | rclone rcat"
$send_cmd \
| xz -9e \
| tee >(
sha256sum | awk -v fn="$FILENAME" '{print $1 " " fn}' > "$SHA_FILE"
) \
| rclone --config /mnt/usb/rclone/rclone.conf rcat $RCLONE_OPTS "$REMOTE_PATH$FILENAME"
echo "- Envoi du fichier checksum"
rclone --config /mnt/usb/rclone/rclone.conf rcat $RCLONE_OPTS "$REMOTE_PATH$FILENAME.sha256" < "$SHA_FILE"
rm -f "$SHA_FILE"
echo "- Montage temporaire et génération du fichier .list"
MOUNT_DIR="/mnt/tmp_snapshot"
mkdir -p "$MOUNT_DIR"
# Monte temporairement le snapshot en lecture seule
mount -t zfs "$SNAPSHOT_NAME" "$MOUNT_DIR" -o ro
if [ $? -ne 0 ]; then
echo "*** ERREUR : impossible de monter le snapshot $SNAPSHOT_NAME"
exit 1
fi
# Génère la liste des fichiers avec taille, date et chemin complet
LIST_FILE="$LOCAL_TEMP_DIR/$FILENAME.list"
find "$MOUNT_DIR" -printf "%P\t%s\t%TY-%Tm-%Td %TH:%TM:%TS\n" > "$LIST_FILE"
# Démonte le snapshot après utilisation
umount "$MOUNT_DIR"
echo "- Envoi du fichier .list"
rclone --config /mnt/usb/rclone/rclone.conf copy $RCLONE_OPTS "$LIST_FILE" "$REMOTE_PATH"
rm -f "$LIST_FILE"
done
echo "*** Backup terminé"
}
check() {
local dataset="$1"
echo "*** Début vérification: ${dataset:-TOUS}"
mount_rclone_conf
ALL_FILE="$LOCAL_TEMP_DIR/ALL.sha256"
mkdir -p "$LOCAL_TEMP_DIR/check"
echo "" > "$ALL_FILE"
if [ -z "$dataset" ]; then
for dir in $(rclone --config /mnt/usb/rclone/rclone.conf lsf $RCLONE_OPTS "pcloud:${REMOTE_DIR}" --dirs-only); do
dir_name="${dir%/}"
echo "*** Dataset: $dir_name"
DATASET_PATH="pcloud:${REMOTE_DIR}/$dir_name/"
tmpdir="$LOCAL_TEMP_DIR/check/$dir_name"
mkdir -p "$tmpdir"
echo "- Téléchargement .sha256 pour dataset $dir_name"
rclone --config /mnt/usb/rclone/rclone.conf copy $RCLONE_OPTS "$DATASET_PATH" "$tmpdir" --include "*.sha256"
echo "- Ajout au fichier ALL.sha256"
for f in "$tmpdir"/*.sha256; do
sed "s# # $dir_name/#" "$f" >> "$ALL_FILE"
done
done
echo "*** Exécution rclone checksum pour tous les chemins listés dans ALL.sha256"
rclone --config /mnt/usb/rclone/rclone.conf checksum $RCLONE_OPTS sha256 "$ALL_FILE" "pcloud:${REMOTE_DIR}" --exclude "*.sha256" --exclude "*.list"
else
echo "*** Dataset: $dataset"
DATASET_SAFE_NAME=$(echo "$dataset" | tr '/' '_')
DATASET_PATH="pcloud:${REMOTE_DIR}/$DATASET_SAFE_NAME/"
tmpdir="$LOCAL_TEMP_DIR/check/$DATASET_SAFE_NAME"
mkdir -p "$tmpdir"
echo "- Téléchargement .sha256 pour dataset $DATASET_SAFE_NAME"
rclone --config /mnt/usb/rclone/rclone.conf copy $RCLONE_OPTS "$DATASET_PATH" "$tmpdir" --include "*.sha256"
echo "- Ajout au fichier ALL.sha256"
for f in "$tmpdir"/*.sha256; do
cat "$f" >> "$ALL_FILE"
done
echo "*** Exécution rclone checksum pour tous les chemins listés dans ALL.sha256"
rclone --config /mnt/usb/rclone/rclone.conf checksum $RCLONE_OPTS sha256 "$ALL_FILE" "pcloud:${REMOTE_DIR}/${DATASET_SAFE_NAME}" --include "*.xz"
fi
echo "*** Vérification terminée"
}
view() {
echo "*** Lecture du répertoire distant ${REMOTE_DIR} via ncdu"
mount_rclone_conf
rclone --config /mnt/usb/rclone/rclone.conf ncdu $RCLONE_OPTS "pcloud:${REMOTE_DIR}"
}
install() {
echo "*** Installation dépendances"
apt update
apt install -y rclone zfsutils-linux xz-utils clevis tang ncdu
echo "*** Installation terminée"
}
restore() {
dataset="$1"
newdataset="$2"
[ -z "$dataset" ] && { echo "*** ERROR: Dataset required"; exit 1; }
mount_rclone_conf
DATASET_SAFE_NAME=$(echo "$dataset" | tr '/' '_')
REMOTE_PATH="pcloud:${REMOTE_DIR}/$DATASET_SAFE_NAME/"
files=( $(rclone --config /mnt/usb/rclone/rclone.conf lsf "$REMOTE_PATH" | grep '\.zfs\.xz$') )
if [ ${#files[@]} -eq 0 ]; then
echo "*** ERROR: No backups found at $REMOTE_PATH"; exit 1
fi
full_files=( $(printf "%s\n" "${files[@]}" | grep -v '\-incr\.zfs\.xz$') )
if [ ${#full_files[@]} -eq 0 ]; then
echo "*** ERROR: No full backups found."; exit 1
fi
preferred_full=$(printf "%s\n" "${full_files[@]}" | sort | tail -n1)
#echo "- Restoring from full backup: $preferred_full"
# Remove descendants and snapshots
zfs destroy -r "${newdataset:-$dataset}" || true
#rclone --config /mnt/usb/rclone/rclone.conf copy "$REMOTE_PATH$preferred_full" "$LOCAL_TEMP_DIR"
#rclone --config /mnt/usb/rclone/rclone.conf copy "$REMOTE_PATH${preferred_full}.sha256" "$LOCAL_TEMP_DIR"
#(cd "$LOCAL_TEMP_DIR" && sha256sum -c "${preferred_full}.sha256") || { echo "*** ERROR: Checksum failed."; exit 1; }
#xz -d < "$LOCAL_TEMP_DIR/$preferred_full" | zfs receive "${newdataset:-$dataset}"
#rm -f "$LOCAL_TEMP_DIR/$preferred_full" "$LOCAL_TEMP_DIR/${preferred_full}.sha256"
# Incrementals
incr_files=( $(printf "%s\n" "${files[@]}" | grep '\-incr\.zfs\.xz$' | sort) )
for inc in "${incr_files[@]}"; do
echo "- Applying incremental: $inc"
rclone --config /mnt/usb/rclone/rclone.conf copy "$REMOTE_PATH$inc" "$LOCAL_TEMP_DIR"
rclone --config /mnt/usb/rclone/rclone.conf copy "$REMOTE_PATH${inc}.sha256" "$LOCAL_TEMP_DIR"
(cd "$LOCAL_TEMP_DIR" && sha256sum -c "${inc}.sha256") || { echo "*** ERROR: Incremental checksum failed."; exit 1; }
xz -d < "$LOCAL_TEMP_DIR/$inc" | zfs receive -F "${newdataset:-$dataset}"
rm -f "$LOCAL_TEMP_DIR/$inc" "$LOCAL_TEMP_DIR/${inc}.sha256"
done
#echo "*** Activate key location"
#zfs set keylocation=file:///root/key "${newdataset:-$dataset}"
#zfs load-key "${newdataset:-$dataset}"
echo "*** Restoration completed successfully."
}
prune() {
mount_rclone_conf
echo "*** Nettoyage snapshots obsolètes"
zfs list -t snapshot -H -o name | grep "^${ds}@" | while read old_snap; do
SNAP_DATE=$(echo "$old_snap" | cut -d'@' -f2 | cut -d'-' -f1-3)
SNAP_TYPE=$(echo "$old_snap" | grep -oE '(full|incr)$')
RETENTION_MONTHS=$KEEP_INCR_SNAP_MONTHS
[ "$SNAP_TYPE" = "full" ] && RETENTION_MONTHS=$KEEP_FULL_SNAP_MONTHS
if [[ "$(date -d "$SNAP_DATE" +%s)" -lt "$(date -d "$RETENTION_MONTHS months ago" +%s)" ]]; then
echo "- Suppression ancien snapshot: $old_snap"
zfs destroy "$old_snap"
fi
done
datasets=$(zfs list -H -o name)
for ds in ${datasets[@]}; do
echo "*** Pruning dataset: $ds"
DATASET_SAFE_NAME=$(echo "$ds" | tr '/' '_')
REMOTE_PATH="pcloud:${REMOTE_DIR}/$DATASET_SAFE_NAME/"
# Snapshot pruning
zfs list -t snapshot -H -o name | grep "^${ds}@" | while read old_snap; do
SNAP_DATE=$(echo "$old_snap" | cut -d'@' -f2 | cut -d'-' -f1-3)
SNAP_TYPE=$(echo "$old_snap" | grep -oE '(full|incr)$')
RETENTION_MONTHS=$KEEP_INCR_SNAP_MONTHS
[ "$SNAP_TYPE" = "full" ] && RETENTION_MONTHS=$KEEP_FULL_SNAP_MONTHS
if [[ "$(date -d "$SNAP_DATE" +%s)" -lt "$(date -d "$RETENTION_MONTHS months ago" +%s)" ]]; then
echo "- Removing old snapshot: $old_snap"
zfs destroy "$old_snap"
# Determine corresponding remote filename
if [ "$SNAP_TYPE" = "incr" ]; then
FILE_DATE_SUFFIX="$SNAP_DATE-incr"
else
FILE_DATE_SUFFIX="$SNAP_DATE"
fi
FILENAME="$DATASET_SAFE_NAME-$FILE_DATE_SUFFIX.zfs.xz"
echo "- Removing remote backup file: $FILENAME"
rclone deletefile --config /mnt/usb/rclone/rclone.conf $RCLONE_OPTS "$REMOTE_PATH$FILENAME"
# Remove the checksum file as well
echo "- Removing remote checksum file: $FILENAME.sha256"
rclone deletefile --config /mnt/usb/rclone/rclone.conf $RCLONE_OPTS "$REMOTE_PATH$FILENAME.sha256"
fi
done
done
echo "*** Pruning completed"
}
pv_info() {
dataset=$1
pvc_name=$(echo "$dataset" | grep -o "pvc-[a-z0-9-]*")
echo "🔍 PVC detected: $pvc_name"
all=$(kubectl get pv "$pvc_name" -o jsonpath='{.spec.claimRef.namespace}/{.spec.claimRef.name}')
pvc_name=$(echo "$all"|cut -d"/" -f2)
pvc_namespace=$(echo "$all"|cut -d"/" -f1)
echo "📌 Associated PV: $pvc_name"
echo "📌 PVC Namespace: $pvc_namespace"
echo "🔎 Workloads using PVC:"
kubectl get pods,statefulset,deployment,replicaset -n "$pvc_namespace" -o jsonpath='{range .items[*]}{.kind}/{.metadata.name}{"\n"}{end}' | while read resource; do
volumes=$(kubectl get "$resource" -n "$pvc_namespace" -o json | jq '.spec.volumes[].persistentVolumeClaim.claimName' -r 2>/dev/null)
if [[ "$volumes" =~ "$pvc_name" ]]; then
count=$(kubectl get "$resource" -n "$pvc_namespace" -o jsonpath='{.spec.replicas}')
echo "📦 $resource (instances: ${count:-1})"
fi
done
}
save_state() {
NAMESPACE="$1"
STATE_FILE="./${NAMESPACE}.state"
echo "🔍 Saving state to $STATE_FILE"
:> "$STATE_FILE"
kubectl get deployment,statefulset,replicaset -n "$NAMESPACE" -o jsonpath='{range .items[*]}{.kind}/{.metadata.name}{" "}{.spec.replicas}{"\n"}{end}' >> "$STATE_FILE" || true
kubectl get sgcluster -n "$NAMESPACE" -o jsonpath='{range .items[*]}sgcluster/{.metadata.name}{" "}{.spec.instances}{"\n"}{end}' >> "$STATE_FILE" || true
echo "✅ State saved."
}
scale_down() {
NAMESPACE="$1"
STATE_FILE="./${NAMESPACE}.state"
if [ -f "$STATE_FILE" ]; then
echo "❌ State file $STATE_FILE already exist. Abort."
exit 1
fi
save_state "$NAMESPACE"
while read kind_full replicas; do
kind=$(echo "$kind_full" | cut -d'/' -f1 | tr '[:upper:]' '[:lower:]')
name=$(echo "$kind_full" | cut -d'/' -f2)
echo "🛑 Scaling down $kind/$name from $replicas replicas"
if [ "$kind" == "sgcluster" ]; then
kubectl patch sgcluster "$name" -n "$NAMESPACE" --type merge -p '{"spec":{"instances":0}}'
else
kubectl scale "$kind" "$name" -n "$NAMESPACE" --replicas=0
fi
done < "$STATE_FILE"
echo "✅ Scaled down successfully."
}
scale_up() {
NAMESPACE="$1"
STATE_FILE="./${NAMESPACE}.state"
if [ ! -f "$STATE_FILE" ]; then
echo "❌ State file $STATE_FILE not found. Abort."
exit 1
fi
while read kind_full replicas; do
kind=$(echo "$kind_full" | cut -d'/' -f1 | tr '[:upper:]' '[:lower:]')
name=$(echo "$kind_full" | cut -d'/' -f2)
if [ -z "$replicas" ]; then
echo "⚠️ No replicas count found for $kind/$name. Skipping..."
continue
fi
echo "🚀 Scaling up $kind/$name to $replicas replicas"
if [ "$kind" == "sgcluster" ]; then
kubectl patch sgcluster "$name" -n "$NAMESPACE" --type merge -p "{\"spec\":{\"instances\":$replicas}}"
else
kubectl scale "$kind" "$name" -n "$NAMESPACE" --replicas="$replicas"
fi
done < "$STATE_FILE"
rm "$STATE_FILE"
echo "✅ Scaled up successfully and state removed."
}
function veleroapp()
{
case "$1" in
init)
trap cleanup EXIT
read -p "👣 MinIO url (with http/s) : " PATH_MINIO
read -p "👣 Bucket name:: " BUCKET
read -p "🔑 MinIO Access Key: " ACCESS_KEY
read -sp "🔒 MinIO Secret Key: " SECRET_KEY
cat <<EOF > /tmp/credentials-vel
[default]
aws_access_key_id=${ACCESS_KEY}
aws_secret_access_key=${SECRET_KEY}
EOF
echo ""
if [[ ! -f /usr/local/bin/velero ]]; then
echo "🔍 Velero non trouvé, téléchargement et installation de v1.16.0…"
wget -q https://github.com/vmware-tanzu/velero/releases/download/v1.16.0/velero-v1.16.0-linux-amd64.tar.gz
tar -xzf velero-v1.16.0-linux-amd64.tar.gz
chmod +x velero-v1.16.0-linux-amd64/velero
sudo chown root:root velero-v1.16.0-linux-amd64/velero
sudo mv velero-v1.16.0-linux-amd64/velero /usr/local/bin/velero
rm -rf velero-v1.16.0-linux-amd64 velero-v1.16.0-linux-amd64.tar.gz
echo "✅ Velero installé."
else
echo "✅ Velero déjà présent, version : $(velero version --client-only | head -n1)"
fi
echo "▶️ Déploiement Velero dans le cluster…"
velero install \
--provider aws \
--plugins velero/velero-plugin-for-aws:v1.10.0,openebs/velero-plugin:3.6.0 \
--features=EnableCSI \
--use-node-agent \
--bucket ${BUCKET} \
--backup-location-config region=us-east-1,s3ForcePathStyle="true",s3Url=${PATH_MINIO},checksumAlgorithm="" \
--snapshot-location-config region=us-east-1 \
--secret-file /tmp/credentials-vel \
--uploader-type kopia \
--kubeconfig /home/user/.kube/config \
|| echo " Velero est peut-être déjà installé dans le cluster."
echo "🎉 Script terminé."
;;
uninstall)
velero uninstall --kubeconfig /home/user/.kube/config
;;
backup)
NOW=$(date +"%Y%m%d-%H%M")
velero backup create backup-complet-cluster2-${NOW} \
--include-namespaces '*' \
--include-resources '*' \
--include-cluster-resources=true \
--kubeconfig /home/user/.kube/config \
--default-volumes-to-fs-backup
echo "${NOW}" > /var/log/lastfilename
;;
info)
velero backup describe $2 --kubeconfig /home/user/.kube/config
;;
restore)
BACKUP_NAME="$2"
read -p "Are you sure you want to restore from backup '$BACKUP_NAME'? [y/N] " confirm
case "$confirm" in
[Yy]* )
echo "▶️ Restoring from backup '$BACKUP_NAME'…"
velero restore create --from-backup "$BACKUP_NAME" \
--include-namespaces '*' \
--include-resources '*' \
--include-cluster-resources=true \
--kubeconfig /home/user/.kube/config \
--restore-volumes \
;;
* )
echo "❌ Restore aborted."
;;
esac
;;
*)
echo "Usage: admin velero [init|backup|restore|info ] [dataname]"
echo ""
echo "Use the velero backend to make some kubernetes backup operations"
echo ""
echo "init - initialize the data storage on kubernetes"
echo "uninstall - uninstall velero on kubernetes"
echo "backup - backup the kubernetes objects"
echo "restore - restore [dataname] backup on kubernetes"
echo "info - get informations about [dataname]"
echo ""
exit 2
;;
esac
}
# Ajouter le case "prune"
case "$1" in
install)
suid $@
install
;;
config)
suid $@
config
;;
backup)
suid $@
backup "$2"
;;
restore)
suid $@
restore "$2" "$3"
;;
check)
suid $@
check "$2"
;;
view)
suid $@
view
;;
prune)
suid $@
prune
;;
test)
pv_info "$2"
;;
state)
save_state "$2"
;;
down)
scale_down "$2"
;;
up)
scale_up "$2"
;;
velero)
veleroapp $2 $3 $4
;;
*)
echo "Usage: $0 {install|config|backup [dataset]|restore dataset [newdataset]|check [dataset]|view|prune}"
echo "Usage: $0 {test <dataset>|state <namespace>|down <namespace>|up <namespace>}"
exit 1
;;
esac