From c20da7b9ea085cb5f171e8de4731d2e5cb6147b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hord=C3=A9=20Nicolas?= Date: Wed, 4 Jun 2025 01:27:00 +0200 Subject: [PATCH] feat: new admin command with ZFS without restic nor encryption, native ZFS encryption --- newadmin.sh | 585 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 585 insertions(+) create mode 100755 newadmin.sh diff --git a/newadmin.sh b/newadmin.sh new file mode 100755 index 0000000..49afb18 --- /dev/null +++ b/newadmin.sh @@ -0,0 +1,585 @@ +#!/bin/bash +set -e +# Configuration +LOCAL_TEMP_DIR="/tmp/zfs_backup" +KEEP_FULL_SNAP_MONTHS=10 +KEEP_INCR_SNAP_MONTHS=3 +DATE=$(date +"%Y-%m-%d") +DAY=$(date +"%d") +DEBUG=0 # Passer à 1 pour activer le mode verbeux (-vv) pour rclone + +PCLOUD_CREDENTIALS_FILE="/root/pcloud" +TANG_URL="https://tang.ia86.cc" + +mkdir -p "$LOCAL_TEMP_DIR" + +suid() +{ + if [ "$EUID" -ne 0 ]; then + echo "*** Ce script nécessite des privilèges administrateur. Relance avec sudo..." + exec sudo "$0" "$@" + fi + trap cleanup EXIT +} + +cleanup() { + echo "*** Effacement credentials" + umount /mnt/usb 2>/dev/null || true + rm -rf /mnt/usb + rm -rf "$LOCAL_TEMP_DIR" + rm -rf /tmp/credentials-vel +} + +mount_tmpfs() { + echo "*** Montage tmpfs pour credentials" + mkdir -p /mnt/usb + mount -t tmpfs tmpfs /mnt/usb + mkdir -p /mnt/usb/rclone +} + +encrypt_tang() { + echo -n "$1" | clevis encrypt tang "{\"url\": \"$TANG_URL\"}" | base64 -w0 +} + +decrypt_tang() { + echo -n "$1" | base64 -d | clevis decrypt tang +} + +# Définit RCLONE_OPTS en fonction de DEBUG +if [ "$DEBUG" -eq 1 ]; then + RCLONE_OPTS="--checksum -vv" +else + RCLONE_OPTS="--checksum" +fi + +config() { + read -ps "Pcloud Token : " token + echo + read -p "Remote Directory : " remote_dir + + cat < "$PCLOUD_CREDENTIALS_FILE" +PCLOUD_TOKEN="$(encrypt_tang "$token")" +REMOTE_DIR="$remote_dir" +EOF + + chmod 600 "$PCLOUD_CREDENTIALS_FILE" + echo "*** Configuration sauvegardée dans $PCLOUD_CREDENTIALS_FILE" +} + +mount_rclone_conf() { + mount_tmpfs + + if [ ! -f "$PCLOUD_CREDENTIALS_FILE" ]; then + echo "*** ERREUR : fichier de credentials $PCLOUD_CREDENTIALS_FILE introuvable" + exit 1 + fi + + source "$PCLOUD_CREDENTIALS_FILE" + PCLOUD_TOKEN=$(decrypt_tang "$PCLOUD_TOKEN") + + cat < /mnt/usb/rclone/rclone.conf +[pcloud] +type = pcloud +hostname = eapi.pcloud.com +token = {"access_token":"${PCLOUD_TOKEN}","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"} +EOF + echo "*** rclone.conf généré" +} + +backup() { + local dataset="$1" + echo "*** Dataset: ${dataset:-TOUS}" + mount_rclone_conf + + if [ -z "$dataset" ]; then + datasets_to_backup=$(zfs list -H -o name) + else + datasets_to_backup=("$dataset") + fi + + for ds in ${datasets_to_backup[@]}; do + echo "*** Dataset: $ds" + DATASET_SAFE_NAME=$(echo "$ds" | tr '/' '_') + REMOTE_PATH="pcloud:${REMOTE_DIR}/$DATASET_SAFE_NAME/" + + echo "- Détermination du type de snapshot" + if ! rclone --config /mnt/usb/rclone/rclone.conf ls $RCLONE_OPTS "$REMOTE_PATH" | grep -q "\.zfs\.xz$"; then + SNAPSHOT_TYPE="full" + elif [ "$DAY" -eq "01" ]; then + SNAPSHOT_TYPE="full" + else + SNAPSHOT_TYPE="incr" + fi + + SNAPSHOT_NAME="$ds@$DATE-$SNAPSHOT_TYPE" + echo "- Gestion des anciens snapshots" + if zfs list -t snapshot -H -o name | grep -q "^${SNAPSHOT_NAME}$"; then + echo "- Suppression ancien snapshot: $SNAPSHOT_NAME" + zfs destroy "$SNAPSHOT_NAME" + fi + + echo "- Création snapshot: $SNAPSHOT_NAME" + zfs snapshot "$SNAPSHOT_NAME" + + # Si incrémental, déterminer le snapshot précédent + if [ "$SNAPSHOT_TYPE" = "incr" ]; then + prev_snap=$(zfs list -t snapshot -H -o name -s creation | grep "^${ds}@" | tail -n2 | head -n1) + echo "- Snapshot précédent pour incrémental: $prev_snap" + fi + + # Nom de fichier : si full => sans '-full', si incr => avec '-incr' + if [ "$SNAPSHOT_TYPE" = "full" ]; then + FILENAME="$DATASET_SAFE_NAME-$DATE.zfs.xz" + else + FILENAME="$DATASET_SAFE_NAME-$DATE-incr.zfs.xz" + fi + + echo "- Compression et envoi streaming" + SHA_FILE="$LOCAL_TEMP_DIR/$FILENAME.sha256" + + if [ "$SNAPSHOT_TYPE" = "full" ]; then + send_cmd="zfs send --raw $SNAPSHOT_NAME" + else + send_cmd="zfs send --raw -i $prev_snap $SNAPSHOT_NAME" + fi + + echo "- $send_cmd | xz -9e | tee | rclone rcat" + $send_cmd \ + | xz -9e \ + | tee >( + sha256sum | awk -v fn="$FILENAME" '{print $1 " " fn}' > "$SHA_FILE" + ) \ + | rclone --config /mnt/usb/rclone/rclone.conf rcat $RCLONE_OPTS "$REMOTE_PATH$FILENAME" + + echo "- Envoi du fichier checksum" + rclone --config /mnt/usb/rclone/rclone.conf rcat $RCLONE_OPTS "$REMOTE_PATH$FILENAME.sha256" < "$SHA_FILE" + rm -f "$SHA_FILE" + + echo "- Montage temporaire et génération du fichier .list" + MOUNT_DIR="/mnt/tmp_snapshot" + mkdir -p "$MOUNT_DIR" + # Monte temporairement le snapshot en lecture seule + mount -t zfs "$SNAPSHOT_NAME" "$MOUNT_DIR" -o ro + if [ $? -ne 0 ]; then + echo "*** ERREUR : impossible de monter le snapshot $SNAPSHOT_NAME" + exit 1 + fi + # Génère la liste des fichiers avec taille, date et chemin complet + LIST_FILE="$LOCAL_TEMP_DIR/$FILENAME.list" + find "$MOUNT_DIR" -printf "%P\t%s\t%TY-%Tm-%Td %TH:%TM:%TS\n" > "$LIST_FILE" + # Démonte le snapshot après utilisation + umount "$MOUNT_DIR" + echo "- Envoi du fichier .list" + rclone --config /mnt/usb/rclone/rclone.conf copy $RCLONE_OPTS "$LIST_FILE" "$REMOTE_PATH" + rm -f "$LIST_FILE" + done + echo "*** Backup terminé" +} + +check() { + local dataset="$1" + + echo "*** Début vérification: ${dataset:-TOUS}" + mount_rclone_conf + + ALL_FILE="$LOCAL_TEMP_DIR/ALL.sha256" + mkdir -p "$LOCAL_TEMP_DIR/check" + echo "" > "$ALL_FILE" + + if [ -z "$dataset" ]; then + for dir in $(rclone --config /mnt/usb/rclone/rclone.conf lsf $RCLONE_OPTS "pcloud:${REMOTE_DIR}" --dirs-only); do + dir_name="${dir%/}" + echo "*** Dataset: $dir_name" + DATASET_PATH="pcloud:${REMOTE_DIR}/$dir_name/" + tmpdir="$LOCAL_TEMP_DIR/check/$dir_name" + mkdir -p "$tmpdir" + + echo "- Téléchargement .sha256 pour dataset $dir_name" + rclone --config /mnt/usb/rclone/rclone.conf copy $RCLONE_OPTS "$DATASET_PATH" "$tmpdir" --include "*.sha256" + + echo "- Ajout au fichier ALL.sha256" + for f in "$tmpdir"/*.sha256; do + sed "s# # $dir_name/#" "$f" >> "$ALL_FILE" + done + done + echo "*** Exécution rclone checksum pour tous les chemins listés dans ALL.sha256" + rclone --config /mnt/usb/rclone/rclone.conf checksum $RCLONE_OPTS sha256 "$ALL_FILE" "pcloud:${REMOTE_DIR}" --exclude "*.sha256" --exclude "*.list" + else + echo "*** Dataset: $dataset" + DATASET_SAFE_NAME=$(echo "$dataset" | tr '/' '_') + DATASET_PATH="pcloud:${REMOTE_DIR}/$DATASET_SAFE_NAME/" + tmpdir="$LOCAL_TEMP_DIR/check/$DATASET_SAFE_NAME" + mkdir -p "$tmpdir" + + echo "- Téléchargement .sha256 pour dataset $DATASET_SAFE_NAME" + rclone --config /mnt/usb/rclone/rclone.conf copy $RCLONE_OPTS "$DATASET_PATH" "$tmpdir" --include "*.sha256" + + echo "- Ajout au fichier ALL.sha256" + for f in "$tmpdir"/*.sha256; do + cat "$f" >> "$ALL_FILE" + done + echo "*** Exécution rclone checksum pour tous les chemins listés dans ALL.sha256" + rclone --config /mnt/usb/rclone/rclone.conf checksum $RCLONE_OPTS sha256 "$ALL_FILE" "pcloud:${REMOTE_DIR}/${DATASET_SAFE_NAME}" --include "*.xz" + fi + echo "*** Vérification terminée" +} + +view() { + echo "*** Lecture du répertoire distant ${REMOTE_DIR} via ncdu" + mount_rclone_conf + rclone --config /mnt/usb/rclone/rclone.conf ncdu $RCLONE_OPTS "pcloud:${REMOTE_DIR}" +} + +install() { + echo "*** Installation dépendances" + apt update + apt install -y rclone zfsutils-linux xz-utils clevis tang ncdu + echo "*** Installation terminée" +} + +restore() { + dataset="$1" + newdataset="$2" + + [ -z "$dataset" ] && { echo "*** ERROR: Dataset required"; exit 1; } + mount_rclone_conf + + DATASET_SAFE_NAME=$(echo "$dataset" | tr '/' '_') + REMOTE_PATH="pcloud:${REMOTE_DIR}/$DATASET_SAFE_NAME/" + + files=( $(rclone --config /mnt/usb/rclone/rclone.conf lsf "$REMOTE_PATH" | grep '\.zfs\.xz$') ) + + if [ ${#files[@]} -eq 0 ]; then + echo "*** ERROR: No backups found at $REMOTE_PATH"; exit 1 + fi + + full_files=( $(printf "%s\n" "${files[@]}" | grep -v '\-incr\.zfs\.xz$') ) + + if [ ${#full_files[@]} -eq 0 ]; then + echo "*** ERROR: No full backups found."; exit 1 + fi + + preferred_full=$(printf "%s\n" "${full_files[@]}" | sort | tail -n1) + + #echo "- Restoring from full backup: $preferred_full" + + # Remove descendants and snapshots + zfs destroy -r "${newdataset:-$dataset}" || true + + #rclone --config /mnt/usb/rclone/rclone.conf copy "$REMOTE_PATH$preferred_full" "$LOCAL_TEMP_DIR" + #rclone --config /mnt/usb/rclone/rclone.conf copy "$REMOTE_PATH${preferred_full}.sha256" "$LOCAL_TEMP_DIR" + + #(cd "$LOCAL_TEMP_DIR" && sha256sum -c "${preferred_full}.sha256") || { echo "*** ERROR: Checksum failed."; exit 1; } + + #xz -d < "$LOCAL_TEMP_DIR/$preferred_full" | zfs receive "${newdataset:-$dataset}" + + #rm -f "$LOCAL_TEMP_DIR/$preferred_full" "$LOCAL_TEMP_DIR/${preferred_full}.sha256" + + # Incrementals + incr_files=( $(printf "%s\n" "${files[@]}" | grep '\-incr\.zfs\.xz$' | sort) ) + + for inc in "${incr_files[@]}"; do + echo "- Applying incremental: $inc" + rclone --config /mnt/usb/rclone/rclone.conf copy "$REMOTE_PATH$inc" "$LOCAL_TEMP_DIR" + rclone --config /mnt/usb/rclone/rclone.conf copy "$REMOTE_PATH${inc}.sha256" "$LOCAL_TEMP_DIR" + + (cd "$LOCAL_TEMP_DIR" && sha256sum -c "${inc}.sha256") || { echo "*** ERROR: Incremental checksum failed."; exit 1; } + + xz -d < "$LOCAL_TEMP_DIR/$inc" | zfs receive -F "${newdataset:-$dataset}" + + rm -f "$LOCAL_TEMP_DIR/$inc" "$LOCAL_TEMP_DIR/${inc}.sha256" + done + #echo "*** Activate key location" + #zfs set keylocation=file:///root/key "${newdataset:-$dataset}" + #zfs load-key "${newdataset:-$dataset}" + + echo "*** Restoration completed successfully." +} + +prune() { + mount_rclone_conf + echo "*** Nettoyage snapshots obsolètes" + zfs list -t snapshot -H -o name | grep "^${ds}@" | while read old_snap; do + SNAP_DATE=$(echo "$old_snap" | cut -d'@' -f2 | cut -d'-' -f1-3) + SNAP_TYPE=$(echo "$old_snap" | grep -oE '(full|incr)$') + RETENTION_MONTHS=$KEEP_INCR_SNAP_MONTHS + [ "$SNAP_TYPE" = "full" ] && RETENTION_MONTHS=$KEEP_FULL_SNAP_MONTHS + + if [[ "$(date -d "$SNAP_DATE" +%s)" -lt "$(date -d "$RETENTION_MONTHS months ago" +%s)" ]]; then + echo "- Suppression ancien snapshot: $old_snap" + zfs destroy "$old_snap" + fi + done + datasets=$(zfs list -H -o name) + for ds in ${datasets[@]}; do + echo "*** Pruning dataset: $ds" + DATASET_SAFE_NAME=$(echo "$ds" | tr '/' '_') + REMOTE_PATH="pcloud:${REMOTE_DIR}/$DATASET_SAFE_NAME/" + + # Snapshot pruning + zfs list -t snapshot -H -o name | grep "^${ds}@" | while read old_snap; do + SNAP_DATE=$(echo "$old_snap" | cut -d'@' -f2 | cut -d'-' -f1-3) + SNAP_TYPE=$(echo "$old_snap" | grep -oE '(full|incr)$') + + RETENTION_MONTHS=$KEEP_INCR_SNAP_MONTHS + [ "$SNAP_TYPE" = "full" ] && RETENTION_MONTHS=$KEEP_FULL_SNAP_MONTHS + + if [[ "$(date -d "$SNAP_DATE" +%s)" -lt "$(date -d "$RETENTION_MONTHS months ago" +%s)" ]]; then + echo "- Removing old snapshot: $old_snap" + zfs destroy "$old_snap" + + # Determine corresponding remote filename + if [ "$SNAP_TYPE" = "incr" ]; then + FILE_DATE_SUFFIX="$SNAP_DATE-incr" + else + FILE_DATE_SUFFIX="$SNAP_DATE" + fi + + FILENAME="$DATASET_SAFE_NAME-$FILE_DATE_SUFFIX.zfs.xz" + echo "- Removing remote backup file: $FILENAME" + rclone deletefile --config /mnt/usb/rclone/rclone.conf $RCLONE_OPTS "$REMOTE_PATH$FILENAME" + + # Remove the checksum file as well + echo "- Removing remote checksum file: $FILENAME.sha256" + rclone deletefile --config /mnt/usb/rclone/rclone.conf $RCLONE_OPTS "$REMOTE_PATH$FILENAME.sha256" + fi + done + done + + echo "*** Pruning completed" +} + +pv_info() { + dataset=$1 + pvc_name=$(echo "$dataset" | grep -o "pvc-[a-z0-9-]*") + echo "🔍 PVC detected: $pvc_name" + all=$(kubectl get pv "$pvc_name" -o jsonpath='{.spec.claimRef.namespace}/{.spec.claimRef.name}') + pvc_name=$(echo "$all"|cut -d"/" -f2) + pvc_namespace=$(echo "$all"|cut -d"/" -f1) + echo "📌 Associated PV: $pvc_name" + echo "📌 PVC Namespace: $pvc_namespace" + echo "🔎 Workloads using PVC:" + kubectl get pods,statefulset,deployment,replicaset -n "$pvc_namespace" -o jsonpath='{range .items[*]}{.kind}/{.metadata.name}{"\n"}{end}' | while read resource; do + volumes=$(kubectl get "$resource" -n "$pvc_namespace" -o json | jq '.spec.volumes[].persistentVolumeClaim.claimName' -r 2>/dev/null) + if [[ "$volumes" =~ "$pvc_name" ]]; then + count=$(kubectl get "$resource" -n "$pvc_namespace" -o jsonpath='{.spec.replicas}') + echo "📦 $resource (instances: ${count:-1})" + fi + done +} + +save_state() { + NAMESPACE="$1" + STATE_FILE="./${NAMESPACE}.state" + + echo "🔍 Saving state to $STATE_FILE" + :> "$STATE_FILE" + + kubectl get deployment,statefulset,replicaset -n "$NAMESPACE" -o jsonpath='{range .items[*]}{.kind}/{.metadata.name}{" "}{.spec.replicas}{"\n"}{end}' >> "$STATE_FILE" || true + + kubectl get sgcluster -n "$NAMESPACE" -o jsonpath='{range .items[*]}sgcluster/{.metadata.name}{" "}{.spec.instances}{"\n"}{end}' >> "$STATE_FILE" || true + + echo "✅ State saved." +} + +scale_down() { + NAMESPACE="$1" + STATE_FILE="./${NAMESPACE}.state" + + if [ -f "$STATE_FILE" ]; then + echo "❌ State file $STATE_FILE already exist. Abort." + exit 1 + fi + + save_state "$NAMESPACE" + + while read kind_full replicas; do + kind=$(echo "$kind_full" | cut -d'/' -f1 | tr '[:upper:]' '[:lower:]') + name=$(echo "$kind_full" | cut -d'/' -f2) + echo "🛑 Scaling down $kind/$name from $replicas replicas" + + if [ "$kind" == "sgcluster" ]; then + kubectl patch sgcluster "$name" -n "$NAMESPACE" --type merge -p '{"spec":{"instances":0}}' + else + kubectl scale "$kind" "$name" -n "$NAMESPACE" --replicas=0 + fi + done < "$STATE_FILE" + + echo "✅ Scaled down successfully." +} + +scale_up() { + NAMESPACE="$1" + STATE_FILE="./${NAMESPACE}.state" + + if [ ! -f "$STATE_FILE" ]; then + echo "❌ State file $STATE_FILE not found. Abort." + exit 1 + fi + + while read kind_full replicas; do + kind=$(echo "$kind_full" | cut -d'/' -f1 | tr '[:upper:]' '[:lower:]') + name=$(echo "$kind_full" | cut -d'/' -f2) + + if [ -z "$replicas" ]; then + echo "⚠️ No replicas count found for $kind/$name. Skipping..." + continue + fi + + echo "🚀 Scaling up $kind/$name to $replicas replicas" + + if [ "$kind" == "sgcluster" ]; then + kubectl patch sgcluster "$name" -n "$NAMESPACE" --type merge -p "{\"spec\":{\"instances\":$replicas}}" + else + kubectl scale "$kind" "$name" -n "$NAMESPACE" --replicas="$replicas" + fi + + done < "$STATE_FILE" + + rm "$STATE_FILE" + echo "✅ Scaled up successfully and state removed." +} + +function veleroapp() +{ + case "$1" in + init) + trap cleanup EXIT + read -p "👣 MinIO url (with http/s) : " PATH_MINIO + read -p "👣 Bucket name:: " BUCKET + read -p "🔑 MinIO Access Key: " ACCESS_KEY + read -sp "🔒 MinIO Secret Key: " SECRET_KEY + cat < /tmp/credentials-vel +[default] +aws_access_key_id=${ACCESS_KEY} +aws_secret_access_key=${SECRET_KEY} +EOF + echo "" + if [[ ! -f /usr/local/bin/velero ]]; then + echo "🔍 Velero non trouvé, téléchargement et installation de v1.16.0…" + wget -q https://github.com/vmware-tanzu/velero/releases/download/v1.16.0/velero-v1.16.0-linux-amd64.tar.gz + tar -xzf velero-v1.16.0-linux-amd64.tar.gz + chmod +x velero-v1.16.0-linux-amd64/velero + sudo chown root:root velero-v1.16.0-linux-amd64/velero + sudo mv velero-v1.16.0-linux-amd64/velero /usr/local/bin/velero + rm -rf velero-v1.16.0-linux-amd64 velero-v1.16.0-linux-amd64.tar.gz + echo "✅ Velero installé." + else + echo "✅ Velero déjà présent, version : $(velero version --client-only | head -n1)" + fi + echo "▶️ Déploiement Velero dans le cluster…" + velero install \ + --provider aws \ + --plugins velero/velero-plugin-for-aws:v1.10.0,openebs/velero-plugin:3.6.0 \ + --features=EnableCSI \ + --use-node-agent \ + --bucket ${BUCKET} \ + --backup-location-config region=us-east-1,s3ForcePathStyle="true",s3Url=${PATH_MINIO},checksumAlgorithm="" \ + --snapshot-location-config region=us-east-1 \ + --secret-file /tmp/credentials-vel \ + --uploader-type kopia \ + --kubeconfig /home/user/.kube/config \ + || echo "ℹ️ Velero est peut-être déjà installé dans le cluster." + echo "🎉 Script terminé." + ;; + uninstall) + velero uninstall --kubeconfig /home/user/.kube/config + ;; + backup) + NOW=$(date +"%Y%m%d-%H%M") + velero backup create backup-complet-cluster2-${NOW} \ + --include-namespaces '*' \ + --include-resources '*' \ + --include-cluster-resources=true \ + --kubeconfig /home/user/.kube/config \ + --default-volumes-to-fs-backup + echo "${NOW}" > /var/log/lastfilename + ;; + info) + velero backup describe $2 --kubeconfig /home/user/.kube/config + ;; + restore) + BACKUP_NAME="$2" + read -p "Are you sure you want to restore from backup '$BACKUP_NAME'? [y/N] " confirm + case "$confirm" in + [Yy]* ) + echo "▶️ Restoring from backup '$BACKUP_NAME'…" + velero restore create --from-backup "$BACKUP_NAME" \ + --include-namespaces '*' \ + --include-resources '*' \ + --include-cluster-resources=true \ + --kubeconfig /home/user/.kube/config \ + --restore-volumes \ + ;; + * ) + echo "❌ Restore aborted." + ;; + esac + ;; + *) + echo "Usage: admin velero [init|backup|restore|info ] [dataname]" + echo "" + echo "Use the velero backend to make some kubernetes backup operations" + echo "" + echo "init - initialize the data storage on kubernetes" + echo "uninstall - uninstall velero on kubernetes" + echo "backup - backup the kubernetes objects" + echo "restore - restore [dataname] backup on kubernetes" + echo "info - get informations about [dataname]" + echo "" + exit 2 + ;; +esac +} + +# Ajouter le case "prune" +case "$1" in + install) + suid $@ + install + ;; + config) + suid $@ + config + ;; + backup) + suid $@ + backup "$2" + ;; + restore) + suid $@ + restore "$2" "$3" + ;; + check) + suid $@ + check "$2" + ;; + view) + suid $@ + view + ;; + prune) + suid $@ + prune + ;; + test) + pv_info "$2" + ;; + state) + save_state "$2" + ;; + down) + scale_down "$2" + ;; + up) + scale_up "$2" + ;; + velero) + veleroapp $2 $3 $4 + ;; + *) + echo "Usage: $0 {install|config|backup [dataset]|restore dataset [newdataset]|check [dataset]|view|prune}" + echo "Usage: $0 {test |state |down |up }" + exit 1 + ;; +esac