# CH-J Server Manager - fixed remote helper for chj.disks (POSIX sh: dash, bash, BusyBox ash). # Started by Core as "sh -c SCRIPT chj OPERATION ARGS..." through PrivilegedExecutor (root/sudo for anything beyond the # unprivileged inventory). Every argument is validated by Core and again here; there is no generic command execution. # Changes run only as a detached job (job-start -> job-run) from a plan of whitelisted steps, so an SSH outage does not # stop a running change. A job is never restarted automatically; its state and log stay on the server. set -u umask 077 LC_ALL=C; export LC_ALL PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; export PATH STATE_DIR=/var/lib/chj-server-manager/disks JOBS=$STATE_DIR/jobs BACKUPS=$STATE_DIR/backups LOCK=/run/chj-disks.lock op=${1:-} [ $# -gt 0 ] && shift [ "$op" = job-run ] || printf '\036CHJ_READY\n' fail() { printf '\036CHJ_ERR=%s\n' "$1"; [ $# -gt 1 ] && printf '%s\n' "$2" | head -c 2000 >&2; exit 3; } have() { command -v "$1" >/dev/null 2>&1; } is_root() { [ "$(id -u)" = 0 ]; } need_root() { is_root || fail PRIVILEGE_REQUIRED "Root privileges are required."; } sec() { printf '\036CHJ_SECTION=%s\n' "$1"; } clean() { tr -d '|\r\n' | cut -c1-200; } readf() { [ -r "$1" ] && head -c 400 "$1" 2>/dev/null | tr -d '|\r\n' | sed 's/[[:space:]]*$//'; } b64d() { if printf 'YQ==' | base64 -d >/dev/null 2>&1; then base64 -d elif printf 'YQ==' | openssl base64 -d -A >/dev/null 2>&1; then openssl base64 -d -A else return 127; fi } sha_stream() { if have sha256sum; then sha256sum | cut -c1-64; else openssl dgst -sha256 | sed 's/^.*= *//'; fi; } # ---------- validation (Core validates first; this is the second, independent check) ---------- valid_dev() { case $1 in /dev/*) ;; *) return 1 ;; esac case $1 in *[!A-Za-z0-9._/:+-]*|*..*|*//*) return 1 ;; esac [ ${#1} -le 96 ] && [ -b "$1" ] } valid_num() { case $1 in ''|*[!0-9]*) return 1 ;; esac; [ ${#1} -le 19 ]; } valid_label() { case $1 in -) return 0 ;; ''|*[!A-Za-z0-9._-]*) return 1 ;; esac; [ ${#1} -le 36 ]; } valid_id() { case $1 in ''|*[!a-z0-9-]*) return 1 ;; esac; [ ${#1} -ge 8 ] && [ ${#1} -le 64 ]; } valid_lv() { case $1 in */*/*|/*|*/) return 1 ;; */*) ;; *) return 1 ;; esac; case $1 in *[!A-Za-z0-9._+/-]*|*..*) return 1 ;; esac; [ ${#1} -le 128 ]; } valid_type() { case $1 in ''|*[!0-9A-Fa-f-]*) return 1 ;; esac; [ ${#1} -le 36 ]; } valid_attrs() { case $1 in -) return 0 ;; ''|*[!A-Za-z0-9,:-]*) return 1 ;; esac; [ ${#1} -le 120 ]; } valid_bootlist() { case $1 in -) return 0 ;; ''|*[!0-9,]*) return 1 ;; esac; [ ${#1} -le 40 ]; } valid_mapname() { case $1 in ''|*[!A-Za-z0-9._+-]*|*..*) return 1 ;; esac; [ ${#1} -le 96 ]; } valid_mountpoint() { case $1 in /mnt/*|/media/*) ;; *) return 1 ;; esac case $1 in *[!A-Za-z0-9._/-]*|*..*|*//*|*/) return 1 ;; esac [ ${#1} -le 120 ] } # Kernel name (sda1, dm-3, nvme0n1p2) of a block device path, following /dev/mapper links. kname() { r=$(readlink -f "$1" 2>/dev/null) || return 1; printf '%s\n' "${r##*/}"; } majmin() { cat "/sys/class/block/$(kname "$1")/dev" 2>/dev/null; } sectors_of() { cat "/sys/class/block/$(kname "$1")/size" 2>/dev/null; } # Partition node of DISK number NUM (works for sda1, nvme0n1p1, loop0p1, mmcblk0p1). part_node() { d=$(kname "$1"); n=$2 for p in /sys/block/"$d"/"$d"*; do [ -r "$p/partition" ] || continue [ "$(cat "$p/partition")" = "$n" ] && { printf '/dev/%s\n' "${p##*/}"; return 0; } done return 1 } is_mounted() { mm=$(majmin "$1") || return 1; [ -n "$mm" ] && awk -v mm="$mm" '$3==mm { found=1 } END { exit !found }' /proc/self/mountinfo; } is_swap_active() { awk -v d="$(readlink -f "$1")" 'NR>1 && $1==d { found=1 } END { exit !found }' /proc/swaps; } holders_of() { ls "/sys/class/block/$(kname "$1")/holders" 2>/dev/null; } # Not mounted, not active swap and no device stacked on top (dm-crypt, LVM, md). not_in_use() { is_mounted "$1" && return 1 is_swap_active "$1" && return 1 [ -z "$(holders_of "$1")" ] } # Any partition of DISK in use? disk_busy() { is_mounted "$1" && return 0 [ -n "$(holders_of "$1")" ] && return 0 d=$(kname "$1") for p in /sys/block/"$d"/"$d"*; do [ -r "$p/partition" ] || continue not_in_use "/dev/${p##*/}" || return 0 done return 1 } mountpoint_of() { mm=$(majmin "$1"); awk -v mm="$mm" '$3==mm { gsub(/\\040/, " ", $5); print $5; exit }' /proc/self/mountinfo; } table_hash() { sfdisk -d "$1" 2>/dev/null | sha_stream; } # ---------- inventory ---------- op_inspect() { sec meta # shellcheck disable=SC1091 [ -r /etc/os-release ] && . /etc/os-release 2>/dev/null printf 'uid=%s\nos=%s\nosId=%s\nosLike=%s\nkernel=%s\n' "$(id -u)" "${PRETTY_NAME:-}" "${ID:-}" "${ID_LIKE:-}" "$(uname -r)" | clean_lines [ -d /run/systemd/system ] && echo systemd=1 || echo systemd=0 { [ -d /etc/pve ] || have pveversion; } && echo pve=1 || echo pve=0 have getenforce && printf 'selinux=%s\n' "$(getenforce 2>/dev/null | clean)" [ -r /proc/self/mountinfo ] && echo mountinfo=1 printf 'time=%s\n' "$(date +%s)" sec tools for t in lsblk blkid findmnt sfdisk sgdisk parted partx partprobe blockdev wipefs udevadm growpart e2fsck resize2fs dumpe2fs tune2fs e2label mkfs.ext4 xfs_growfs xfs_repair xfs_admin xfs_info mkfs.xfs btrfs mkfs.btrfs fsck.vfat fatlabel mkfs.vfat fatresize ntfsresize ntfslabel mkfs.ntfs mkswap swaplabel pvs vgs lvs pvresize lvextend lvreduce cryptsetup mdadm smartctl nvme zpool systemd-run setsid nohup base64 sha256sum mount umount; do p=$(command -v "$t" 2>/dev/null) && printf '%s=%s\n' "$t" "$p" done sec versions have sfdisk && printf 'util-linux=%s\n' "$(sfdisk --version 2>&1 | head -n 1 | clean)" have e2fsck && printf 'e2fsprogs=%s\n' "$(e2fsck -V 2>&1 | head -n 1 | clean)" have mkfs.xfs && printf 'xfsprogs=%s\n' "$(mkfs.xfs -V 2>&1 | head -n 1 | clean)" have btrfs && printf 'btrfs-progs=%s\n' "$(btrfs --version 2>&1 | head -n 1 | clean)" have lvm && printf 'lvm2=%s\n' "$(lvm version 2>/dev/null | head -n 1 | clean)" have cryptsetup && printf 'cryptsetup=%s\n' "$(cryptsetup --version 2>&1 | head -n 1 | clean)" have parted && printf 'parted=%s\n' "$(parted --version 2>&1 | head -n 1 | clean)" have smartctl && printf 'smartmontools=%s\n' "$(smartctl --version 2>&1 | head -n 1 | clean)" have nvme && printf 'nvme-cli=%s\n' "$(nvme version 2>&1 | head -n 1 | clean)" have ntfsresize && printf 'ntfs-3g=%s\n' "$(ntfsresize --version 2>&1 | head -n 1 | clean)" sec sysblock for d in /sys/class/block/*; do [ -r "$d/size" ] || continue n=${d##*/} real=$(readlink -f "$d") part=$(cat "$d/partition" 2>/dev/null || echo -) start=$(cat "$d/start" 2>/dev/null || echo -) parent=-; base=$d if [ "$part" != - ]; then parent=${real%/*}; parent=${parent##*/}; base=/sys/block/$parent; fi q=$base/queue printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \ "$n" "$(readf "$d/dev")" "$(readf "$d/size")" "$(readf "$d/ro")" "$(readf "$base/removable")" "$part" "$start" "$parent" \ "$(readf "$q/rotational")" "$(readf "$q/logical_block_size")" "$(readf "$q/physical_block_size")" \ "$(readf "$base/device/model")" "$(readf "$base/device/vendor")" "$(readf "$base/device/serial")" "$(readf "$base/device/wwid")" \ "$(ls "$d/holders" 2>/dev/null | tr '\n' ',')" "$(ls "$d/slaves" 2>/dev/null | tr '\n' ',')" \ "$(readf "$d/dm/name")" "$(readf "$d/dm/uuid")" "$(readf "$d/loop/backing_file")" "$(readf "$d/md/level")" \ "$(printf '%s' "$real" | clean)" "$(readf "$q/discard_max_bytes")" done if have lsblk; then sec lsblk lsblk -J -b -l -O 2>/dev/null || lsblk -J -b -l -o NAME,KNAME,PATH,TYPE,SIZE,FSTYPE,FSVER,LABEL,UUID,PARTUUID,PARTLABEL,PARTTYPE,PARTFLAGS,PTUUID,PTTYPE,MOUNTPOINT,TRAN,ROTA,MODEL,SERIAL,VENDOR,HCTL,WWN 2>/dev/null fi sec blkid if have blkid; then if is_root; then blkid -c /dev/null -o export 2>/dev/null; else blkid -o export 2>/dev/null; fi fi sec mountinfo cat /proc/self/mountinfo 2>/dev/null sec df if have timeout; then timeout 15 df -kP 2>/dev/null; else df -kP 2>/dev/null; fi sec swaps cat /proc/swaps 2>/dev/null sec mdstat cat /proc/mdstat 2>/dev/null if have zpool; then sec zpool if have timeout; then timeout 15 zpool list -vHP 2>/dev/null; else zpool list -vHP 2>/dev/null; fi fi is_root || return 0 sec pt have sfdisk && for d in /sys/block/*; do n=${d##*/} case $n in sr*|ram*|zram*|fd*|dm-*|md*|nbd*) continue ;; esac [ "$(cat "$d/size" 2>/dev/null)" = 0 ] && continue case $n in loop*) [ -n "$(readf "$d/loop/backing_file")" ] && [ "$(readf "$d/ro")" = 0 ] || continue ;; esac json=$(sfdisk --json "/dev/$n" 2>/dev/null) || continue printf '@@disk %s %s\n' "$n" "$(table_hash "/dev/$n")" printf '%s\n' "$json" printf '@@end\n' done if have pvs; then sec lvm printf '@@pvs\n'; pvs --reportformat json --units b --nosuffix -o pv_name,vg_name,pv_size,pv_free,dev_size,pv_uuid,pv_attr 2>/dev/null printf '@@vgs\n'; vgs --reportformat json --units b --nosuffix -o vg_name,vg_uuid,vg_size,vg_free,vg_extent_size,vg_free_count,pv_count,lv_count,vg_attr 2>/dev/null printf '@@lvs\n'; lvs -a --reportformat json --units b --nosuffix -o lv_name,vg_name,lv_uuid,lv_size,lv_attr,lv_path,lv_dm_path,pool_lv,data_percent,metadata_percent,segtype,origin 2>/dev/null fi if have cryptsetup; then sec crypt for d in /sys/class/block/dm-*; do [ -r "$d/dm/uuid" ] || continue case $(cat "$d/dm/uuid") in CRYPT-*) ;; *) continue ;; esac name=$(cat "$d/dm/name") valid_mapname "$name" || continue printf '@@map %s\n' "$name" cryptsetup status "$name" 2>/dev/null | sed 's/^[[:space:]]*//' done fi sec fsinfo if have blkid; then blkid -c /dev/null -o export 2>/dev/null | awk 'BEGIN { RS="" } { dev=""; type=""; for (i = 1; i <= NF; i++) { if ($i ~ /^DEVNAME=/) dev = substr($i, 9); if ($i ~ /^TYPE=/) type = substr($i, 6) } if (dev != "") print dev, type }' | while read -r dev type; do valid_dev "$dev" || continue case $type in ext2|ext3|ext4) have dumpe2fs || continue printf '@@fs %s %s\n' "$dev" "$type" dumpe2fs -h "$dev" 2>/dev/null | grep -E '^(Block count|Block size|Free blocks|Reserved block count|Filesystem state|Filesystem features):' ;; xfs) have xfs_info || continue mp=$(mountpoint_of "$dev") printf '@@fs %s %s\n' "$dev" "$type" if [ -n "$mp" ]; then xfs_info "$mp" 2>/dev/null | grep -E '^(data|meta-data)'; else xfs_info "$dev" 2>/dev/null | grep -E '^(data|meta-data)'; fi ;; esac done fi if have btrfs; then sec btrfs btrfs filesystem show --raw 2>/dev/null fi } clean_lines() { tr -d '|\r' | cut -c1-200; } # ---------- SMART / NVMe health (read-only) ---------- op_smart() { dev=${1:-} valid_dev "$dev" || fail ARGUMENTS_INVALID "Invalid device." sec smartctl if have smartctl; then smartctl -j -a "$dev" 2>/dev/null || true; else echo '{"chj_unavailable":"smartctl"}'; fi case $(kname "$dev") in nvme*) sec nvme if have nvme; then printf '@@smart\n'; nvme smart-log -o json "$dev" 2>/dev/null printf '@@id\n'; nvme id-ctrl -o json "$dev" 2>/dev/null | head -c 200000 fi ;; esac } # ---------- jobs ---------- now() { date -u +%Y-%m-%dT%H:%M:%SZ; } put() { printf '%s\n' "$2" > "$JOBDIR/$1.tmp" && mv -f "$JOBDIR/$1.tmp" "$JOBDIR/$1"; } log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*"; } runner_alive() { dir=$1 [ -r "$dir/pid" ] || return 1 pid=$(cat "$dir/pid"); valid_num "$pid" || return 1 kill -0 "$pid" 2>/dev/null || return 1 tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | grep -q "job-run ${dir##*/}" } lock_owner_alive() { [ -d "$LOCK" ] || return 1 owner=$(cat "$LOCK/job" 2>/dev/null) valid_id "$owner" && runner_alive "$JOBS/$owner" } # Each plan line is "STEP arg..." with arguments from a restricted character set (no quoting, no globbing). plan_line_valid() { case $1 in *[!A-Za-z0-9._/:+,=@\ -]*) return 1 ;; esac [ ${#1} -le 400 ] } op_job_start() { need_root id=${1:-}; valid_id "$id" || fail ARGUMENTS_INVALID "Invalid job ID." b64d /dev/null 2>&1; [ $? -ne 127 ] || fail TOOL_MISSING "base64 is required." mkdir -p "$JOBS" "$BACKUPS" && chmod 700 "$STATE_DIR" "$JOBS" "$BACKUPS" || fail STATE_DIR_FAILED "The job directory cannot be created." find "$JOBS" -mindepth 1 -maxdepth 1 -type d -mtime +30 -name '[a-z0-9]*' 2>/dev/null | while IFS= read -r old; do valid_id "${old##*/}" && ! runner_alive "$old" && rm -rf -- "$old" done JOBDIR=$JOBS/$id # The same job ID never starts twice: a retried request after a lost connection is refused here. mkdir "$JOBDIR" 2>/dev/null || fail JOB_EXISTS "This job was already submitted; check its status instead of starting it again." lock_owner_alive && { rm -rf -- "$JOBDIR"; fail BUSY "Another disk job is running on this server."; } got_plan=0; got_helper=0 while IFS= read -r line; do case $line in CHJPLAN:*) printf '%s' "${line#CHJPLAN:}" | b64d > "$JOBDIR/plan" && got_plan=1 ;; CHJHELPER:*) printf '%s' "${line#CHJHELPER:}" | b64d > "$JOBDIR/helper.sh" && got_helper=1 ;; CHJTITLE:*) printf '%s' "${line#CHJTITLE:}" | b64d | head -c 300 | tr -d '\r\n|' > "$JOBDIR/title" ;; esac [ $got_plan = 1 ] && [ $got_helper = 1 ] && [ -f "$JOBDIR/title" ] && break done [ $got_plan = 1 ] && [ $got_helper = 1 ] || { rm -rf -- "$JOBDIR"; fail PROTOCOL_ERROR "The plan was not received."; } total=0 while IFS= read -r line; do plan_line_valid "$line" || { rm -rf -- "$JOBDIR"; fail PLAN_INVALID "The plan contains an invalid step."; } total=$((total + 1)) done < "$JOBDIR/plan" [ $total -ge 1 ] && [ $total -le 200 ] || { rm -rf -- "$JOBDIR"; fail PLAN_INVALID "The plan has no steps or too many steps."; } put total "$total"; put created "$(now)"; put state queued; : > "$JOBDIR/log" if [ -d /run/systemd/system ] && have systemd-run; then put runner systemd systemd-run --unit="chj-disks-$id" --description="CH-J Server Manager disk job $id" --collect --quiet /bin/sh "$JOBDIR/helper.sh" job-run "$id" >/dev/null 2>&1 || systemd-run --unit="chj-disks-$id" --description="CH-J Server Manager disk job $id" --quiet /bin/sh "$JOBDIR/helper.sh" job-run "$id" >/dev/null 2>&1 || { put state failed; put error "RUNNER_FAILED|systemd-run could not start the job."; fail RUNNER_FAILED "systemd-run could not start the job."; } else put runner setsid if have setsid; then setsid /bin/sh "$JOBDIR/helper.sh" job-run "$id" /dev/null 2>&1 & else nohup /bin/sh "$JOBDIR/helper.sh" job-run "$id" /dev/null 2>&1 & fi fi i=0; tick=0.1; sleep 0.1 2>/dev/null || tick=1 while [ $i -lt 50 ] && [ "$(cat "$JOBDIR/state")" = queued ]; do sleep $tick; i=$((i + 1)); [ $tick = 1 ] && i=$((i + 9)); done sec job printf 'id=%s\nstate=%s\nrunner=%s\n' "$id" "$(cat "$JOBDIR/state")" "$(cat "$JOBDIR/runner")" } op_job_status() { need_root id=${1:-}; valid_id "$id" || fail ARGUMENTS_INVALID "Invalid job ID." offset=${2:-0}; valid_num "$offset" || offset=0 JOBDIR=$JOBS/$id [ -d "$JOBDIR" ] || fail JOB_NOT_FOUND "The job is unknown on this server." # Liveness first, state second: a runner that finishes in between then shows its final state, never "unknown". alive=0; runner_alive "$JOBDIR" && alive=1 sec job printf 'id=%s\nalive=%s\n' "$id" "$alive" for f in state step total created started finished runner error title; do [ -r "$JOBDIR/$f" ] && printf '%s=%s\n' "$f" "$(head -c 400 "$JOBDIR/$f" | tr -d '\r\n')" done size=$(wc -c < "$JOBDIR/log" | tr -d ' ') printf 'logSize=%s\n' "$size" sec plan cat "$JOBDIR/plan" sec log [ "$offset" -lt "$size" ] && tail -c +"$((offset + 1))" "$JOBDIR/log" | head -c 65536 return 0 } op_job_list() { need_root sec jobs [ -d "$JOBS" ] || return 0 ls -1t "$JOBS" 2>/dev/null | head -n 30 | while IFS= read -r id; do valid_id "$id" || continue dir=$JOBS/$id alive=0; runner_alive "$dir" && alive=1 printf '%s|%s|%s|%s|%s|%s|%s|%s\n' "$id" "$(cat "$dir/state" 2>/dev/null)" "$(cat "$dir/step" 2>/dev/null)" "$(cat "$dir/total" 2>/dev/null)" \ "$(cat "$dir/created" 2>/dev/null)" "$(cat "$dir/finished" 2>/dev/null)" "$alive" "$(head -c 300 "$dir/title" 2>/dev/null | tr -d '|\r\n')" done } # ---------- steps (each returns 0 = ok, 10 = stop: reboot/rescue needed, anything else = failed) ---------- ERR= step_fail() { ERR="$1|$2"; log "ERROR $1: $2"; return 1; } need_tool() { have "$1" || { step_fail TOOL_MISSING "$1 is not installed on this server."; return 1; }; } need_dev() { valid_dev "$1" || { step_fail DEVICE_INVALID "$1 is not a valid block device."; return 1; }; } require_free() { not_in_use "$1" || { step_fail DEVICE_IN_USE "$1 is mounted, active swap or used by another device; it is never released automatically."; return 1; }; } settle() { have udevadm && udevadm settle -t 15 >/dev/null 2>&1; return 0; } part_geometry() { pn=$(part_node "$1" "$2") || return 1; k=$(kname "$pn"); printf '%s %s\n' "$(cat "/sys/class/block/$k/start")" "$(cat "/sys/class/block/$k/size")"; } # Kernel-side geometry of partition NUM, in 512-byte units as sysfs reports, converted from table sectors. expect_kernel() { disk=$1; num=$2; want_start=$3; want_size=$4 lbs=$(cat "/sys/block/$(kname "$disk")/queue/logical_block_size"); f=$((lbs / 512)) geo=$(part_geometry "$disk" "$num") || return 1 [ "$geo" = "$((want_start * f)) $((want_size * f))" ] } # Tell the kernel about partition NUM (add|update|delete). A disk with nothing in use is re-read completely. kernel_sync() { KS_DISK=$1; KS_ACTION=$2; KS_NUM=$3 settle if ! disk_busy "$KS_DISK" && have blockdev; then if blockdev --rereadpt "$KS_DISK" 2>&1; then settle; log "kernel: partition table of $KS_DISK re-read"; return 0; fi log "kernel: re-reading $KS_DISK failed; updating partition $KS_NUM only" fi if have partx; then case $KS_ACTION in add) partx -a -n "$KS_NUM" "$KS_DISK" ;; update) partx -u -n "$KS_NUM" "$KS_DISK" ;; delete) partx -d -n "$KS_NUM" "$KS_DISK" ;; esac 2>&1 log "kernel: partx $KS_ACTION partition $KS_NUM of $KS_DISK (exit code $?)" elif have partprobe; then partprobe "$KS_DISK" 2>&1; log "kernel: partprobe $KS_DISK (exit code $?)" fi settle return 0 } pending_reboot() { ERR="REBOOT_REQUIRED|$1" log "STOP: $1" log "The partition table on disk is already changed. The running kernel still uses the old layout; reboot the server (or finish in a rescue system) before continuing. The remaining steps were not executed." return 10 } s_verify_disk() { # DISK SECTORS TABLEHASH need_dev "$1" || return 1 [ "$(sectors_of "$1")" = "$2" ] || { step_fail LAYOUT_CHANGED "The size of $1 changed since the plan was made."; return 1; } need_tool sfdisk || return 1 h=$(table_hash "$1"); [ -n "$(sfdisk -d "$1" 2>/dev/null)" ] || h=none [ "$h" = "$3" ] || { step_fail LAYOUT_CHANGED "The partition table of $1 changed since the plan was made; nothing was modified."; return 1; } log "verified: $1 has the planned size and partition table" } s_verify_size() { # DEV BYTES (current kernel size) need_dev "$1" || return 1 [ "$(( $(sectors_of "$1") * 512 ))" = "$2" ] || { step_fail LAYOUT_CHANGED "The size of $1 changed since the plan was made."; return 1; } log "verified: $1 is $2 bytes" } s_backup_table() { # DISK need_dev "$1" || return 1; need_tool sfdisk || return 1 n=$(kname "$1"); stamp=$(date -u +%Y%m%dT%H%M%SZ) mkdir -p "$BACKUPS" && chmod 700 "$BACKUPS" if sfdisk -d "$1" > "$JOBDIR/table-$n.sfdisk" 2>/dev/null && [ -s "$JOBDIR/table-$n.sfdisk" ]; then cp "$JOBDIR/table-$n.sfdisk" "$BACKUPS/$n-$stamp-$JOBID.sfdisk" log "partition table metadata of $1 saved to $BACKUPS/$n-$stamp-$JOBID.sfdisk (restore with: sfdisk $1 < FILE). This is NOT a backup of the data." else log "$1 has no partition table; nothing to save" fi # Keep the 50 newest metadata backups. ls -1t "$BACKUPS" 2>/dev/null | tail -n +51 | while IFS= read -r old; do rm -f -- "$BACKUPS/$old"; done } s_require_free() { need_dev "$1" || return 1; require_free "$1" || return 1; log "checked: $1 is not in use"; } s_gpt_relocate() { # DISK need_dev "$1" || return 1; need_tool sfdisk || return 1 sfdisk --no-reread --no-tell-kernel --relocate gpt-bak-std "$1" || { have sgdisk && sgdisk -e "$1"; } || { step_fail TOOL_FAILED "The backup GPT header could not be moved to the end of $1."; return 1; } log "backup GPT header moved to the end of $1" } s_create_table() { # DISK gpt|dos need_dev "$1" || return 1; need_tool sfdisk || return 1 case $2 in gpt|dos) ;; *) step_fail ARGUMENTS_INVALID "Unsupported partition table type."; return 1 ;; esac disk_busy "$1" && { step_fail DEVICE_IN_USE "A partition of $1 is in use."; return 1; } printf 'label: %s\n' "$2" | sfdisk --wipe always --no-reread --no-tell-kernel "$1" || { step_fail TOOL_FAILED "sfdisk could not write the new table."; return 1; } kernel_sync "$1" add 1 d=$(kname "$1") for p in /sys/block/"$d"/"$d"*; do [ -r "$p/partition" ] && { pending_reboot "The kernel still lists old partitions of $1."; return 10; }; done log "new empty $2 partition table written to $1" } s_create_part() { # DISK NUM START SIZE TYPE NAME need_dev "$1" || return 1; need_tool sfdisk || return 1 part_node "$1" "$2" >/dev/null && { step_fail LAYOUT_CHANGED "Partition $2 already exists on $1."; return 1; } line="start=$3, size=$4, type=$5"; [ "$6" != - ] && line="$line, name=\"$6\"" # Fields are digits/GUID/hex and a validated name; sfdisk reads them from stdin, never from a shell expansion. printf '%s\n' "$line" | sfdisk --no-reread --no-tell-kernel --wipe-partitions always -N "$2" "$1" || { step_fail TOOL_FAILED "sfdisk could not create partition $2."; return 1; } kernel_sync "$1" add "$2" expect_kernel "$1" "$2" "$3" "$4" || { pending_reboot "The kernel did not register the new partition $2 of $1."; return 10; } log "created partition $2 on $1 (start $3, $4 sectors)" } s_delete_part() { # DISK NUM START SIZE need_dev "$1" || return 1; need_tool sfdisk || return 1 pn=$(part_node "$1" "$2") || { step_fail LAYOUT_CHANGED "Partition $2 of $1 does not exist."; return 1; } expect_kernel "$1" "$2" "$3" "$4" || { step_fail LAYOUT_CHANGED "Partition $2 of $1 is not where the plan expects it."; return 1; } require_free "$pn" || return 1 sfdisk --no-reread --no-tell-kernel --delete "$1" "$2" || { step_fail TOOL_FAILED "sfdisk could not delete partition $2."; return 1; } kernel_sync "$1" delete "$2" part_node "$1" "$2" >/dev/null && { pending_reboot "The kernel still lists the deleted partition $2 of $1."; return 10; } log "deleted partition $2 of $1" } s_resize_part() { # DISK NUM START OLDSIZE NEWSIZE need_dev "$1" || return 1; need_tool sfdisk || return 1 part_node "$1" "$2" >/dev/null || { step_fail LAYOUT_CHANGED "Partition $2 of $1 does not exist."; return 1; } expect_kernel "$1" "$2" "$3" "$4" || { step_fail LAYOUT_CHANGED "Partition $2 of $1 is not where the plan expects it."; return 1; } printf 'start=%s, size=%s\n' "$3" "$5" | sfdisk --no-reread --no-tell-kernel -N "$2" "$1" || { step_fail TOOL_FAILED "sfdisk could not resize partition $2."; return 1; } kernel_sync "$1" update "$2" expect_kernel "$1" "$2" "$3" "$5" || { pending_reboot "The kernel did not take over the new size of partition $2 of $1."; return 10; } log "partition $2 of $1 now has $5 sectors (was $4)" } s_set_type() { need_dev "$1" || return 1; need_tool sfdisk || return 1; sfdisk --no-reread --no-tell-kernel --part-type "$1" "$2" "$3" || { step_fail TOOL_FAILED "The partition type could not be changed."; return 1; }; settle; log "partition $2 of $1: type $3"; } s_set_name() { need_dev "$1" || return 1; need_tool sfdisk || return 1; n=$3; [ "$n" = - ] && n=""; sfdisk --no-reread --no-tell-kernel --part-label "$1" "$2" "$n" || { step_fail TOOL_FAILED "The partition name could not be changed."; return 1; }; settle; log "partition $2 of $1: name '$n'"; } s_set_attrs() { need_dev "$1" || return 1; need_tool sfdisk || return 1; a=$3; [ "$a" = - ] && a=""; sfdisk --no-reread --no-tell-kernel --part-attrs "$1" "$2" "$a" || { step_fail TOOL_FAILED "The GPT attributes could not be changed."; return 1; }; settle; log "partition $2 of $1: attributes '$a'"; } s_set_boot() { # DISK LIST(1,3 or -) need_dev "$1" || return 1; need_tool sfdisk || return 1 if [ "$2" = - ]; then sfdisk --no-reread --no-tell-kernel --activate "$1" - ; else sfdisk --no-reread --no-tell-kernel --activate "$1" $(printf '%s' "$2" | tr ',' ' '); fi || { step_fail TOOL_FAILED "The boot flag could not be changed."; return 1; } log "boot flag of $1: $2" } s_mkfs() { # DEV FSTYPE LABEL need_dev "$1" || return 1; require_free "$1" || return 1 l=$3 case $2 in ext4) need_tool mkfs.ext4 || return 1; if [ "$l" = - ]; then mkfs.ext4 -F -q "$1"; else mkfs.ext4 -F -q -L "$l" "$1"; fi ;; xfs) need_tool mkfs.xfs || return 1; if [ "$l" = - ]; then mkfs.xfs -f -q "$1"; else mkfs.xfs -f -q -L "$l" "$1"; fi ;; btrfs) need_tool mkfs.btrfs || return 1; if [ "$l" = - ]; then mkfs.btrfs -f -q "$1"; else mkfs.btrfs -f -q -L "$l" "$1"; fi ;; vfat) need_tool mkfs.vfat || return 1; if [ "$l" = - ]; then mkfs.vfat -F 32 "$1"; else mkfs.vfat -F 32 -n "$l" "$1"; fi ;; ntfs) need_tool mkfs.ntfs || return 1; if [ "$l" = - ]; then mkfs.ntfs -Q -F "$1"; else mkfs.ntfs -Q -F -L "$l" "$1"; fi ;; swap) need_tool mkswap || return 1; if [ "$l" = - ]; then mkswap "$1"; else mkswap -L "$l" "$1"; fi ;; wipe) need_tool wipefs || return 1; wipefs -a "$1" ;; *) step_fail ARGUMENTS_INVALID "Unsupported file system."; return 1 ;; esac >/dev/null || { step_fail TOOL_FAILED "Creating $2 on $1 failed."; return 1; } settle log "created $2 on $1" } s_set_label() { # DEV FSTYPE LABEL need_dev "$1" || return 1; l=$3; [ "$l" = - ] && l="" case $2 in ext2|ext3|ext4) need_tool e2label || return 1; e2label "$1" "$l" ;; xfs) need_tool xfs_admin || return 1; require_free "$1" || return 1; if [ -n "$l" ]; then xfs_admin -L "$l" "$1"; else xfs_admin -L -- "$1"; fi ;; btrfs) need_tool btrfs || return 1; mp=$(mountpoint_of "$1"); btrfs filesystem label "${mp:-$1}" "$l" ;; vfat) need_tool fatlabel || return 1; require_free "$1" || return 1; [ -n "$l" ] || { step_fail ARGUMENTS_INVALID "FAT labels cannot be removed here."; return 1; }; fatlabel "$1" "$l" ;; ntfs) need_tool ntfslabel || return 1; require_free "$1" || return 1; ntfslabel "$1" "$l" ;; swap) need_tool swaplabel || return 1; swaplabel -L "$l" "$1" ;; *) step_fail ARGUMENTS_INVALID "Labels of this file system are not supported."; return 1 ;; esac >/dev/null || { step_fail TOOL_FAILED "The label could not be set."; return 1; } settle; log "label of $1: '$l'" } s_fsck() { # DEV FSTYPE check|repair need_dev "$1" || return 1; require_free "$1" || return 1 case $2:$3 in ext2:check|ext3:check|ext4:check) need_tool e2fsck || return 1; e2fsck -f -n "$1"; rc=$? ;; ext2:repair|ext3:repair|ext4:repair) need_tool e2fsck || return 1; e2fsck -f -p "$1"; rc=$?; [ $rc -le 1 ] && rc=0 ;; xfs:check) need_tool xfs_repair || return 1; xfs_repair -n "$1"; rc=$? ;; xfs:repair) need_tool xfs_repair || return 1; xfs_repair "$1"; rc=$? ;; btrfs:check) need_tool btrfs || return 1; btrfs check --readonly "$1"; rc=$? ;; vfat:check) need_tool fsck.vfat || return 1; fsck.vfat -n "$1"; rc=$? ;; vfat:repair) need_tool fsck.vfat || return 1; fsck.vfat -a "$1"; rc=$?; [ $rc -le 1 ] && rc=0 ;; *) step_fail ARGUMENTS_INVALID "This check is not supported for $2."; return 1 ;; esac [ "$rc" = 0 ] || { step_fail FSCK_ERRORS "The check of $1 reported problems (exit code $rc); see the log."; return 1; } log "file system check of $1 ($3): clean" } ext_size() { dumpe2fs -h "$1" 2>/dev/null | awk -F: '/^Block count/ { c=$2+0 } /^Block size/ { s=$2+0 } END { printf "%.0f\n", c*s }'; } s_grow_fs() { # DEV FSTYPE need_dev "$1" || return 1 devbytes=$(( $(sectors_of "$1") * 512 )) case $2 in ext2|ext3|ext4) need_tool resize2fs || return 1 resize2fs "$1" || { step_fail TOOL_FAILED "resize2fs could not grow $1."; return 1; } log "ext file system on $1 is now $(ext_size "$1") bytes (device $devbytes)" ;; xfs) need_tool xfs_growfs || return 1 mp=$(mountpoint_of "$1"); [ -n "$mp" ] || { step_fail FS_NOT_MOUNTED "XFS can only be grown while mounted; mount $1 first."; return 1; } xfs_growfs "$mp" || { step_fail TOOL_FAILED "xfs_growfs failed."; return 1; } log "XFS on $mp grown to the size of $1" ;; btrfs) need_tool btrfs || return 1 mp=$(mountpoint_of "$1"); [ -n "$mp" ] || { step_fail FS_NOT_MOUNTED "Btrfs can only be resized while mounted; mount $1 first."; return 1; } devid=$(btrfs filesystem show "$mp" 2>/dev/null | awk -v d="$(readlink -f "$1")" '$1=="devid" && $NF==d { print $2 }') valid_num "$devid" || { step_fail DEVICE_INVALID "$1 is not a member of the Btrfs file system at $mp."; return 1; } btrfs filesystem resize "$devid:max" "$mp" || { step_fail TOOL_FAILED "btrfs resize failed."; return 1; } log "Btrfs device $devid on $mp grown to max" ;; ntfs) need_tool ntfsresize || return 1; require_free "$1" || return 1 ntfsresize --no-action --force "$1" >/dev/null || { step_fail TOOL_FAILED "ntfsresize refused the dry run."; return 1; } printf 'y\n' | ntfsresize --force "$1" || { step_fail TOOL_FAILED "ntfsresize failed."; return 1; } log "NTFS on $1 grown to the partition size" ;; swap) need_tool mkswap || return 1; require_free "$1" || return 1 uuid=$(blkid -s UUID -o value "$1" 2>/dev/null); lab=$(blkid -s LABEL -o value "$1" 2>/dev/null) case $uuid in *[!A-Fa-f0-9-]*) uuid= ;; esac set -- "$1"; [ -n "$uuid" ] && set -- -U "$uuid" "$@"; [ -n "$lab" ] && valid_label "$lab" && set -- -L "$lab" "$@" mkswap "$@" >/dev/null || { step_fail TOOL_FAILED "mkswap failed."; return 1; } log "swap signature rewritten for the new size (UUID and label kept)" ;; *) step_fail ARGUMENTS_INVALID "Growing $2 is not supported."; return 1 ;; esac settle } s_shrink_fs() { # DEV FSTYPE NEWBYTES need_dev "$1" || return 1; valid_num "$3" || { step_fail ARGUMENTS_INVALID "Invalid size."; return 1; } case $2 in ext2|ext3|ext4) need_tool resize2fs || return 1; need_tool e2fsck || return 1 require_free "$1" || return 1 log "consistency check before shrinking (e2fsck -f -p)" e2fsck -f -p "$1"; rc=$? [ $rc -le 1 ] || { step_fail FSCK_ERRORS "e2fsck reported problems (exit code $rc); the file system was not shrunk."; return 1; } bs=$(dumpe2fs -h "$1" 2>/dev/null | awk -F: '/^Block size/ { print $2+0 }') min=$(resize2fs -P "$1" 2>/dev/null | awk -F: '/[Mm]inimum/ { print $2+0 }') valid_num "$bs" && valid_num "$min" || { step_fail TOOL_FAILED "The minimum size of $1 could not be determined."; return 1; } minbytes=$((min * bs)); margin=$((minbytes / 10)); [ $margin -ge 134217728 ] || margin=134217728 [ "$3" -ge $((minbytes + margin)) ] || { step_fail SIZE_TOO_SMALL "The requested size is below the minimum $minbytes bytes plus a safety reserve of $margin bytes."; return 1; } blocks=$(($3 / bs)) log "shrinking ext file system on $1 to $blocks blocks of $bs bytes (minimum $min blocks)" resize2fs "$1" "$blocks" || { step_fail TOOL_FAILED "resize2fs could not shrink $1."; return 1; } [ "$(ext_size "$1")" = "$((blocks * bs))" ] || { step_fail VERIFY_FAILED "The file system size after shrinking is not the expected value."; return 1; } log "ext file system on $1 is now $((blocks * bs)) bytes" ;; btrfs) need_tool btrfs || return 1 mp=$(mountpoint_of "$1"); [ -n "$mp" ] || { step_fail FS_NOT_MOUNTED "Btrfs can only be resized while mounted."; return 1; } devs=$(btrfs filesystem show "$mp" 2>/dev/null | awk '$1=="devid"' | wc -l) [ "$devs" = 1 ] || { step_fail UNSUPPORTED "Shrinking multi-device Btrfs is not supported."; return 1; } devid=$(btrfs filesystem show --raw "$mp" 2>/dev/null | awk '$1=="devid" { print $2 }') used=$(btrfs filesystem show --raw "$mp" 2>/dev/null | awk '$1=="devid" { print $6 }') valid_num "$devid" || { step_fail TOOL_FAILED "The Btrfs device ID could not be read."; return 1; } valid_num "$used" || { step_fail TOOL_FAILED "The Btrfs usage could not be read."; return 1; } margin=$((used / 10)); [ $margin -ge 268435456 ] || margin=268435456 [ "$3" -ge $((used + margin)) ] || { step_fail SIZE_TOO_SMALL "The requested size is below the allocated $used bytes plus a reserve of $margin bytes."; return 1; } btrfs filesystem resize "$devid:$3" "$mp" || { step_fail TOOL_FAILED "btrfs resize failed."; return 1; } log "Btrfs on $mp shrunk to $3 bytes" ;; ntfs) need_tool ntfsresize || return 1; require_free "$1" || return 1 ntfsresize --no-action --force -s "$3" "$1" >/dev/null || { step_fail SIZE_TOO_SMALL "ntfsresize refused the dry run for $3 bytes."; return 1; } printf 'y\n' | ntfsresize --force -s "$3" "$1" || { step_fail TOOL_FAILED "ntfsresize failed."; return 1; } log "NTFS on $1 shrunk to $3 bytes" ;; *) step_fail UNSUPPORTED "Shrinking $2 is not supported."; return 1 ;; esac settle } s_luks_resize() { # MAPNAME need_tool cryptsetup || return 1; valid_mapname "$1" || { step_fail ARGUMENTS_INVALID "Invalid mapping name."; return 1; } [ -b "/dev/mapper/$1" ] || { step_fail DEVICE_INVALID "The LUKS mapping $1 is not open."; return 1; } # Never prompts: without a reusable volume key cryptsetup fails here and nothing else changes. cryptsetup resize --batch-mode "$1" /dev/null)" ] || { step_fail ARGUMENTS_INVALID "$2 is not empty."; return 1; } else mkdir -p "$2" && chmod 755 "$2"; fi mount "$1" "$2" || { step_fail TOOL_FAILED "mount failed."; return 1; } log "$1 mounted at $2 (temporary, /etc/fstab unchanged)" } system_mount() { case $1 in /|/boot|/boot/*|/efi|/usr|/usr/*|/var|/var/*|/home|/opt|/srv|/root|/tmp|/nix|/snap|/snap/*) return 0 ;; esac return 1 } s_unmount() { # DEV need_dev "$1" || return 1 mm=$(majmin "$1") mps=$(awk -v mm="$mm" '$3==mm { gsub(/\\040/, " ", $5); print $5 }' /proc/self/mountinfo) [ -n "$mps" ] || { log "$1 is not mounted"; return 0; } oldifs=$IFS; IFS=' ' for mp in $mps; do system_mount "$mp" && { IFS=$oldifs; step_fail SYSTEM_MOUNT "$mp is needed by the running system; CH-J never unmounts it."; return 1; }; done for mp in $mps; do umount "$mp" || { IFS=$oldifs; step_fail DEVICE_BUSY "$mp is busy and was not unmounted (no forced or lazy unmount)."; return 1; }; done IFS=$oldifs log "$1 unmounted" } s_verify_part() { expect_kernel "$1" "$2" "$3" "$4" || { step_fail VERIFY_FAILED "Partition $2 of $1 does not have the expected geometry."; return 1; }; log "verified: partition $2 of $1"; } s_verify_fs() { # DEV FSTYPE need_dev "$1" || return 1 case $2 in ext2|ext3|ext4) if is_mounted "$1"; then log "mounted ext file system on $1: $(ext_size "$1") bytes (online check skipped)" else e2fsck -f -n "$1" >/dev/null 2>&1 || { step_fail VERIFY_FAILED "e2fsck -n reported problems on $1."; return 1; }; log "verified: e2fsck -n clean, $(ext_size "$1") bytes"; fi ;; xfs) mp=$(mountpoint_of "$1"); if [ -n "$mp" ]; then xfs_info "$mp" 2>/dev/null | grep -E '^data' ; else xfs_repair -n "$1" >/dev/null 2>&1 || { step_fail VERIFY_FAILED "xfs_repair -n reported problems on $1."; return 1; }; fi; log "verified XFS on $1" ;; *) log "verified: $1 present" ;; esac } s_disabled() { step_fail OPERATION_DISABLED "This operation is not enabled in this version."; } run_step() { name=$1; shift case $name in verify-disk) [ $# = 3 ] && s_verify_disk "$@" ;; verify-size) [ $# = 2 ] && s_verify_size "$@" ;; backup-table) [ $# = 1 ] && s_backup_table "$@" ;; require-free) [ $# = 1 ] && s_require_free "$@" ;; gpt-relocate) [ $# = 1 ] && s_gpt_relocate "$@" ;; create-table) [ $# = 2 ] && s_create_table "$@" ;; create-part) [ $# = 6 ] && valid_num "$2" && valid_num "$3" && valid_num "$4" && valid_type "$5" && valid_label "$6" && s_create_part "$@" ;; delete-part) [ $# = 4 ] && valid_num "$2" && valid_num "$3" && valid_num "$4" && s_delete_part "$@" ;; resize-part) [ $# = 5 ] && valid_num "$2" && valid_num "$3" && valid_num "$4" && valid_num "$5" && s_resize_part "$@" ;; set-type) [ $# = 3 ] && valid_num "$2" && valid_type "$3" && s_set_type "$@" ;; set-name) [ $# = 3 ] && valid_num "$2" && valid_label "$3" && s_set_name "$@" ;; set-attrs) [ $# = 3 ] && valid_num "$2" && valid_attrs "$3" && s_set_attrs "$@" ;; set-boot) [ $# = 2 ] && valid_bootlist "$2" && s_set_boot "$@" ;; mkfs) [ $# = 3 ] && valid_label "$3" && s_mkfs "$@" ;; set-label) [ $# = 3 ] && valid_label "$3" && s_set_label "$@" ;; fsck) [ $# = 3 ] && s_fsck "$@" ;; grow-fs) [ $# = 2 ] && s_grow_fs "$@" ;; shrink-fs) [ $# = 3 ] && s_shrink_fs "$@" ;; luks-resize) [ $# = 1 ] && s_luks_resize "$@" ;; pv-resize) [ $# = 1 ] && s_pv_resize "$@" ;; lv-extend) [ $# = 2 ] && s_lv_extend "$@" ;; lv-reduce) [ $# = 2 ] && s_lv_reduce "$@" ;; mount) [ $# = 2 ] && s_mount "$@" ;; unmount) [ $# = 1 ] && s_unmount "$@" ;; verify-part) [ $# = 4 ] && s_verify_part "$@" ;; verify-fs) [ $# = 2 ] && s_verify_fs "$@" ;; move-part|copy-part) s_disabled ;; *) step_fail PLAN_INVALID "Unknown step $name."; return 1 ;; esac rc=$? [ $rc = 0 ] || [ -n "$ERR" ] || ERR="PLAN_INVALID|Invalid arguments for step $name." return $rc } op_job_run() { JOBID=${1:-}; valid_id "$JOBID" || exit 2 JOBDIR=$JOBS/$JOBID [ -d "$JOBDIR" ] && [ "$(cat "$JOBDIR/state" 2>/dev/null)" = queued ] || exit 2 trap '' HUP INT PIPE cd / || exit 2 exec >>"$JOBDIR/log" 2>&1 "$JOBDIR/pid" # One modifying disk job per server: conflicting changes on shared layers (disk, LUKS, LVM) are impossible. if ! mkdir "$LOCK" 2>/dev/null; then if lock_owner_alive; then put state failed; put error "BUSY|Another disk job is running on this server."; put finished "$(now)"; exit 1; fi rm -rf -- "$LOCK"; mkdir "$LOCK" || { put state failed; put error "BUSY|The job lock is unavailable."; exit 1; } fi printf '%s\n' "$JOBID" > "$LOCK/job" trap 'rm -rf -- "$LOCK"' EXIT put started "$(now)"; put state running log "job $JOBID started ($(cat "$JOBDIR/total") steps, runner $(cat "$JOBDIR/runner" 2>/dev/null))" # Step functions share the global namespace (POSIX sh has no locals), so the runner uses JOB_* names only. JOB_N=0; JOB_TOTAL=$(cat "$JOBDIR/total"); JOB_FINAL=succeeded while IFS= read -r JOB_LINE <&3; do JOB_N=$((JOB_N + 1)); put step "$JOB_N" plan_line_valid "$JOB_LINE" || { ERR="PLAN_INVALID|Invalid step."; JOB_FINAL=failed; break; } printf '\036STEP %s|%s|running|%s\n' "$JOB_N" "$JOB_TOTAL" "$JOB_LINE" log "step $JOB_N/$JOB_TOTAL: $JOB_LINE" ERR= # Plan lines contain no glob or quote characters (plan_line_valid), so plain word splitting is safe. # shellcheck disable=SC2086 run_step $JOB_LINE JOB_RC=$? if [ $JOB_RC = 0 ]; then printf '\036STEP %s|%s|done|\n' "$JOB_N" "$JOB_TOTAL"; continue; fi if [ $JOB_RC = 10 ]; then JOB_FINAL=pending-reboot; printf '\036STEP %s|%s|stopped|\n' "$JOB_N" "$JOB_TOTAL"; else JOB_FINAL=failed; printf '\036STEP %s|%s|failed|\n' "$JOB_N" "$JOB_TOTAL"; fi break done 3< "$JOBDIR/plan" [ -n "$ERR" ] && put error "$ERR" put finished "$(now)"; put state "$JOB_FINAL" log "job $JOBID finished: $JOB_FINAL" } case $op in inspect) op_inspect ;; smart) op_smart "$@" ;; job-start) op_job_start "$@" ;; job-status) op_job_status "$@" ;; job-list) op_job_list ;; job-run) op_job_run "$@" ;; *) fail ARGUMENTS_INVALID "Unknown operation." ;; esac