homelab-docker-stacks/manage.sh

#!/usr/bin/env bash

set -uo pipefail
shopt -s nullglob

SCRIPT_DIR="$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")"
STACKS_DIR="${STACKS_DIR:-$SCRIPT_DIR}"
CONFIG_FILE="${STACKS_CONFIG:-$STACKS_DIR/manage.conf}"
LOCK_FILE="${STACKS_LOCK_FILE:-/run/lock/stacks-manage.lock}"

DEFAULT_MAX_PARALLEL=2
WAIT_TIMEOUT=120

BEFORE_STAGES=()
BEFORE_LIMITS=()
AFTER_STAGES=()
AFTER_LIMITS=()
CONFIGURED_STACKS=()
EXCLUDED_STACKS=()

DISCOVERED_STACKS=()
MANAGED_STACKS=()
STAGE_NAMES=()
STAGE_LIMITS=()
STAGE_STACKS=()

RUN_SUCCESS=0
RUN_FAILED=0
RUN_FAILED_STACKS=()
RUN_SKIPPED_STAGES=()

die() {
	echo "ERROR: $*" >&2
	exit 1
}

if ((BASH_VERSINFO[0] < 5 || (\
	BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] < 1))); then
	die "Bash 5.1 or newer is required"
fi

handle_signal() {
	local status="$1"
	local pid attempt alive
	local -a pids=()

	trap '' INT TERM
	mapfile -t pids < <(jobs -p)
	for pid in "${pids[@]}"; do
		kill -TERM -- "-$pid" 2>/dev/null || :
	done

	# Allow five seconds for cleanup, then stop every remaining descendant.
	for ((attempt = 0; attempt < 50; attempt++)); do
		alive=0
		for pid in "${pids[@]}"; do
			kill -0 -- "-$pid" 2>/dev/null && alive=1
		done
		((alive)) || break
		sleep 0.1
	done
	for pid in "${pids[@]}"; do
		kill -KILL -- "-$pid" 2>/dev/null || :
	done
	((${#pids[@]} == 0)) || wait "${pids[@]}" 2>/dev/null
	printf '\n'
	exit "$status"
}

usage() {
	cat <<EOF
Usage: $(basename "$0") COMMAND [STACK]

Lifecycle:
  up [STACK]       Create and start containers
  start [STACK]    Start existing containers
  stop [STACK]     Stop containers
  restart [STACK]  Restart containers
  down [STACK]     Stop and remove containers and networks
  pull [STACK]     Pull images
  update [STACK]   Pull images and apply Compose changes

Inspect:
  ps [STACK]       Show container status
  logs STACK       Follow logs
  config STACK     Render Compose configuration
  images [STACK]   Show configured images and versions
  list             List stacks, stages, and management status
  help, -h, --help Show this help

Stage configuration:
  max_parallel N                   Set concurrency, 0-65535 (0 = unlimited)
  wait_timeout N                   Wait 0-86400 seconds for startup (default 120;
                                   0 disables waiting)
  before [--parallel N] STACK...    Add an early stage
  after [--parallel N] STACK...     Add a late stage
  exclude STACK...                  Exclude stacks from bulk operations

  Each before/after declaration is one ordered stage; unlisted stacks form
  the default stage. Lifecycle commands run before -> default -> after,
  reversed for stop/down. A failed stage skips all remaining stages.
  up/start/update wait for running/healthy services when waiting is enabled.
  Services need healthchecks to verify readiness beyond the running state.
  up/update validate every selected Compose configuration before making changes.
  Numeric settings do not accept leading zeros.

Config: $CONFIG_FILE
EOF
}

contains() {
	local target="$1"
	shift
	local item

	for item in "$@"; do
		[[ "$item" == "$target" ]] && return 0
	done

	return 1
}

validate_integer_range() {
	local kind="$1"
	local value="$2"
	local maximum="$3"

	[[ "$value" =~ ^(0|[1-9][0-9]*)$ ]] &&
		((${#value} <= ${#maximum})) && ((value <= maximum)) ||
		die "$kind must be an integer between 0 and $maximum without leading zeros: $value"
}

validate_parallel_limit() {
	validate_integer_range "parallel limit" "$1" 65535
}

wait_timeout() {
	[[ $# -eq 1 ]] || die "wait_timeout expects exactly one value"
	validate_integer_range "wait timeout" "$1" 86400
	WAIT_TIMEOUT="$1"
}

validate_name() {
	local kind="$1"
	local value="$2"

	[[ "$value" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]] ||
		die "invalid $kind name: $value"
}

# Set the maximum concurrency within each stage.
max_parallel() {
	[[ $# -eq 1 ]] || die "max_parallel expects exactly one value"
	validate_parallel_limit "$1"
	DEFAULT_MAX_PARALLEL="$1"
}

add_stage() {
	local stages_name="$1"
	local limits_name="$2"
	local stage="$3"
	local limit=""
	local stack
	local -n stages="$stages_name"
	local -n limits="$limits_name"
	shift 3

	if [[ "${1:-}" == "--parallel" ]]; then
		[[ $# -ge 3 ]] || die "$stage --parallel expects a value and at least one stack"
		limit="$2"
		validate_parallel_limit "$limit"
		shift 2
	fi

	[[ $# -ge 1 ]] || die "$stage expects at least one stack"

	for stack in "$@"; do
		validate_name "stack" "$stack"
		contains "$stack" "${CONFIGURED_STACKS[@]}" &&
			die "stack is assigned more than once: $stack"
		contains "$stack" "${EXCLUDED_STACKS[@]}" &&
			die "stack is both assigned and excluded: $stack"
		CONFIGURED_STACKS+=("$stack")
	done

	stages+=("$*")
	limits+=("$limit")
}

before() { add_stage BEFORE_STAGES BEFORE_LIMITS before "$@"; }
after() { add_stage AFTER_STAGES AFTER_LIMITS after "$@"; }

# Exclude stacks from all bulk operations.
exclude() {
	[[ $# -ge 1 ]] || die "exclude expects at least one stack"

	local stack
	for stack in "$@"; do
		validate_name "stack" "$stack"
		contains "$stack" "${EXCLUDED_STACKS[@]}" &&
			die "stack is excluded more than once: $stack"
		contains "$stack" "${CONFIGURED_STACKS[@]}" &&
			die "stack is both assigned and excluded: $stack"
		EXCLUDED_STACKS+=("$stack")
	done
}

load_config() {
	[[ -f "$CONFIG_FILE" ]] || return 0

	# The trusted config calls the configuration functions above.
	source "$CONFIG_FILE" || die "cannot load config file: $CONFIG_FILE"
}

discover_stacks() {
	local compose_file stack

	DISCOVERED_STACKS=()

	for compose_file in "$STACKS_DIR"/*/compose.yaml; do
		stack="${compose_file%/compose.yaml}"
		stack="${stack##*/}"
		validate_name "stack directory" "$stack"
		DISCOVERED_STACKS+=("$stack")
	done
}

is_discovered() {
	contains "$1" "${DISCOVERED_STACKS[@]}"
}

is_managed() {
	is_discovered "$1" && ! contains "$1" "${EXCLUDED_STACKS[@]}"
}

validate_config_stacks() {
	local stack

	for stack in "${CONFIGURED_STACKS[@]}"; do
		is_discovered "$stack" || die "configured stack not found: $stack"
	done

	for stack in "${EXCLUDED_STACKS[@]}"; do
		is_discovered "$stack" || die "excluded stack not found: $stack"
	done
}

append_stages() {
	local stage="$1"
	local stages_name="$2"
	local limits_name="$3"
	local i name
	local -a stacks=()
	local -n stages="$stages_name"
	local -n limits="$limits_name"

	for ((i = 0; i < ${#stages[@]}; i++)); do
		name="$stage"
		((${#stages[@]} > 1)) && name+="-$((i + 1))"
		STAGE_NAMES+=("$name")
		STAGE_LIMITS+=("${limits[i]:-$DEFAULT_MAX_PARALLEL}")
		STAGE_STACKS+=("${stages[i]}")
		read -r -a stacks <<<"${stages[i]}"
		MANAGED_STACKS+=("${stacks[@]}")
	done
}

build_execution_stages() {
	local stack
	local -a default_stacks=()

	STAGE_NAMES=()
	STAGE_LIMITS=()
	STAGE_STACKS=()
	MANAGED_STACKS=()

	append_stages before BEFORE_STAGES BEFORE_LIMITS

	for stack in "${DISCOVERED_STACKS[@]}"; do
		if ! contains "$stack" "${CONFIGURED_STACKS[@]}" &&
			! contains "$stack" "${EXCLUDED_STACKS[@]}"; then
			default_stacks+=("$stack")
		fi
	done

	if ((${#default_stacks[@]} > 0)); then
		STAGE_NAMES+=("default")
		STAGE_LIMITS+=("$DEFAULT_MAX_PARALLEL")
		STAGE_STACKS+=("${default_stacks[*]}")
		MANAGED_STACKS+=("${default_stacks[@]}")
	fi

	append_stages after AFTER_STAGES AFTER_LIMITS
}

stack_stage() {
	local target="$1"
	local i
	local -a stacks=()

	for ((i = 0; i < ${#STAGE_NAMES[@]}; i++)); do
		read -r -a stacks <<<"${STAGE_STACKS[i]}"
		if contains "$target" "${stacks[@]}"; then
			echo "${STAGE_NAMES[i]}"
			return 0
		fi
	done

	return 1
}

compose() {
	local stack="$1"
	shift
	local -a docker_args=(compose)

	if [[ "${STACKS_QUIET_PROGRESS:-0}" == "1" ]]; then
		docker_args+=(--progress quiet)
	fi

	(
		cd -- "$STACKS_DIR/$stack" || exit 1
		docker "${docker_args[@]}" "$@"
	)
}

run_stack() {
	local stack="$1"
	local action="$2"
	local show_heading="${3:-1}"
	local -a wait_args=()

	if ((WAIT_TIMEOUT > 0)); then
		wait_args=(--wait --wait-timeout "$WAIT_TIMEOUT")
	fi

	if [[ "$show_heading" == "1" ]]; then
		echo
		echo "==> $stack: $action"
	fi

	case "$action" in
	up)
		compose "$stack" up -d "${wait_args[@]}"
		;;
	start)
		compose "$stack" start "${wait_args[@]}"
		;;
	update)
		compose "$stack" pull &&
			compose "$stack" up -d "${wait_args[@]}"
		;;
	*)
		compose "$stack" "$action"
		;;
	esac
}

preflight_configs() {
	local stack
	local -a failed_stacks=()

	(($# > 0)) || return 0
	printf '\n==> Checking Compose configurations\n'
	for stack in "$@"; do
		compose "$stack" config -q || failed_stacks+=("$stack")
	done
	if ((${#failed_stacks[@]} > 0)); then
		printf 'ERROR: invalid Compose configuration: %s\n' "${failed_stacks[*]}" >&2
		return 1
	fi
}

run_stage() {
	local stage_index="$1"
	local action="$2"
	local stage_name="${STAGE_NAMES[stage_index]}"
	local limit="${STAGE_LIMITS[stage_index]}"
	local stack pid finished_pid status success_result working_result details
	local index finished_index total
	local next=0 running=0 completed=0 failed=0 stack_width=0 interactive=0 rendered=0
	local success_color="" failure_color="" color_reset=""
	local -a stacks=() states=() markers=() state_colors=()
	local -A pid_indexes=()

	if [[ -t 1 ]]; then
		interactive=1
		if [[ -z "${NO_COLOR:-}" ]]; then
			success_color=$'\033[32m'
			failure_color=$'\033[31m'
			color_reset=$'\033[0m'
		fi
	fi

	read -r -a stacks <<<"${STAGE_STACKS[stage_index]}"
	total="${#stacks[@]}"
	((total > 0)) || return 0
	for ((index = 0; index < total; index++)); do
		stack="${stacks[index]}"
		((${#stack} > stack_width)) && stack_width="${#stack}"
		states+=("Waiting")
		markers+=(" ")
		state_colors+=("")
	done

	if ((limit == 0 || limit > total)); then
		limit="$total"
	fi

	if ((total == 1)); then
		details="1 stack"
	elif ((limit == total)); then
		details="$total stacks in parallel"
	else
		details="$total stacks, max $limit in parallel"
	fi

	echo
	if ((!interactive)); then
		echo "==> Stage $stage_name: $action ($details)"
	fi

	case "$action" in
	up | start)
		working_result="Starting"
		success_result="Started"
		;;
	restart)
		working_result="Restarting"
		success_result="Restarted"
		;;
	stop)
		working_result="Stopping"
		success_result="Stopped"
		;;
	down)
		working_result="Removing"
		success_result="Removed"
		;;
	pull)
		working_result="Pulling"
		success_result="Pulled"
		;;
	update)
		working_result="Updating"
		success_result="Updated"
		;;
	esac

	while :; do
		while ((next < total && running < limit)); do
			stack="${stacks[next]}"
			states[next]="$working_result"
			STACKS_QUIET_PROGRESS=1 run_stack "$stack" "$action" 0 &
			pid="$!"
			pid_indexes["$pid"]="$next"
			((next += 1, running += 1))
		done

		if ((interactive)); then
			((rendered)) && printf '\033[%dA' "$((total + 1))"
			printf '\r\033[2K[+] %s: %s %d/%d\n' \
				"$stage_name" "$action" "$completed" "$total"
			for ((index = 0; index < total; index++)); do
				printf '\r\033[2K %s%s%s Stack %-*s %s%s%s\n' \
					"${state_colors[index]}" "${markers[index]}" "$color_reset" \
					"$stack_width" "${stacks[index]}" \
					"${state_colors[index]}" "${states[index]}" "$color_reset"
			done
			rendered=1
		fi

		((running > 0)) || break

		finished_pid=""
		status=0
		wait -n -p finished_pid || status=$?
		[[ -n "$finished_pid" ]] || die "cannot identify completed stack"

		finished_index="${pid_indexes[$finished_pid]}"
		unset 'pid_indexes[$finished_pid]'
		stack="${stacks[finished_index]}"
		((running -= 1))

		if ((status == 0)); then
			markers[finished_index]="✔"
			states[finished_index]="$success_result"
			state_colors[finished_index]="$success_color"
			((RUN_SUCCESS += 1))
		else
			markers[finished_index]="✘"
			states[finished_index]="Failed"
			state_colors[finished_index]="$failure_color"
			((RUN_FAILED += 1))
			RUN_FAILED_STACKS+=("$stack")
			((failed += 1))
		fi

		((completed += 1))
		if ((!interactive)); then
			printf ' %s%s%s Stack %-*s %s%s%s\n' \
				"${state_colors[finished_index]}" "${markers[finished_index]}" "$color_reset" \
				"$stack_width" "$stack" \
				"${state_colors[finished_index]}" "${states[finished_index]}" "$color_reset"
		fi
	done

	((failed == 0))
}

print_run_summary() {
	echo
	echo "Result: $RUN_SUCCESS succeeded, $RUN_FAILED failed"

	if ((${#RUN_FAILED_STACKS[@]} > 0)); then
		echo "Failed stacks: ${RUN_FAILED_STACKS[*]}"
	fi

	if ((${#RUN_SKIPPED_STAGES[@]} > 0)); then
		echo "Skipped stages: ${RUN_SKIPPED_STAGES[*]}"
	fi
}

collect_skipped_stages() {
	local failed_index="$1"
	local direction="$2"
	local i

	RUN_SKIPPED_STAGES=()

	if [[ "$direction" == "reverse" ]]; then
		for ((i = failed_index - 1; i >= 0; i--)); do
			RUN_SKIPPED_STAGES+=("${STAGE_NAMES[i]}")
		done
	else
		for ((i = failed_index + 1; i < ${#STAGE_NAMES[@]}; i++)); do
			RUN_SKIPPED_STAGES+=("${STAGE_NAMES[i]}")
		done
	fi
}

run_all_staged() {
	local action="$1"
	local direction="forward"
	local start=0 end="${#STAGE_NAMES[@]}" step=1
	local i

	RUN_SUCCESS=0
	RUN_FAILED=0
	RUN_FAILED_STACKS=()
	RUN_SKIPPED_STAGES=()
	if [[ "$action" == "stop" || "$action" == "down" ]]; then
		direction="reverse"
		start=$((end - 1))
		end=-1
		step=-1
	fi

	for ((i = start; i != end; i += step)); do
		if ! run_stage "$i" "$action"; then
			collect_skipped_stages "$i" "$direction"
			print_run_summary
			return 1
		fi
	done

	print_run_summary
}

run_all_sequential() {
	local action="$1"
	local stack
	local -a failed_stacks=()
	for stack in "${MANAGED_STACKS[@]}"; do
		run_stack "$stack" "$action" || failed_stacks+=("$stack")
	done

	if ((${#failed_stacks[@]} > 0)); then
		echo
		echo "Failed stacks: ${failed_stacks[*]}" >&2
		return 1
	fi
}

list_stacks() {
	local stack stage_name status

	printf '%-24s %-16s %s\n' "STACK" "STAGE" "STATUS"
	printf '%-24s %-16s %s\n' "------------------------" "----------------" "---------------"

	for stack in "${DISCOVERED_STACKS[@]}"; do
		if contains "$stack" "${EXCLUDED_STACKS[@]}"; then
			stage_name="-"
			status="excluded"
		else
			stage_name="$(stack_stage "$stack")"
			status="managed"
		fi

		printf '%-24s %-16s %s\n' "$stack" "$stage_name" "$status"
	done
}

stack_images() {
	local stack="$1"
	local output image

	output="$(compose "$stack" config --images)" || return 1

	while IFS= read -r image; do
		[[ -n "$image" ]] || continue
		printf '%-24s %s\n' "$stack" "$image"
	done <<<"$output"
}

list_images() {
	local target="${1:-}"
	local stack
	local failed=0

	printf '%-24s %s\n' "STACK" "IMAGE"
	printf '%-24s %s\n' "------------------------" "----------------------------------------"

	if [[ -n "$target" ]]; then
		require_target "images" "$target"
		stack_images "$target"
		return
	fi

	for stack in "${MANAGED_STACKS[@]}"; do
		stack_images "$stack" || ((failed += 1))
	done

	((failed == 0))
}

require_target() {
	local command="$1"
	local target="$2"

	[[ -n "$target" ]] || die "$command requires a stack name"
	is_discovered "$target" || die "stack not found: $target"
	is_managed "$target" || die "stack is excluded: $target"
}

acquire_lock() {
	mkdir -p "$(dirname "$LOCK_FILE")" ||
		die "cannot create lock directory"
	exec {LOCK_FD}>"$LOCK_FILE" ||
		die "cannot open lock file: $LOCK_FILE"
	flock -n "$LOCK_FD" ||
		die "another manage.sh operation is already running"
}

main() {
	local ACTION="${1:-}"
	local TARGET="${2:-}"
	local -a targets=()

	[[ $# -le 2 ]] || die "too many arguments"

	case "$ACTION" in
	help | -h | --help)
		usage
		exit 0
		;;
	esac

	load_config
	discover_stacks
	validate_config_stacks
	build_execution_stages

	case "$ACTION" in
	up | start | stop | restart | down | pull | update)
		acquire_lock

		if [[ -n "$TARGET" ]]; then
			require_target "$ACTION" "$TARGET"
			targets=("$TARGET")
		else
			targets=("${MANAGED_STACKS[@]}")
		fi
		if [[ "$ACTION" == "up" || "$ACTION" == "update" ]]; then
			preflight_configs "${targets[@]}" || return 1
		fi

		if [[ -n "$TARGET" ]]; then
			run_stack "$TARGET" "$ACTION"
		else
			run_all_staged "$ACTION"
		fi
		;;
	ps)
		if [[ -n "$TARGET" ]]; then
			require_target "$ACTION" "$TARGET"
			run_stack "$TARGET" "$ACTION"
		else
			run_all_sequential "$ACTION"
		fi
		;;
	logs)
		require_target "$ACTION" "$TARGET"
		compose "$TARGET" logs -f --tail=100
		;;
	config)
		require_target "$ACTION" "$TARGET"
		compose "$TARGET" config
		;;
	images)
		list_images "$TARGET"
		;;
	list)
		[[ -z "$TARGET" ]] || die "list does not accept a stack name"
		list_stacks
		;;
	*)
		usage
		exit 1
		;;
	esac
}

if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
	trap 'handle_signal 130' INT
	trap 'handle_signal 143' TERM
	# One isolated process group covers the scheduler and all Docker descendants.
	# Disable job control inside it so parallel workers stay in the same group.
	set -m
	(
		set +m
		trap - INT TERM
		main "$@"
	) <&0 &
	main_pid=$!
	set +m
	wait "$main_pid"
fi