#!/usr/bin/env bash set -uo pipefail shopt -s nullglob SCRIPT_DIR="$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")" STACKS_DIR="${STACKS_DIR:-$SCRIPT_DIR}" CONFIG_FILE="${STACKS_CONFIG:-$STACKS_DIR/manage.conf}" LOCK_FILE="${STACKS_LOCK_FILE:-/run/lock/stacks-manage.lock}" DEFAULT_MAX_PARALLEL=2 WAIT_TIMEOUT=120 BEFORE_STAGES=() BEFORE_LIMITS=() AFTER_STAGES=() AFTER_LIMITS=() CONFIGURED_STACKS=() EXCLUDED_STACKS=() DISCOVERED_STACKS=() MANAGED_STACKS=() STAGE_NAMES=() STAGE_LIMITS=() STAGE_STACKS=() RUN_SUCCESS=0 RUN_FAILED=0 RUN_FAILED_STACKS=() RUN_SKIPPED_STAGES=() RUN_SKIPPED_STACKS=() die() { echo "ERROR: $*" >&2 exit 1 } if ((BASH_VERSINFO[0] < 5 || (\ BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] < 1))); then die "Bash 5.1 or newer is required" fi handle_signal() { local status="$1" local pid attempt alive local -a pids=() trap '' INT TERM mapfile -t pids < <(jobs -p) for pid in "${pids[@]}"; do kill -TERM -- "-$pid" 2>/dev/null || : done # Allow five seconds for cleanup, then stop every remaining descendant. for ((attempt = 0; attempt < 50; attempt++)); do alive=0 for pid in "${pids[@]}"; do kill -0 -- "-$pid" 2>/dev/null && alive=1 done ((alive)) || break sleep 0.1 done for pid in "${pids[@]}"; do kill -KILL -- "-$pid" 2>/dev/null || : done ((${#pids[@]} == 0)) || wait "${pids[@]}" 2>/dev/null printf '\n' exit "$status" } usage() { cat < default -> after. stop/down reverse the full stage order. Stages run one at a time. Stacks within a stage run in parallel, limited by max_parallel or that stage's --parallel setting. A stage finishes all its stacks; any failure skips the remaining stages. Startup checks: up/update validate all selected Compose configurations before making changes. up/start/update wait up to wait_timeout seconds for each stack to be ready. Services with healthchecks must be healthy; other services must be running. Set wait_timeout to 0 to disable waiting. Config: $CONFIG_FILE EOF } contains() { local target="$1" shift local item for item in "$@"; do [[ "$item" == "$target" ]] && return 0 done return 1 } validate_integer_range() { local kind="$1" local value="$2" local maximum="$3" [[ "$value" =~ ^(0|[1-9][0-9]*)$ ]] && ((${#value} <= ${#maximum})) && ((value <= maximum)) || die "$kind must be an integer between 0 and $maximum without leading zeros: $value" } validate_parallel_limit() { validate_integer_range "parallel limit" "$1" 65535 } wait_timeout() { [[ $# -eq 1 ]] || die "wait_timeout expects exactly one value" validate_integer_range "wait timeout" "$1" 86400 WAIT_TIMEOUT="$1" } validate_name() { local kind="$1" local value="$2" [[ "$value" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]] || die "invalid $kind name: $value" } # Rewrite a stack argument to its bare stack name in place. Plain names # pass through unchanged. A folder reference such as ./stack/ or an # absolute path must resolve, after following symlinks, to the stack # folder inside STACKS_DIR; paths to folders elsewhere are rejected. normalize_target() { local target_name="$1" local -n value="$target_name" local original="$value" local name path resolved canonical root while [[ "$value" == */ ]]; do value="${value%/}" done # Names without a path component are validated against the stacks. if [[ "$value" != */* ]]; then [[ -n "$value" ]] || die "invalid stack path: $original" return 0 fi name="${value##*/}" [[ -n "$name" ]] || die "invalid stack path: $original" root="$(readlink -f -- "$STACKS_DIR" 2>/dev/null)" || die "invalid STACKS_DIR: $STACKS_DIR" path="$value" [[ "$path" == /* ]] || path="$STACKS_DIR/$path" resolved="$(readlink -f -- "$path" 2>/dev/null)" || die "stack folder not found: $original" [[ -d "$resolved" ]] || die "stack folder not found: $original" canonical="$(readlink -f -- "$root/$name" 2>/dev/null)" [[ "$resolved" == "$canonical" ]] || die "not a stack folder in $root: $original" value="$name" } # Set the maximum concurrency within each stage. max_parallel() { [[ $# -eq 1 ]] || die "max_parallel expects exactly one value" validate_parallel_limit "$1" DEFAULT_MAX_PARALLEL="$1" } add_stage() { local stages_name="$1" local limits_name="$2" local stage="$3" local limit="" local stack local -n stages="$stages_name" local -n limits="$limits_name" local -a normalized=() shift 3 if [[ "${1:-}" == "--parallel" ]]; then [[ $# -ge 3 ]] || die "$stage --parallel expects a value and at least one stack" limit="$2" validate_parallel_limit "$limit" shift 2 fi [[ $# -ge 1 ]] || die "$stage expects at least one stack" for stack in "$@"; do normalize_target stack validate_name "stack" "$stack" contains "$stack" "${CONFIGURED_STACKS[@]}" && die "stack is assigned more than once: $stack" contains "$stack" "${EXCLUDED_STACKS[@]}" && die "stack is both assigned and excluded: $stack" CONFIGURED_STACKS+=("$stack") normalized+=("$stack") done stages+=("${normalized[*]}") limits+=("$limit") } before() { add_stage BEFORE_STAGES BEFORE_LIMITS before "$@"; } after() { add_stage AFTER_STAGES AFTER_LIMITS after "$@"; } # Exclude stacks from all bulk operations. exclude() { [[ $# -ge 1 ]] || die "exclude expects at least one stack" local stack for stack in "$@"; do normalize_target stack validate_name "stack" "$stack" contains "$stack" "${EXCLUDED_STACKS[@]}" && die "stack is excluded more than once: $stack" contains "$stack" "${CONFIGURED_STACKS[@]}" && die "stack is both assigned and excluded: $stack" EXCLUDED_STACKS+=("$stack") done } load_config() { [[ -f "$CONFIG_FILE" ]] || return 0 # The trusted config calls the configuration functions above. source "$CONFIG_FILE" || die "cannot load config file: $CONFIG_FILE" } discover_stacks() { local compose_file stack DISCOVERED_STACKS=() for compose_file in "$STACKS_DIR"/*/compose.yaml; do stack="${compose_file%/compose.yaml}" stack="${stack##*/}" validate_name "stack directory" "$stack" DISCOVERED_STACKS+=("$stack") done } is_discovered() { contains "$1" "${DISCOVERED_STACKS[@]}" } is_managed() { is_discovered "$1" && ! contains "$1" "${EXCLUDED_STACKS[@]}" } validate_config_stacks() { local stack for stack in "${CONFIGURED_STACKS[@]}"; do is_discovered "$stack" || die "configured stack not found: $stack" done for stack in "${EXCLUDED_STACKS[@]}"; do is_discovered "$stack" || die "excluded stack not found: $stack" done } append_stages() { local stage="$1" local stages_name="$2" local limits_name="$3" local i name local -a stacks=() local -n stages="$stages_name" local -n limits="$limits_name" for ((i = 0; i < ${#stages[@]}; i++)); do name="$stage" ((${#stages[@]} > 1)) && name+="-$((i + 1))" STAGE_NAMES+=("$name") STAGE_LIMITS+=("${limits[i]:-$DEFAULT_MAX_PARALLEL}") STAGE_STACKS+=("${stages[i]}") read -r -a stacks <<<"${stages[i]}" MANAGED_STACKS+=("${stacks[@]}") done } build_execution_stages() { local stack local -a default_stacks=() STAGE_NAMES=() STAGE_LIMITS=() STAGE_STACKS=() MANAGED_STACKS=() append_stages before BEFORE_STAGES BEFORE_LIMITS for stack in "${DISCOVERED_STACKS[@]}"; do if ! contains "$stack" "${CONFIGURED_STACKS[@]}" && ! contains "$stack" "${EXCLUDED_STACKS[@]}"; then default_stacks+=("$stack") fi done if ((${#default_stacks[@]} > 0)); then STAGE_NAMES+=("default") STAGE_LIMITS+=("$DEFAULT_MAX_PARALLEL") STAGE_STACKS+=("${default_stacks[*]}") MANAGED_STACKS+=("${default_stacks[@]}") fi append_stages after AFTER_STAGES AFTER_LIMITS } stack_stage() { local target="$1" local i local -a stacks=() for ((i = 0; i < ${#STAGE_NAMES[@]}; i++)); do read -r -a stacks <<<"${STAGE_STACKS[i]}" if contains "$target" "${stacks[@]}"; then echo "${STAGE_NAMES[i]}" return 0 fi done return 1 } compose() { local stack="$1" shift local -a docker_args=(compose) if [[ "${STACKS_QUIET_PROGRESS:-0}" == "1" ]]; then docker_args+=(--progress quiet) fi ( cd -- "$STACKS_DIR/$stack" || exit 1 docker "${docker_args[@]}" "$@" ) } run_stack() { local stack="$1" local action="$2" local show_heading="${3:-1}" local -a wait_args=() if ((WAIT_TIMEOUT > 0)); then wait_args=(--wait --wait-timeout "$WAIT_TIMEOUT") fi if [[ "$show_heading" == "1" ]]; then echo echo "==> $stack: $action" fi case "$action" in up) compose "$stack" up -d "${wait_args[@]}" ;; start) compose "$stack" start "${wait_args[@]}" ;; update) compose "$stack" pull && compose "$stack" up -d "${wait_args[@]}" ;; *) compose "$stack" "$action" ;; esac } preflight_configs() { local stack local -a failed_stacks=() (($# > 0)) || return 0 printf '\n==> Checking Compose configurations\n' for stack in "$@"; do compose "$stack" config -q || failed_stacks+=("$stack") done if ((${#failed_stacks[@]} > 0)); then printf 'ERROR: invalid Compose configuration: %s\n' "${failed_stacks[*]}" >&2 return 1 fi } run_stage() { local stage_index="$1" local action="$2" local stage_name="${STAGE_NAMES[stage_index]}" local limit="${STAGE_LIMITS[stage_index]}" local stack pid finished_pid status success_result working_result details local index finished_index total local next=0 running=0 completed=0 failed=0 stack_width=0 interactive=0 rendered=0 local success_color="" failure_color="" color_reset="" local -a stacks=() states=() markers=() state_colors=() local -A pid_indexes=() if [[ -t 1 ]]; then interactive=1 if [[ -z "${NO_COLOR:-}" ]]; then success_color=$'\033[32m' failure_color=$'\033[31m' color_reset=$'\033[0m' fi fi read -r -a stacks <<<"${STAGE_STACKS[stage_index]}" total="${#stacks[@]}" ((total > 0)) || return 0 for ((index = 0; index < total; index++)); do stack="${stacks[index]}" ((${#stack} > stack_width)) && stack_width="${#stack}" states+=("Waiting") markers+=(" ") state_colors+=("") done if ((limit == 0 || limit > total)); then limit="$total" fi if ((total == 1)); then details="1 stack" elif ((limit == total)); then details="$total stacks in parallel" else details="$total stacks, max $limit in parallel" fi echo if ((!interactive)); then echo "==> Stage $stage_name: $action ($details)" fi case "$action" in up | start) working_result="Starting" success_result="Started" ;; restart) working_result="Restarting" success_result="Restarted" ;; stop) working_result="Stopping" success_result="Stopped" ;; down) working_result="Removing" success_result="Removed" ;; pull) working_result="Pulling" success_result="Pulled" ;; update) working_result="Updating" success_result="Updated" ;; esac while :; do while ((next < total && running < limit)); do stack="${stacks[next]}" states[next]="$working_result" STACKS_QUIET_PROGRESS=1 run_stack "$stack" "$action" 0 & pid="$!" pid_indexes["$pid"]="$next" ((next += 1, running += 1)) done if ((interactive)); then ((rendered)) && printf '\033[%dA' "$((total + 1))" printf '\r\033[2K[+] %s: %s %d/%d\n' \ "$stage_name" "$action" "$completed" "$total" for ((index = 0; index < total; index++)); do printf '\r\033[2K %s%s%s Stack %-*s %s%s%s\n' \ "${state_colors[index]}" "${markers[index]}" "$color_reset" \ "$stack_width" "${stacks[index]}" \ "${state_colors[index]}" "${states[index]}" "$color_reset" done rendered=1 fi ((running > 0)) || break finished_pid="" status=0 wait -n -p finished_pid || status=$? [[ -n "$finished_pid" ]] || die "cannot identify completed stack" finished_index="${pid_indexes[$finished_pid]}" unset 'pid_indexes[$finished_pid]' stack="${stacks[finished_index]}" ((running -= 1)) if ((status == 0)); then markers[finished_index]="✔" states[finished_index]="$success_result" state_colors[finished_index]="$success_color" ((RUN_SUCCESS += 1)) else markers[finished_index]="✘" states[finished_index]="Failed" state_colors[finished_index]="$failure_color" ((RUN_FAILED += 1)) RUN_FAILED_STACKS+=("$stack") ((failed += 1)) fi ((completed += 1)) if ((!interactive)); then printf ' %s%s%s Stack %-*s %s%s%s\n' \ "${state_colors[finished_index]}" "${markers[finished_index]}" "$color_reset" \ "$stack_width" "$stack" \ "${state_colors[finished_index]}" "${states[finished_index]}" "$color_reset" fi done ((failed == 0)) } print_run_summary() { echo echo "Result: $RUN_SUCCESS succeeded, $RUN_FAILED failed" if ((${#RUN_FAILED_STACKS[@]} > 0)); then echo "Failed stacks: ${RUN_FAILED_STACKS[*]}" fi if ((${#RUN_SKIPPED_STACKS[@]} > 0)); then echo "Skipped stacks: ${RUN_SKIPPED_STACKS[*]}" fi if ((${#RUN_SKIPPED_STAGES[@]} > 0)); then echo "Skipped stages: ${RUN_SKIPPED_STAGES[*]}" fi } collect_skipped_stages() { local failed_index="$1" local direction="$2" local i RUN_SKIPPED_STAGES=() if [[ "$direction" == "reverse" ]]; then for ((i = failed_index - 1; i >= 0; i--)); do RUN_SKIPPED_STAGES+=("${STAGE_NAMES[i]}") done else for ((i = failed_index + 1; i < ${#STAGE_NAMES[@]}; i++)); do RUN_SKIPPED_STAGES+=("${STAGE_NAMES[i]}") done fi } run_all_staged() { local action="$1" local direction="forward" local start=0 end="${#STAGE_NAMES[@]}" step=1 local i RUN_SUCCESS=0 RUN_FAILED=0 RUN_FAILED_STACKS=() RUN_SKIPPED_STAGES=() RUN_SKIPPED_STACKS=() if [[ "$action" == "stop" || "$action" == "down" ]]; then direction="reverse" start=$((end - 1)) end=-1 step=-1 fi for ((i = start; i != end; i += step)); do if ! run_stage "$i" "$action"; then collect_skipped_stages "$i" "$direction" print_run_summary return 1 fi done print_run_summary } # Run the given stacks one at a time, in the given order. A failure stops # the run; the remaining stacks are reported as skipped. run_stacks_sequential() { local action="$1" local index stack local -a stacks=("${@:2}") RUN_SUCCESS=0 RUN_FAILED=0 RUN_FAILED_STACKS=() RUN_SKIPPED_STAGES=() RUN_SKIPPED_STACKS=() for ((index = 0; index < ${#stacks[@]}; index++)); do stack="${stacks[index]}" if run_stack "$stack" "$action"; then ((RUN_SUCCESS += 1)) else ((RUN_FAILED += 1)) RUN_FAILED_STACKS+=("$stack") RUN_SKIPPED_STACKS=("${stacks[@]:$((index + 1))}") break fi done print_run_summary ((RUN_FAILED == 0)) } run_all_sequential() { local action="$1" shift local stack local -a failed_stacks=() for stack in "$@"; do run_stack "$stack" "$action" || failed_stacks+=("$stack") done if ((${#failed_stacks[@]} > 0)); then echo echo "Failed stacks: ${failed_stacks[*]}" >&2 return 1 fi } print_table() { local columns="$1" shift local -a cells=("$@") widths=() local index column row for ((index = 0; index < ${#cells[@]}; index++)); do column=$((index % columns)) if ((${#cells[index]} > ${widths[column]:-0})); then widths[column]="${#cells[index]}" fi done for ((row = 0; row < ${#cells[@]}; row += columns)); do for ((column = 0; column < columns; column++)); do ((column > 0)) && printf ' ' if ((column == columns - 1)); then printf '%s\n' "${cells[row + column]}" else printf '%-*s' "${widths[column]}" "${cells[row + column]}" fi done done } list_stacks() { local stack stage_name status local -a cells=(STACK STAGE STATUS) for stack in "${DISCOVERED_STACKS[@]}"; do if contains "$stack" "${EXCLUDED_STACKS[@]}"; then stage_name="-" status="excluded" else stage_name="$(stack_stage "$stack")" status="managed" fi cells+=("$stack" "$stage_name" "$status") done print_table 3 "${cells[@]}" } list_images() { local stack output service image local failed=0 local -a cells=(STACK SERVICE IMAGE) for stack in "$@"; do # Read service/image pairs from Compose's canonical YAML (two-space indent). if ! output="$(compose "$stack" config --format yaml | awk -v project="$stack" ' { gsub(/["\047]/, "") } /^name: / { project = $2 } /^[^ ]/ { in_services = ($0 == "services:") } in_services && /^ [^ ]/ { service = $1 sub(/:$/, "", service) services[++count] = service images[service] = project "-" service } in_services && /^ image: / { images[service] = $2 } END { for (i = 1; i <= count; i++) print services[i] "\t" images[services[i]] } ')"; then ((failed += 1)) continue fi while IFS=$'\t' read -r service image; do [[ -n "$service" ]] || continue cells+=("$stack" "$service" "$image") done <<<"$output" done print_table 3 "${cells[@]}" ((failed == 0)) } require_target() { local command="$1" local target="$2" [[ -n "$target" ]] || die "$command requires a stack name" is_discovered "$target" || die "stack not found: $target" is_managed "$target" || die "stack is excluded: $target" } # Validate every requested stack before acting on any of them. require_targets() { local command="$1" local target shift for target in "$@"; do require_target "$command" "$target" done } acquire_lock() { mkdir -p "$(dirname "$LOCK_FILE")" || die "cannot create lock directory" exec {LOCK_FD}>"$LOCK_FILE" || die "cannot open lock file: $LOCK_FILE" flock -n "$LOCK_FD" || die "another manage.sh operation is already running" } main() { local ACTION="${1:-}" local target local -a requested=("${@:2}") local -a targets=() local -a stacks=() case "$ACTION" in help | -h | --help) usage exit 0 ;; esac for target in "${requested[@]}"; do normalize_target target contains "$target" "${targets[@]}" && die "stack is specified more than once: $target" targets+=("$target") done load_config discover_stacks validate_config_stacks build_execution_stages # Commands act on the requested stacks, or on every managed stack when # none were requested. stacks=("${targets[@]}") ((${#stacks[@]} > 0)) || stacks=("${MANAGED_STACKS[@]}") case "$ACTION" in up | start | stop | restart | down | pull | update) acquire_lock require_targets "$ACTION" "${targets[@]}" if [[ "$ACTION" == "up" || "$ACTION" == "update" ]]; then preflight_configs "${stacks[@]}" || return 1 fi if ((${#targets[@]} == 0)); then run_all_staged "$ACTION" elif ((${#targets[@]} == 1)); then run_stack "${stacks[0]}" "$ACTION" else run_stacks_sequential "$ACTION" "${stacks[@]}" fi ;; ps) require_targets "$ACTION" "${targets[@]}" run_all_sequential "$ACTION" "${stacks[@]}" ;; logs) ((${#targets[@]} <= 1)) || die "$ACTION expects a single stack" require_target "$ACTION" "${targets[0]:-}" compose "${targets[0]}" logs -f --tail=100 ;; config) ((${#targets[@]} <= 1)) || die "$ACTION expects a single stack" require_target "$ACTION" "${targets[0]:-}" compose "${targets[0]}" config ;; images) require_targets "$ACTION" "${targets[@]}" list_images "${stacks[@]}" ;; list) ((${#targets[@]} == 0)) || die "list does not accept a stack name" list_stacks ;; *) usage exit 1 ;; esac } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then trap 'handle_signal 130' INT trap 'handle_signal 143' TERM # One isolated process group covers the scheduler and all Docker descendants. # Disable job control inside it so parallel workers stay in the same group. set -m ( set +m trap - INT TERM main "$@" ) <&0 & main_pid=$! set +m wait "$main_pid" fi