fix: harden stack validation and lifecycle execution
This commit is contained in:
1 parent
e483a4ec3a
commit
fdcc47cb5b
2 files changed
+158
-69
No files matched your search
+6
-1
@@ -1,7 +1,12 @@
|
|||||||
# Maximum number of stacks executed concurrently within each stage.
|
# Maximum number of stacks executed concurrently within each stage.
|
||||||
# Use 0 for unlimited concurrency.
|
# Use 0 for unlimited concurrency. Range: 0-65535; no leading zeros.
|
||||||
max_parallel 3
|
max_parallel 3
|
||||||
|
|
||||||
|
# Wait for running/healthy services on up, start, and update.
|
||||||
|
# Healthchecks are needed to verify application readiness.
|
||||||
|
# Range: 0-86400 seconds; 0 disables waiting. Default: 120 seconds.
|
||||||
|
wait_timeout 120
|
||||||
|
|
||||||
# Each declaration is an ordered stage; stacks on the same line run in parallel.
|
# Each declaration is an ordered stage; stacks on the same line run in parallel.
|
||||||
# Add --parallel N before the stacks to override max_parallel for that stage.
|
# Add --parallel N before the stacks to override max_parallel for that stage.
|
||||||
# Example: before --parallel 2 stack-a stack-b
|
# Example: before --parallel 2 stack-a stack-b
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ CONFIG_FILE="${STACKS_CONFIG:-$STACKS_DIR/manage.conf}"
|
|||||||
LOCK_FILE="${STACKS_LOCK_FILE:-/run/lock/stacks-manage.lock}"
|
LOCK_FILE="${STACKS_LOCK_FILE:-/run/lock/stacks-manage.lock}"
|
||||||
|
|
||||||
DEFAULT_MAX_PARALLEL=2
|
DEFAULT_MAX_PARALLEL=2
|
||||||
|
WAIT_TIMEOUT=120
|
||||||
|
|
||||||
BEFORE_STAGES=()
|
BEFORE_STAGES=()
|
||||||
BEFORE_LIMITS=()
|
BEFORE_LIMITS=()
|
||||||
@@ -40,19 +41,32 @@ fi
|
|||||||
|
|
||||||
handle_signal() {
|
handle_signal() {
|
||||||
local status="$1"
|
local status="$1"
|
||||||
|
local pid attempt alive
|
||||||
local -a pids=()
|
local -a pids=()
|
||||||
|
|
||||||
trap - INT TERM
|
trap '' INT TERM
|
||||||
mapfile -t pids < <(jobs -pr)
|
mapfile -t pids < <(jobs -p)
|
||||||
((${#pids[@]} == 0)) || kill -TERM "${pids[@]}" 2>/dev/null
|
for pid in "${pids[@]}"; do
|
||||||
wait 2>/dev/null
|
kill -TERM -- "-$pid" 2>/dev/null || :
|
||||||
|
done
|
||||||
|
|
||||||
|
# Allow five seconds for cleanup, then stop every remaining descendant.
|
||||||
|
for ((attempt = 0; attempt < 50; attempt++)); do
|
||||||
|
alive=0
|
||||||
|
for pid in "${pids[@]}"; do
|
||||||
|
kill -0 -- "-$pid" 2>/dev/null && alive=1
|
||||||
|
done
|
||||||
|
((alive)) || break
|
||||||
|
sleep 0.1
|
||||||
|
done
|
||||||
|
for pid in "${pids[@]}"; do
|
||||||
|
kill -KILL -- "-$pid" 2>/dev/null || :
|
||||||
|
done
|
||||||
|
((${#pids[@]} == 0)) || wait "${pids[@]}" 2>/dev/null
|
||||||
printf '\n'
|
printf '\n'
|
||||||
exit "$status"
|
exit "$status"
|
||||||
}
|
}
|
||||||
|
|
||||||
trap 'handle_signal 130' INT
|
|
||||||
trap 'handle_signal 143' TERM
|
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
Usage: $(basename "$0") COMMAND [STACK]
|
Usage: $(basename "$0") COMMAND [STACK]
|
||||||
@@ -75,7 +89,9 @@ Inspect:
|
|||||||
help, -h, --help Show this help
|
help, -h, --help Show this help
|
||||||
|
|
||||||
Stage configuration:
|
Stage configuration:
|
||||||
max_parallel N Set default concurrency (0 = unlimited)
|
max_parallel N Set concurrency, 0-65535 (0 = unlimited)
|
||||||
|
wait_timeout N Wait 0-86400 seconds for startup (default 120;
|
||||||
|
0 disables waiting)
|
||||||
before [--parallel N] STACK... Add an early stage
|
before [--parallel N] STACK... Add an early stage
|
||||||
after [--parallel N] STACK... Add a late stage
|
after [--parallel N] STACK... Add a late stage
|
||||||
exclude STACK... Exclude stacks from bulk operations
|
exclude STACK... Exclude stacks from bulk operations
|
||||||
@@ -83,6 +99,10 @@ Stage configuration:
|
|||||||
Each before/after declaration is one ordered stage; unlisted stacks form
|
Each before/after declaration is one ordered stage; unlisted stacks form
|
||||||
the default stage. Lifecycle commands run before -> default -> after,
|
the default stage. Lifecycle commands run before -> default -> after,
|
||||||
reversed for stop/down. A failed stage skips all remaining stages.
|
reversed for stop/down. A failed stage skips all remaining stages.
|
||||||
|
up/start/update wait for running/healthy services when waiting is enabled.
|
||||||
|
Services need healthchecks to verify readiness beyond the running state.
|
||||||
|
up/update validate every selected Compose configuration before making changes.
|
||||||
|
Numeric settings do not accept leading zeros.
|
||||||
|
|
||||||
Config: $CONFIG_FILE
|
Config: $CONFIG_FILE
|
||||||
EOF
|
EOF
|
||||||
@@ -100,11 +120,24 @@ contains() {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
validate_parallel_limit() {
|
validate_integer_range() {
|
||||||
local value="$1"
|
local kind="$1"
|
||||||
|
local value="$2"
|
||||||
|
local maximum="$3"
|
||||||
|
|
||||||
[[ "$value" =~ ^[0-9]+$ ]] ||
|
[[ "$value" =~ ^(0|[1-9][0-9]*)$ ]] &&
|
||||||
die "parallel limit must be a non-negative integer: $value"
|
((${#value} <= ${#maximum})) && ((value <= maximum)) ||
|
||||||
|
die "$kind must be an integer between 0 and $maximum without leading zeros: $value"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_parallel_limit() {
|
||||||
|
validate_integer_range "parallel limit" "$1" 65535
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_timeout() {
|
||||||
|
[[ $# -eq 1 ]] || die "wait_timeout expects exactly one value"
|
||||||
|
validate_integer_range "wait timeout" "$1" 86400
|
||||||
|
WAIT_TIMEOUT="$1"
|
||||||
}
|
}
|
||||||
|
|
||||||
validate_name() {
|
validate_name() {
|
||||||
@@ -175,7 +208,7 @@ exclude() {
|
|||||||
load_config() {
|
load_config() {
|
||||||
[[ -f "$CONFIG_FILE" ]] || return 0
|
[[ -f "$CONFIG_FILE" ]] || return 0
|
||||||
|
|
||||||
# The trusted config calls max_parallel, before, after, and exclude above.
|
# The trusted config calls the configuration functions above.
|
||||||
source "$CONFIG_FILE" || die "cannot load config file: $CONFIG_FILE"
|
source "$CONFIG_FILE" || die "cannot load config file: $CONFIG_FILE"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -186,7 +219,9 @@ discover_stacks() {
|
|||||||
|
|
||||||
for compose_file in "$STACKS_DIR"/*/compose.yaml; do
|
for compose_file in "$STACKS_DIR"/*/compose.yaml; do
|
||||||
stack="${compose_file%/compose.yaml}"
|
stack="${compose_file%/compose.yaml}"
|
||||||
DISCOVERED_STACKS+=("${stack##*/}")
|
stack="${stack##*/}"
|
||||||
|
validate_name "stack directory" "$stack"
|
||||||
|
DISCOVERED_STACKS+=("$stack")
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -293,6 +328,11 @@ run_stack() {
|
|||||||
local stack="$1"
|
local stack="$1"
|
||||||
local action="$2"
|
local action="$2"
|
||||||
local show_heading="${3:-1}"
|
local show_heading="${3:-1}"
|
||||||
|
local -a wait_args=()
|
||||||
|
|
||||||
|
if ((WAIT_TIMEOUT > 0)); then
|
||||||
|
wait_args=(--wait --wait-timeout "$WAIT_TIMEOUT")
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ "$show_heading" == "1" ]]; then
|
if [[ "$show_heading" == "1" ]]; then
|
||||||
echo
|
echo
|
||||||
@@ -301,13 +341,14 @@ run_stack() {
|
|||||||
|
|
||||||
case "$action" in
|
case "$action" in
|
||||||
up)
|
up)
|
||||||
compose "$stack" config -q &&
|
compose "$stack" up -d "${wait_args[@]}"
|
||||||
compose "$stack" up -d
|
;;
|
||||||
|
start)
|
||||||
|
compose "$stack" start "${wait_args[@]}"
|
||||||
;;
|
;;
|
||||||
update)
|
update)
|
||||||
compose "$stack" config -q &&
|
compose "$stack" pull &&
|
||||||
compose "$stack" pull &&
|
compose "$stack" up -d "${wait_args[@]}"
|
||||||
compose "$stack" up -d
|
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
compose "$stack" "$action"
|
compose "$stack" "$action"
|
||||||
@@ -315,6 +356,21 @@ run_stack() {
|
|||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
|
preflight_configs() {
|
||||||
|
local stack
|
||||||
|
local -a failed_stacks=()
|
||||||
|
|
||||||
|
(($# > 0)) || return 0
|
||||||
|
printf '\n==> Checking Compose configurations\n'
|
||||||
|
for stack in "$@"; do
|
||||||
|
compose "$stack" config -q || failed_stacks+=("$stack")
|
||||||
|
done
|
||||||
|
if ((${#failed_stacks[@]} > 0)); then
|
||||||
|
printf 'ERROR: invalid Compose configuration: %s\n' "${failed_stacks[*]}" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
run_stage() {
|
run_stage() {
|
||||||
local stage_index="$1"
|
local stage_index="$1"
|
||||||
local action="$2"
|
local action="$2"
|
||||||
@@ -596,59 +652,87 @@ acquire_lock() {
|
|||||||
die "another manage.sh operation is already running"
|
die "another manage.sh operation is already running"
|
||||||
}
|
}
|
||||||
|
|
||||||
ACTION="${1:-}"
|
main() {
|
||||||
TARGET="${2:-}"
|
local ACTION="${1:-}"
|
||||||
|
local TARGET="${2:-}"
|
||||||
|
local -a targets=()
|
||||||
|
|
||||||
[[ $# -le 2 ]] || die "too many arguments"
|
[[ $# -le 2 ]] || die "too many arguments"
|
||||||
|
|
||||||
case "$ACTION" in
|
case "$ACTION" in
|
||||||
help | -h | --help)
|
help | -h | --help)
|
||||||
usage
|
usage
|
||||||
exit 0
|
exit 0
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
load_config
|
load_config
|
||||||
discover_stacks
|
discover_stacks
|
||||||
validate_config_stacks
|
validate_config_stacks
|
||||||
build_execution_stages
|
build_execution_stages
|
||||||
|
|
||||||
case "$ACTION" in
|
case "$ACTION" in
|
||||||
up | start | stop | restart | down | pull | update)
|
up | start | stop | restart | down | pull | update)
|
||||||
acquire_lock
|
acquire_lock
|
||||||
|
|
||||||
if [[ -n "$TARGET" ]]; then
|
if [[ -n "$TARGET" ]]; then
|
||||||
|
require_target "$ACTION" "$TARGET"
|
||||||
|
targets=("$TARGET")
|
||||||
|
else
|
||||||
|
targets=("${MANAGED_STACKS[@]}")
|
||||||
|
fi
|
||||||
|
if [[ "$ACTION" == "up" || "$ACTION" == "update" ]]; then
|
||||||
|
preflight_configs "${targets[@]}" || return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$TARGET" ]]; then
|
||||||
|
run_stack "$TARGET" "$ACTION"
|
||||||
|
else
|
||||||
|
run_all_staged "$ACTION"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
ps)
|
||||||
|
if [[ -n "$TARGET" ]]; then
|
||||||
|
require_target "$ACTION" "$TARGET"
|
||||||
|
run_stack "$TARGET" "$ACTION"
|
||||||
|
else
|
||||||
|
run_all_sequential "$ACTION"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
logs)
|
||||||
require_target "$ACTION" "$TARGET"
|
require_target "$ACTION" "$TARGET"
|
||||||
run_stack "$TARGET" "$ACTION"
|
compose "$TARGET" logs -f --tail=100
|
||||||
else
|
;;
|
||||||
run_all_staged "$ACTION"
|
config)
|
||||||
fi
|
|
||||||
;;
|
|
||||||
ps)
|
|
||||||
if [[ -n "$TARGET" ]]; then
|
|
||||||
require_target "$ACTION" "$TARGET"
|
require_target "$ACTION" "$TARGET"
|
||||||
run_stack "$TARGET" "$ACTION"
|
compose "$TARGET" config
|
||||||
else
|
;;
|
||||||
run_all_sequential "$ACTION"
|
images)
|
||||||
fi
|
list_images "$TARGET"
|
||||||
;;
|
;;
|
||||||
logs)
|
list)
|
||||||
require_target "$ACTION" "$TARGET"
|
[[ -z "$TARGET" ]] || die "list does not accept a stack name"
|
||||||
compose "$TARGET" logs -f --tail=100
|
list_stacks
|
||||||
;;
|
;;
|
||||||
config)
|
*)
|
||||||
require_target "$ACTION" "$TARGET"
|
usage
|
||||||
compose "$TARGET" config
|
exit 1
|
||||||
;;
|
;;
|
||||||
images)
|
esac
|
||||||
list_images "$TARGET"
|
}
|
||||||
;;
|
|
||||||
list)
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
||||||
[[ -z "$TARGET" ]] || die "list does not accept a stack name"
|
trap 'handle_signal 130' INT
|
||||||
list_stacks
|
trap 'handle_signal 143' TERM
|
||||||
;;
|
# One isolated process group covers the scheduler and all Docker descendants.
|
||||||
*)
|
# Disable job control inside it so parallel workers stay in the same group.
|
||||||
usage
|
set -m
|
||||||
exit 1
|
(
|
||||||
;;
|
set +m
|
||||||
esac
|
trap - INT TERM
|
||||||
|
main "$@"
|
||||||
|
) <&0 &
|
||||||
|
main_pid=$!
|
||||||
|
set +m
|
||||||
|
wait "$main_pid"
|
||||||
|
fi
|
||||||
Reference in new issue
Block a user