feat: improve staged execution safety
This commit is contained in:
lingkai5wu committed
1 parent
ebab7d3213
commit
1143ae9f62
1 file changed
+84
-64
@@ -10,29 +10,49 @@ LOCK_FILE="${STACKS_LOCK_FILE:-/run/lock/stacks-manage.lock}"
|
||||
|
||||
DEFAULT_MAX_PARALLEL=2
|
||||
|
||||
BEFORE_GROUPS=()
|
||||
BEFORE_STAGES=()
|
||||
BEFORE_LIMITS=()
|
||||
AFTER_GROUPS=()
|
||||
AFTER_STAGES=()
|
||||
AFTER_LIMITS=()
|
||||
CONFIGURED_STACKS=()
|
||||
EXCLUDED_STACKS=()
|
||||
|
||||
DISCOVERED_STACKS=()
|
||||
MANAGED_STACKS=()
|
||||
GROUP_NAMES=()
|
||||
GROUP_LIMITS=()
|
||||
GROUP_STACKS=()
|
||||
STAGE_NAMES=()
|
||||
STAGE_LIMITS=()
|
||||
STAGE_STACKS=()
|
||||
|
||||
RUN_SUCCESS=0
|
||||
RUN_FAILED=0
|
||||
RUN_FAILED_STACKS=()
|
||||
RUN_SKIPPED_GROUPS=()
|
||||
RUN_SKIPPED_STAGES=()
|
||||
|
||||
die() {
|
||||
echo "ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if ((BASH_VERSINFO[0] < 5 ||
|
||||
(BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] < 1))); then
|
||||
die "Bash 5.1 or newer is required"
|
||||
fi
|
||||
|
||||
handle_signal() {
|
||||
local status="$1"
|
||||
local -a pids=()
|
||||
|
||||
trap - INT TERM
|
||||
mapfile -t pids < <(jobs -pr)
|
||||
((${#pids[@]} == 0)) || kill -TERM "${pids[@]}" 2>/dev/null
|
||||
wait 2>/dev/null
|
||||
printf '\n'
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap 'handle_signal 130' INT
|
||||
trap 'handle_signal 143' TERM
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $(basename "$0") COMMAND [STACK]
|
||||
@@ -49,13 +69,13 @@ Commands:
|
||||
logs Follow logs for a stack
|
||||
config Render Compose configuration for a stack
|
||||
images Show configured images and versions
|
||||
list List discovered stacks and their group/status
|
||||
list List discovered stacks and their stage/status
|
||||
|
||||
Bulk execution:
|
||||
up/start/restart/pull/update run before, default, then after.
|
||||
stop/down run the same groups in reverse order.
|
||||
Each before/after declaration creates an ordered group.
|
||||
Stacks within a group run in parallel up to its configured limit.
|
||||
stop/down run the same stages in reverse order.
|
||||
Each before/after declaration creates an ordered stage.
|
||||
Stacks within a stage run in parallel up to its configured limit.
|
||||
ps/images/list keep ordered output.
|
||||
|
||||
Configuration: $CONFIG_FILE
|
||||
@@ -96,13 +116,13 @@ max_parallel() {
|
||||
DEFAULT_MAX_PARALLEL="$1"
|
||||
}
|
||||
|
||||
add_stage_group() {
|
||||
local groups_name="$1"
|
||||
add_stage() {
|
||||
local stages_name="$1"
|
||||
local limits_name="$2"
|
||||
local stage="$3"
|
||||
local limit=""
|
||||
local stack
|
||||
local -n groups="$groups_name"
|
||||
local -n stages="$stages_name"
|
||||
local -n limits="$limits_name"
|
||||
shift 3
|
||||
|
||||
@@ -124,12 +144,12 @@ add_stage_group() {
|
||||
CONFIGURED_STACKS+=("$stack")
|
||||
done
|
||||
|
||||
groups+=("$*")
|
||||
stages+=("$*")
|
||||
limits+=("$limit")
|
||||
}
|
||||
|
||||
before() { add_stage_group BEFORE_GROUPS BEFORE_LIMITS before "$@"; }
|
||||
after() { add_stage_group AFTER_GROUPS AFTER_LIMITS after "$@"; }
|
||||
before() { add_stage BEFORE_STAGES BEFORE_LIMITS before "$@"; }
|
||||
after() { add_stage AFTER_STAGES AFTER_LIMITS after "$@"; }
|
||||
|
||||
# Exclude stacks from all bulk operations.
|
||||
exclude() {
|
||||
@@ -184,36 +204,36 @@ validate_config_stacks() {
|
||||
done
|
||||
}
|
||||
|
||||
append_stage_groups() {
|
||||
append_stages() {
|
||||
local stage="$1"
|
||||
local groups_name="$2"
|
||||
local stages_name="$2"
|
||||
local limits_name="$3"
|
||||
local i name
|
||||
local -a stacks=()
|
||||
local -n groups="$groups_name"
|
||||
local -n stages="$stages_name"
|
||||
local -n limits="$limits_name"
|
||||
|
||||
for ((i = 0; i < ${#groups[@]}; i++)); do
|
||||
for ((i = 0; i < ${#stages[@]}; i++)); do
|
||||
name="$stage"
|
||||
((${#groups[@]} > 1)) && name+="-$((i + 1))"
|
||||
GROUP_NAMES+=("$name")
|
||||
GROUP_LIMITS+=("${limits[i]:-$DEFAULT_MAX_PARALLEL}")
|
||||
GROUP_STACKS+=("${groups[i]}")
|
||||
read -r -a stacks <<<"${groups[i]}"
|
||||
((${#stages[@]} > 1)) && name+="-$((i + 1))"
|
||||
STAGE_NAMES+=("$name")
|
||||
STAGE_LIMITS+=("${limits[i]:-$DEFAULT_MAX_PARALLEL}")
|
||||
STAGE_STACKS+=("${stages[i]}")
|
||||
read -r -a stacks <<<"${stages[i]}"
|
||||
MANAGED_STACKS+=("${stacks[@]}")
|
||||
done
|
||||
}
|
||||
|
||||
build_execution_groups() {
|
||||
build_execution_stages() {
|
||||
local stack
|
||||
local -a default_stacks=()
|
||||
|
||||
GROUP_NAMES=()
|
||||
GROUP_LIMITS=()
|
||||
GROUP_STACKS=()
|
||||
STAGE_NAMES=()
|
||||
STAGE_LIMITS=()
|
||||
STAGE_STACKS=()
|
||||
MANAGED_STACKS=()
|
||||
|
||||
append_stage_groups before BEFORE_GROUPS BEFORE_LIMITS
|
||||
append_stages before BEFORE_STAGES BEFORE_LIMITS
|
||||
|
||||
for stack in "${DISCOVERED_STACKS[@]}"; do
|
||||
if ! contains "$stack" "${CONFIGURED_STACKS[@]}" &&
|
||||
@@ -223,24 +243,24 @@ build_execution_groups() {
|
||||
done
|
||||
|
||||
if ((${#default_stacks[@]} > 0)); then
|
||||
GROUP_NAMES+=("default")
|
||||
GROUP_LIMITS+=("$DEFAULT_MAX_PARALLEL")
|
||||
GROUP_STACKS+=("${default_stacks[*]}")
|
||||
STAGE_NAMES+=("default")
|
||||
STAGE_LIMITS+=("$DEFAULT_MAX_PARALLEL")
|
||||
STAGE_STACKS+=("${default_stacks[*]}")
|
||||
MANAGED_STACKS+=("${default_stacks[@]}")
|
||||
fi
|
||||
|
||||
append_stage_groups after AFTER_GROUPS AFTER_LIMITS
|
||||
append_stages after AFTER_STAGES AFTER_LIMITS
|
||||
}
|
||||
|
||||
stack_group() {
|
||||
stack_stage() {
|
||||
local target="$1"
|
||||
local i
|
||||
local -a stacks=()
|
||||
|
||||
for ((i = 0; i < ${#GROUP_NAMES[@]}; i++)); do
|
||||
read -r -a stacks <<<"${GROUP_STACKS[i]}"
|
||||
for ((i = 0; i < ${#STAGE_NAMES[@]}; i++)); do
|
||||
read -r -a stacks <<<"${STAGE_STACKS[i]}"
|
||||
if contains "$target" "${stacks[@]}"; then
|
||||
echo "${GROUP_NAMES[i]}"
|
||||
echo "${STAGE_NAMES[i]}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
@@ -289,11 +309,11 @@ run_stack() {
|
||||
esac
|
||||
}
|
||||
|
||||
run_group() {
|
||||
local group_index="$1"
|
||||
run_stage() {
|
||||
local stage_index="$1"
|
||||
local action="$2"
|
||||
local group_name="${GROUP_NAMES[group_index]}"
|
||||
local limit="${GROUP_LIMITS[group_index]}"
|
||||
local stage_name="${STAGE_NAMES[stage_index]}"
|
||||
local limit="${STAGE_LIMITS[stage_index]}"
|
||||
local stack pid finished_pid status success_result working_result details
|
||||
local index finished_index total
|
||||
local next=0 running=0 completed=0 failed=0 stack_width=0 interactive=0 rendered=0
|
||||
@@ -310,7 +330,7 @@ run_group() {
|
||||
fi
|
||||
fi
|
||||
|
||||
read -r -a stacks <<<"${GROUP_STACKS[group_index]}"
|
||||
read -r -a stacks <<<"${STAGE_STACKS[stage_index]}"
|
||||
total="${#stacks[@]}"
|
||||
((total > 0)) || return 0
|
||||
for ((index = 0; index < total; index++)); do
|
||||
@@ -335,7 +355,7 @@ run_group() {
|
||||
|
||||
echo
|
||||
if ((!interactive)); then
|
||||
echo "==> Group $group_name: $action ($details)"
|
||||
echo "==> Stage $stage_name: $action ($details)"
|
||||
fi
|
||||
|
||||
case "$action" in
|
||||
@@ -378,7 +398,7 @@ run_group() {
|
||||
if ((interactive)); then
|
||||
((rendered)) && printf '\033[%dA' "$((total + 1))"
|
||||
printf '\r\033[2K[+] %s: %s %d/%d\n' \
|
||||
"$group_name" "$action" "$completed" "$total"
|
||||
"$stage_name" "$action" "$completed" "$total"
|
||||
for ((index = 0; index < total; index++)); do
|
||||
printf '\r\033[2K %s%s%s Stack %-*s %s%s%s\n' \
|
||||
"${state_colors[index]}" "${markers[index]}" "$color_reset" \
|
||||
@@ -434,39 +454,39 @@ print_run_summary() {
|
||||
echo "Failed stacks: ${RUN_FAILED_STACKS[*]}"
|
||||
fi
|
||||
|
||||
if ((${#RUN_SKIPPED_GROUPS[@]} > 0)); then
|
||||
echo "Skipped groups: ${RUN_SKIPPED_GROUPS[*]}"
|
||||
if ((${#RUN_SKIPPED_STAGES[@]} > 0)); then
|
||||
echo "Skipped stages: ${RUN_SKIPPED_STAGES[*]}"
|
||||
fi
|
||||
}
|
||||
|
||||
collect_skipped_groups() {
|
||||
collect_skipped_stages() {
|
||||
local failed_index="$1"
|
||||
local direction="$2"
|
||||
local i
|
||||
|
||||
RUN_SKIPPED_GROUPS=()
|
||||
RUN_SKIPPED_STAGES=()
|
||||
|
||||
if [[ "$direction" == "reverse" ]]; then
|
||||
for ((i = failed_index - 1; i >= 0; i--)); do
|
||||
RUN_SKIPPED_GROUPS+=("${GROUP_NAMES[i]}")
|
||||
RUN_SKIPPED_STAGES+=("${STAGE_NAMES[i]}")
|
||||
done
|
||||
else
|
||||
for ((i = failed_index + 1; i < ${#GROUP_NAMES[@]}; i++)); do
|
||||
RUN_SKIPPED_GROUPS+=("${GROUP_NAMES[i]}")
|
||||
for ((i = failed_index + 1; i < ${#STAGE_NAMES[@]}; i++)); do
|
||||
RUN_SKIPPED_STAGES+=("${STAGE_NAMES[i]}")
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
run_all_grouped() {
|
||||
run_all_staged() {
|
||||
local action="$1"
|
||||
local direction="forward"
|
||||
local start=0 end="${#GROUP_NAMES[@]}" step=1
|
||||
local start=0 end="${#STAGE_NAMES[@]}" step=1
|
||||
local i
|
||||
|
||||
RUN_SUCCESS=0
|
||||
RUN_FAILED=0
|
||||
RUN_FAILED_STACKS=()
|
||||
RUN_SKIPPED_GROUPS=()
|
||||
RUN_SKIPPED_STAGES=()
|
||||
if [[ "$action" == "stop" || "$action" == "down" ]]; then
|
||||
direction="reverse"
|
||||
start=$((end - 1))
|
||||
@@ -475,8 +495,8 @@ run_all_grouped() {
|
||||
fi
|
||||
|
||||
for ((i = start; i != end; i += step)); do
|
||||
if ! run_group "$i" "$action"; then
|
||||
collect_skipped_groups "$i" "$direction"
|
||||
if ! run_stage "$i" "$action"; then
|
||||
collect_skipped_stages "$i" "$direction"
|
||||
print_run_summary
|
||||
return 1
|
||||
fi
|
||||
@@ -501,21 +521,21 @@ run_all_sequential() {
|
||||
}
|
||||
|
||||
list_stacks() {
|
||||
local stack group_name status
|
||||
local stack stage_name status
|
||||
|
||||
printf '%-24s %-16s %s\n' "STACK" "GROUP" "STATUS"
|
||||
printf '%-24s %-16s %s\n' "STACK" "STAGE" "STATUS"
|
||||
printf '%-24s %-16s %s\n' "------------------------" "----------------" "---------------"
|
||||
|
||||
for stack in "${DISCOVERED_STACKS[@]}"; do
|
||||
if contains "$stack" "${EXCLUDED_STACKS[@]}"; then
|
||||
group_name="-"
|
||||
stage_name="-"
|
||||
status="excluded"
|
||||
else
|
||||
group_name="$(stack_group "$stack")"
|
||||
stage_name="$(stack_stage "$stack")"
|
||||
status="managed"
|
||||
fi
|
||||
|
||||
printf '%-24s %-16s %s\n' "$stack" "$group_name" "$status"
|
||||
printf '%-24s %-16s %s\n' "$stack" "$stage_name" "$status"
|
||||
done
|
||||
}
|
||||
|
||||
@@ -573,7 +593,7 @@ acquire_lock() {
|
||||
load_config
|
||||
discover_stacks
|
||||
validate_config_stacks
|
||||
build_execution_groups
|
||||
build_execution_stages
|
||||
|
||||
ACTION="${1:-}"
|
||||
TARGET="${2:-}"
|
||||
@@ -588,7 +608,7 @@ up | start | stop | restart | down | pull | update)
|
||||
require_target "$ACTION" "$TARGET"
|
||||
run_stack "$TARGET" "$ACTION"
|
||||
else
|
||||
run_all_grouped "$ACTION"
|
||||
run_all_staged "$ACTION"
|
||||
fi
|
||||
;;
|
||||
ps)
|
||||
|
||||
Reference in new issue
Block a user