feat: implement group management and parallel execution in manage.sh
This commit is contained in:
lingkai5wu committed
1 parent
db0ccd0c70
commit
5e20bc70af
2 files changed
+375
-67
No files matched your search
+26
@@ -0,0 +1,26 @@
|
||||
# Global maximum number of stacks executed concurrently within a group.
|
||||
# Use 0 for unlimited concurrency.
|
||||
max_parallel 2
|
||||
|
||||
# Groups run from top to bottom for up/start/restart/pull/update.
|
||||
# They run from bottom to top for stop/down.
|
||||
group gateway --parallel 1 \
|
||||
nginx
|
||||
|
||||
group network \
|
||||
adguardhome \
|
||||
ddns-go
|
||||
|
||||
group services \
|
||||
samba \
|
||||
immich \
|
||||
gitea \
|
||||
openlist \
|
||||
homeassistant \
|
||||
qbittorrent
|
||||
|
||||
group backup --parallel 1 \
|
||||
backrest
|
||||
|
||||
# Excluded stacks are ignored by all bulk commands.
|
||||
# exclude experimental-stack
|
||||
@@ -8,23 +8,30 @@ STACKS_DIR="${STACKS_DIR:-$SCRIPT_DIR}"
|
||||
CONFIG_FILE="${STACKS_CONFIG:-$STACKS_DIR/manage.conf}"
|
||||
LOCK_FILE="${STACKS_LOCK_FILE:-/run/lock/stacks-manage.lock}"
|
||||
|
||||
WHITELIST=()
|
||||
BLACKLIST=()
|
||||
PRIORITY=()
|
||||
DEFAULT_MAX_PARALLEL=2
|
||||
|
||||
CONFIG_GROUP_NAMES=()
|
||||
CONFIG_GROUP_LIMITS=()
|
||||
CONFIG_GROUP_STACKS=()
|
||||
CONFIGURED_STACKS=()
|
||||
EXCLUDED_STACKS=()
|
||||
|
||||
DISCOVERED_STACKS=()
|
||||
MANAGED_STACKS=()
|
||||
GROUP_NAMES=()
|
||||
GROUP_LIMITS=()
|
||||
GROUP_STACKS=()
|
||||
UNASSIGNED_STACKS=()
|
||||
|
||||
RUN_SUCCESS=0
|
||||
RUN_FAILED=0
|
||||
RUN_FAILED_STACKS=()
|
||||
|
||||
die() {
|
||||
echo "ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ -f "$CONFIG_FILE" ]]; then
|
||||
bash -n "$CONFIG_FILE" || die "Syntax error in config file: $CONFIG_FILE"
|
||||
source "$CONFIG_FILE"
|
||||
fi
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $(basename "$0") COMMAND [STACK]
|
||||
@@ -41,7 +48,15 @@ Commands:
|
||||
logs Follow logs for a stack
|
||||
config Render Compose configuration for a stack
|
||||
images Show configured images and versions
|
||||
list List discovered stacks and their status
|
||||
list List discovered stacks and their group/status
|
||||
|
||||
Bulk execution:
|
||||
up/start/restart/pull/update run groups in declaration order.
|
||||
stop/down run groups in reverse order.
|
||||
Stacks within a group run in parallel up to that group's limit.
|
||||
ps/images/list keep ordered output.
|
||||
|
||||
Configuration: $CONFIG_FILE
|
||||
EOF
|
||||
}
|
||||
|
||||
@@ -57,6 +72,97 @@ contains() {
|
||||
return 1
|
||||
}
|
||||
|
||||
validate_parallel_limit() {
|
||||
local value="$1"
|
||||
|
||||
[[ "$value" =~ ^[0-9]+$ ]] ||
|
||||
die "parallel limit must be a non-negative integer: $value"
|
||||
}
|
||||
|
||||
validate_name() {
|
||||
local kind="$1"
|
||||
local value="$2"
|
||||
|
||||
[[ "$value" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]] ||
|
||||
die "invalid $kind name: $value"
|
||||
}
|
||||
|
||||
# manage.conf DSL: set the default concurrency for groups without an override.
|
||||
max_parallel() {
|
||||
[[ $# -eq 1 ]] || die "max_parallel expects exactly one value"
|
||||
validate_parallel_limit "$1"
|
||||
DEFAULT_MAX_PARALLEL="$1"
|
||||
}
|
||||
|
||||
# manage.conf DSL: declare an ordered group and its stacks.
|
||||
# Usage: group NAME [--parallel N] STACK...
|
||||
group() {
|
||||
[[ $# -ge 2 ]] || die "group expects NAME [--parallel N] STACK..."
|
||||
|
||||
local name="$1"
|
||||
local limit=""
|
||||
local stack
|
||||
shift
|
||||
|
||||
validate_name "group" "$name"
|
||||
[[ "$name" != "unassigned" ]] ||
|
||||
die "group name is reserved: $name"
|
||||
contains "$name" "${CONFIG_GROUP_NAMES[@]}" &&
|
||||
die "group is declared more than once: $name"
|
||||
|
||||
if [[ "${1:-}" == "--parallel" ]]; then
|
||||
[[ $# -ge 3 ]] || die "group $name: --parallel expects a value and at least one stack"
|
||||
limit="$2"
|
||||
validate_parallel_limit "$limit"
|
||||
shift 2
|
||||
fi
|
||||
|
||||
[[ $# -ge 1 ]] || die "group has no stacks: $name"
|
||||
|
||||
for stack in "$@"; do
|
||||
validate_name "stack" "$stack"
|
||||
contains "$stack" "${CONFIGURED_STACKS[@]}" &&
|
||||
die "stack is assigned to more than one group: $stack"
|
||||
contains "$stack" "${EXCLUDED_STACKS[@]}" &&
|
||||
die "stack is both grouped and excluded: $stack"
|
||||
CONFIGURED_STACKS+=("$stack")
|
||||
done
|
||||
|
||||
CONFIG_GROUP_NAMES+=("$name")
|
||||
CONFIG_GROUP_LIMITS+=("$limit")
|
||||
CONFIG_GROUP_STACKS+=("$*")
|
||||
}
|
||||
|
||||
# manage.conf DSL: exclude stacks from all bulk operations.
|
||||
exclude() {
|
||||
[[ $# -ge 1 ]] || die "exclude expects at least one stack"
|
||||
|
||||
local stack
|
||||
for stack in "$@"; do
|
||||
validate_name "stack" "$stack"
|
||||
contains "$stack" "${EXCLUDED_STACKS[@]}" &&
|
||||
die "stack is excluded more than once: $stack"
|
||||
contains "$stack" "${CONFIGURED_STACKS[@]}" &&
|
||||
die "stack is both grouped and excluded: $stack"
|
||||
EXCLUDED_STACKS+=("$stack")
|
||||
done
|
||||
}
|
||||
|
||||
load_config() {
|
||||
[[ -f "$CONFIG_FILE" ]] || return 0
|
||||
|
||||
bash -n "$CONFIG_FILE" || die "syntax error in config file: $CONFIG_FILE"
|
||||
# This is a trusted Bash configuration file that calls the DSL above.
|
||||
source "$CONFIG_FILE"
|
||||
|
||||
local legacy
|
||||
for legacy in WHITELIST BLACKLIST PRIORITY; do
|
||||
if declare -p "$legacy" >/dev/null 2>&1; then
|
||||
die "$legacy is no longer supported; migrate $CONFIG_FILE to group/exclude directives"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
discover_stacks() {
|
||||
local compose_file stack
|
||||
|
||||
@@ -73,30 +179,76 @@ is_discovered() {
|
||||
}
|
||||
|
||||
is_managed() {
|
||||
local stack="$1"
|
||||
|
||||
is_discovered "$stack" || return 1
|
||||
[[ ! -e "$STACKS_DIR/$stack/.disabled" ]] || return 1
|
||||
contains "$stack" "${BLACKLIST[@]}" && return 1
|
||||
((${#WHITELIST[@]} == 0)) || contains "$stack" "${WHITELIST[@]}"
|
||||
is_discovered "$1" && ! contains "$1" "${EXCLUDED_STACKS[@]}"
|
||||
}
|
||||
|
||||
build_managed_stacks() {
|
||||
validate_config_stacks() {
|
||||
local stack
|
||||
|
||||
MANAGED_STACKS=()
|
||||
for stack in "${CONFIGURED_STACKS[@]}"; do
|
||||
is_discovered "$stack" || die "configured stack not found: $stack"
|
||||
done
|
||||
|
||||
for stack in "${PRIORITY[@]}"; do
|
||||
if is_managed "$stack" && ! contains "$stack" "${MANAGED_STACKS[@]}"; then
|
||||
MANAGED_STACKS+=("$stack")
|
||||
fi
|
||||
for stack in "${EXCLUDED_STACKS[@]}"; do
|
||||
is_discovered "$stack" || die "excluded stack not found: $stack"
|
||||
done
|
||||
}
|
||||
|
||||
build_execution_groups() {
|
||||
local i stack limit
|
||||
local -a stacks=()
|
||||
|
||||
GROUP_NAMES=()
|
||||
GROUP_LIMITS=()
|
||||
GROUP_STACKS=()
|
||||
MANAGED_STACKS=()
|
||||
UNASSIGNED_STACKS=()
|
||||
|
||||
for ((i = 0; i < ${#CONFIG_GROUP_NAMES[@]}; i++)); do
|
||||
limit="${CONFIG_GROUP_LIMITS[i]:-$DEFAULT_MAX_PARALLEL}"
|
||||
GROUP_NAMES+=("${CONFIG_GROUP_NAMES[i]}")
|
||||
GROUP_LIMITS+=("$limit")
|
||||
GROUP_STACKS+=("${CONFIG_GROUP_STACKS[i]}")
|
||||
|
||||
read -r -a stacks <<<"${CONFIG_GROUP_STACKS[i]}"
|
||||
MANAGED_STACKS+=("${stacks[@]}")
|
||||
done
|
||||
|
||||
for stack in "${DISCOVERED_STACKS[@]}"; do
|
||||
if is_managed "$stack" && ! contains "$stack" "${MANAGED_STACKS[@]}"; then
|
||||
if ! contains "$stack" "${CONFIGURED_STACKS[@]}" &&
|
||||
! contains "$stack" "${EXCLUDED_STACKS[@]}"; then
|
||||
UNASSIGNED_STACKS+=("$stack")
|
||||
MANAGED_STACKS+=("$stack")
|
||||
fi
|
||||
done
|
||||
|
||||
if ((${#UNASSIGNED_STACKS[@]} > 0)); then
|
||||
GROUP_NAMES+=("unassigned")
|
||||
GROUP_LIMITS+=("$DEFAULT_MAX_PARALLEL")
|
||||
GROUP_STACKS+=("${UNASSIGNED_STACKS[*]}")
|
||||
fi
|
||||
}
|
||||
|
||||
warn_unassigned() {
|
||||
((${#UNASSIGNED_STACKS[@]} == 0)) && return 0
|
||||
|
||||
echo "WARNING: unassigned stacks will run in the final implicit group: ${UNASSIGNED_STACKS[*]}" >&2
|
||||
}
|
||||
|
||||
stack_group() {
|
||||
local target="$1"
|
||||
local i
|
||||
local -a stacks=()
|
||||
|
||||
for ((i = 0; i < ${#GROUP_NAMES[@]}; i++)); do
|
||||
read -r -a stacks <<<"${GROUP_STACKS[i]}"
|
||||
if contains "$target" "${stacks[@]}"; then
|
||||
echo "${GROUP_NAMES[i]}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
compose() {
|
||||
@@ -132,59 +284,186 @@ run_stack() {
|
||||
esac
|
||||
}
|
||||
|
||||
run_all() {
|
||||
local action="$1"
|
||||
local success=0 failed=0 stack i
|
||||
run_stack_prefixed() {
|
||||
local stack="$1"
|
||||
local action="$2"
|
||||
local line status
|
||||
|
||||
if [[ "$action" == "stop" || "$action" == "down" ]]; then
|
||||
for ((i = ${#MANAGED_STACKS[@]} - 1; i >= 0; i--)); do
|
||||
stack="${MANAGED_STACKS[i]}"
|
||||
run_stack "$stack" "$action" 2>&1 |
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
printf '[%s] %s\n' "$stack" "$line"
|
||||
done
|
||||
status="${PIPESTATUS[0]}"
|
||||
|
||||
if run_stack "$stack" "$action"; then
|
||||
((success += 1))
|
||||
else
|
||||
((failed += 1))
|
||||
fi
|
||||
done
|
||||
else
|
||||
for stack in "${MANAGED_STACKS[@]}"; do
|
||||
if run_stack "$stack" "$action"; then
|
||||
((success += 1))
|
||||
else
|
||||
((failed += 1))
|
||||
fi
|
||||
done
|
||||
return "$status"
|
||||
}
|
||||
|
||||
run_group() {
|
||||
local group_index="$1"
|
||||
local action="$2"
|
||||
local group_name="${GROUP_NAMES[group_index]}"
|
||||
local limit="${GROUP_LIMITS[group_index]}"
|
||||
local semaphore_dir semaphore_fifo semaphore_fd
|
||||
local stack pid i status
|
||||
local failed=0
|
||||
local -a stacks=() pids=() pid_stacks=()
|
||||
|
||||
read -r -a stacks <<<"${GROUP_STACKS[group_index]}"
|
||||
((${#stacks[@]} > 0)) || return 0
|
||||
|
||||
if ((limit == 0 || limit > ${#stacks[@]})); then
|
||||
limit="${#stacks[@]}"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Succeeded: $success"
|
||||
echo "Failed: $failed"
|
||||
echo "==> Group $group_name: $action (${#stacks[@]} stacks, parallel=$limit)"
|
||||
|
||||
semaphore_dir="$(mktemp -d "${TMPDIR:-/tmp}/stacks-manage.XXXXXX")" ||
|
||||
die "cannot create semaphore directory"
|
||||
semaphore_fifo="$semaphore_dir/tokens"
|
||||
mkfifo "$semaphore_fifo" || die "cannot create semaphore"
|
||||
exec {semaphore_fd}<>"$semaphore_fifo" || die "cannot open semaphore"
|
||||
|
||||
for ((i = 0; i < limit; i++)); do
|
||||
printf '\n' >&"$semaphore_fd"
|
||||
done
|
||||
|
||||
for stack in "${stacks[@]}"; do
|
||||
IFS= read -r -u "$semaphore_fd"
|
||||
|
||||
(
|
||||
status=0
|
||||
run_stack_prefixed "$stack" "$action" || status=$?
|
||||
printf '\n' >&"$semaphore_fd"
|
||||
exit "$status"
|
||||
) &
|
||||
|
||||
pids+=("$!")
|
||||
pid_stacks+=("$stack")
|
||||
done
|
||||
|
||||
for ((i = 0; i < ${#pids[@]}; i++)); do
|
||||
pid="${pids[i]}"
|
||||
stack="${pid_stacks[i]}"
|
||||
|
||||
if wait "$pid"; then
|
||||
((RUN_SUCCESS += 1))
|
||||
else
|
||||
((RUN_FAILED += 1))
|
||||
RUN_FAILED_STACKS+=("$stack")
|
||||
((failed += 1))
|
||||
fi
|
||||
done
|
||||
|
||||
exec {semaphore_fd}>&-
|
||||
rm -f -- "$semaphore_fifo"
|
||||
rmdir -- "$semaphore_dir"
|
||||
|
||||
((failed == 0))
|
||||
}
|
||||
|
||||
stack_status() {
|
||||
local stack="$1"
|
||||
print_run_summary() {
|
||||
echo
|
||||
echo "Succeeded: $RUN_SUCCESS"
|
||||
echo "Failed: $RUN_FAILED"
|
||||
|
||||
if [[ -e "$STACKS_DIR/$stack/.disabled" ]]; then
|
||||
echo "disabled"
|
||||
elif contains "$stack" "${BLACKLIST[@]}"; then
|
||||
echo "blacklisted"
|
||||
elif ((${#WHITELIST[@]} > 0)) && ! contains "$stack" "${WHITELIST[@]}"; then
|
||||
echo "not whitelisted"
|
||||
else
|
||||
echo "managed"
|
||||
if ((${#RUN_FAILED_STACKS[@]} > 0)); then
|
||||
echo "Failures: ${RUN_FAILED_STACKS[*]}"
|
||||
fi
|
||||
}
|
||||
|
||||
list_stacks() {
|
||||
print_skipped_groups() {
|
||||
local failed_index="$1"
|
||||
local direction="$2"
|
||||
local i
|
||||
local -a skipped=()
|
||||
|
||||
if [[ "$direction" == "reverse" ]]; then
|
||||
for ((i = failed_index - 1; i >= 0; i--)); do
|
||||
skipped+=("${GROUP_NAMES[i]}")
|
||||
done
|
||||
else
|
||||
for ((i = failed_index + 1; i < ${#GROUP_NAMES[@]}; i++)); do
|
||||
skipped+=("${GROUP_NAMES[i]}")
|
||||
done
|
||||
fi
|
||||
|
||||
if ((${#skipped[@]} > 0)); then
|
||||
echo "Skipped groups after failure: ${skipped[*]}" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
run_all_grouped() {
|
||||
local action="$1"
|
||||
local direction="forward"
|
||||
local i
|
||||
|
||||
RUN_SUCCESS=0
|
||||
RUN_FAILED=0
|
||||
RUN_FAILED_STACKS=()
|
||||
warn_unassigned
|
||||
|
||||
if [[ "$action" == "stop" || "$action" == "down" ]]; then
|
||||
direction="reverse"
|
||||
for ((i = ${#GROUP_NAMES[@]} - 1; i >= 0; i--)); do
|
||||
if ! run_group "$i" "$action"; then
|
||||
print_skipped_groups "$i" "$direction"
|
||||
print_run_summary
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
else
|
||||
for ((i = 0; i < ${#GROUP_NAMES[@]}; i++)); do
|
||||
if ! run_group "$i" "$action"; then
|
||||
print_skipped_groups "$i" "$direction"
|
||||
print_run_summary
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
print_run_summary
|
||||
}
|
||||
|
||||
run_all_sequential() {
|
||||
local action="$1"
|
||||
local stack
|
||||
|
||||
printf '%-24s %s\n' "STACK" "STATUS"
|
||||
printf '%-24s %s\n' "------------------------" "---------------"
|
||||
RUN_SUCCESS=0
|
||||
RUN_FAILED=0
|
||||
RUN_FAILED_STACKS=()
|
||||
warn_unassigned
|
||||
|
||||
for stack in "${MANAGED_STACKS[@]}"; do
|
||||
if run_stack "$stack" "$action"; then
|
||||
((RUN_SUCCESS += 1))
|
||||
else
|
||||
((RUN_FAILED += 1))
|
||||
RUN_FAILED_STACKS+=("$stack")
|
||||
fi
|
||||
done
|
||||
|
||||
print_run_summary
|
||||
((RUN_FAILED == 0))
|
||||
}
|
||||
|
||||
list_stacks() {
|
||||
local stack group_name status
|
||||
|
||||
warn_unassigned
|
||||
printf '%-24s %-16s %s\n' "STACK" "GROUP" "STATUS"
|
||||
printf '%-24s %-16s %s\n' "------------------------" "----------------" "---------------"
|
||||
|
||||
for stack in "${DISCOVERED_STACKS[@]}"; do
|
||||
printf '%-24s %s\n' "$stack" "$(stack_status "$stack")"
|
||||
if contains "$stack" "${EXCLUDED_STACKS[@]}"; then
|
||||
group_name="-"
|
||||
status="excluded"
|
||||
else
|
||||
group_name="$(stack_group "$stack")"
|
||||
status="managed"
|
||||
fi
|
||||
|
||||
printf '%-24s %-16s %s\n' "$stack" "$group_name" "$status"
|
||||
done
|
||||
}
|
||||
|
||||
@@ -214,6 +493,7 @@ list_images() {
|
||||
return
|
||||
fi
|
||||
|
||||
warn_unassigned
|
||||
for stack in "${MANAGED_STACKS[@]}"; do
|
||||
stack_images "$stack" || ((failed += 1))
|
||||
done
|
||||
@@ -226,26 +506,28 @@ require_target() {
|
||||
local target="$2"
|
||||
|
||||
[[ -n "$target" ]] || die "$command requires a stack name"
|
||||
is_discovered "$target" || die "Stack not found: $target"
|
||||
is_managed "$target" || die "Stack is excluded: $target"
|
||||
is_discovered "$target" || die "stack not found: $target"
|
||||
is_managed "$target" || die "stack is excluded: $target"
|
||||
}
|
||||
|
||||
acquire_lock() {
|
||||
mkdir -p "$(dirname "$LOCK_FILE")" ||
|
||||
die "Cannot create lock directory"
|
||||
die "cannot create lock directory"
|
||||
exec {LOCK_FD}>"$LOCK_FILE" ||
|
||||
die "Cannot open lock file: $LOCK_FILE"
|
||||
die "cannot open lock file: $LOCK_FILE"
|
||||
flock -n "$LOCK_FD" ||
|
||||
die "Another manage.sh operation is already running"
|
||||
die "another manage.sh operation is already running"
|
||||
}
|
||||
|
||||
load_config
|
||||
discover_stacks
|
||||
build_managed_stacks
|
||||
validate_config_stacks
|
||||
build_execution_groups
|
||||
|
||||
ACTION="${1:-}"
|
||||
TARGET="${2:-}"
|
||||
|
||||
[[ $# -le 2 ]] || die "Too many arguments"
|
||||
[[ $# -le 2 ]] || die "too many arguments"
|
||||
|
||||
case "$ACTION" in
|
||||
up | start | stop | restart | down | pull | update)
|
||||
@@ -255,7 +537,7 @@ up | start | stop | restart | down | pull | update)
|
||||
require_target "$ACTION" "$TARGET"
|
||||
run_stack "$TARGET" "$ACTION"
|
||||
else
|
||||
run_all "$ACTION"
|
||||
run_all_grouped "$ACTION"
|
||||
fi
|
||||
;;
|
||||
ps)
|
||||
@@ -263,7 +545,7 @@ ps)
|
||||
require_target "$ACTION" "$TARGET"
|
||||
run_stack "$TARGET" "$ACTION"
|
||||
else
|
||||
run_all "$ACTION"
|
||||
run_all_sequential "$ACTION"
|
||||
fi
|
||||
;;
|
||||
logs)
|
||||
|
||||
Reference in new issue
Block a user