mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-10-17 05:14:19 +02:00
fix(installer): check mount propagation and ports before installing
The installer created /opt/project-nomad, generated credentials, and started pulling images before it could tell whether the host was able to run the management stack at all. An incompatible root mount propagation or an already occupied published port surfaced only as an opaque Compose failure, after the installer had already written to the host and after the user had answered every prompt. Add a side-effect-free host preflight that runs after the environment checks and immediately before create_nomad_directory. It derives the propagation-sensitive host mounts and every published host port from the management Compose file instead of hardcoding 8080 and 9999, inspects them with findmnt and ss, reports every failed check together with its remediation, and exits nonzero before any installation resource exists. The Compose file fetched for the preflight is reused for the installation rather than downloaded a second time. Wrap the top-level call sequence in a guarded main so the script can be sourced by tests without performing an installation. Add install/tests/install_nomad_preflight_test.sh, a dependency-free harness covering Compose parsing, shared, slave, private and undetectable propagation, free, occupied and undetectable ports, the existing-install port warning, call order, Compose reuse, and failure before the installation directory is created. Refs #1320 Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Robert Cronk
parent
a483e15f7d
commit
349042eedc
|
|
@ -37,6 +37,7 @@ UPDATE_SCRIPT_URL="https://raw.githubusercontent.com/Crosstalk-Solutions/project
|
|||
script_option_debug='true'
|
||||
accepted_terms='false'
|
||||
local_ip_address=''
|
||||
preflight_compose_file=''
|
||||
|
||||
###################################################################################################################################################################################################
|
||||
# #
|
||||
|
|
@ -425,9 +426,17 @@ create_nomad_directory(){
|
|||
|
||||
download_management_compose_file() {
|
||||
local compose_file_path="${NOMAD_DIR}/compose.yml"
|
||||
local source_file="${1:-}"
|
||||
|
||||
echo -e "${YELLOW}#${RESET} Downloading docker-compose file for management...\\n"
|
||||
if ! curl -fsSL "$MANAGEMENT_COMPOSE_FILE_URL" -o "$compose_file_path"; then
|
||||
if [[ -n "$source_file" ]]; then
|
||||
# Redirection rather than cp so the file keeps the umask-derived mode that
|
||||
# the curl path below produces; cp would copy the temp file's 0600 instead.
|
||||
if ! cat "$source_file" > "$compose_file_path"; then
|
||||
echo -e "${RED}#${RESET} Failed to copy the preflighted docker compose file."
|
||||
exit 1
|
||||
fi
|
||||
elif ! curl -fsSL "$MANAGEMENT_COMPOSE_FILE_URL" -o "$compose_file_path"; then
|
||||
echo -e "${RED}#${RESET} Failed to download the docker compose file. Please check the URL and try again."
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -501,6 +510,120 @@ get_local_ip() {
|
|||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Prints "<port> <protocol>" for every host port published by the compose file.
|
||||
# Ranges such as "6333-6334:6333-6334" are expanded so each host port is checked.
|
||||
get_published_management_ports() {
|
||||
local compose_file="$1"
|
||||
|
||||
awk '
|
||||
/^[[:space:]]+ports:[[:space:]]*$/ { in_ports = 1; next }
|
||||
in_ports && /^[[:space:]]+-[[:space:]]*/ {
|
||||
value = $0
|
||||
sub(/^[[:space:]]+-[[:space:]]*"?/, "", value)
|
||||
sub(/"?[[:space:]]*(#.*)?$/, "", value)
|
||||
|
||||
protocol = "tcp"
|
||||
if (value ~ /\//) {
|
||||
protocol = value
|
||||
sub(/^.*\//, "", protocol)
|
||||
sub(/\/.*$/, "", value)
|
||||
}
|
||||
|
||||
count = split(value, parts, ":")
|
||||
if (count < 2) next
|
||||
host_ports = parts[count - 1]
|
||||
|
||||
if (host_ports ~ /^[0-9]+-[0-9]+$/) {
|
||||
split(host_ports, bounds, "-")
|
||||
for (port = bounds[1]; port <= bounds[2]; port++) print port, protocol
|
||||
} else if (host_ports ~ /^[0-9]+$/) {
|
||||
print host_ports, protocol
|
||||
}
|
||||
next
|
||||
}
|
||||
in_ports && /^[[:space:]]+[[:alnum:]_-]+:/ { in_ports = 0 }
|
||||
' "$compose_file" | sort -u -k1,1n -k2,2
|
||||
}
|
||||
|
||||
get_propagation_mount_paths() {
|
||||
local compose_file="$1"
|
||||
|
||||
awk '
|
||||
/:[^#]*:(ro,)?r?(shared|slave)(,|[[:space:]#]|$)/ {
|
||||
value = $0
|
||||
sub(/^[[:space:]]+-[[:space:]]*"?/, "", value)
|
||||
sub(/"?[[:space:]]*(#.*)?$/, "", value)
|
||||
sub(/:[^:]*:(ro,)?r?(shared|slave).*/, "", value)
|
||||
# Only host paths have mount propagation; a named volume is not inspectable.
|
||||
if (value ~ /^[\/.~]/) print value
|
||||
}
|
||||
' "$compose_file" | sort -u
|
||||
}
|
||||
|
||||
mount_propagation_for_path() {
|
||||
findmnt -no PROPAGATION --target "$1" 2>/dev/null
|
||||
}
|
||||
|
||||
is_port_in_use() {
|
||||
local port="$1"
|
||||
local protocol="${2:-tcp}"
|
||||
local family='-t'
|
||||
|
||||
[[ "$protocol" == "udp" ]] && family='-u'
|
||||
ss -H -ln "$family" "sport = :${port}" 2>/dev/null | grep -q .
|
||||
}
|
||||
|
||||
run_host_preflight() {
|
||||
local compose_file="$1"
|
||||
local mount_path propagation port protocol
|
||||
local failed=false
|
||||
local port_conflict=false
|
||||
|
||||
while IFS= read -r mount_path; do
|
||||
[[ -z "$mount_path" ]] && continue
|
||||
if ! propagation=$(mount_propagation_for_path "$mount_path"); then
|
||||
echo -e "${RED}#${RESET} Unable to inspect mount propagation for ${mount_path}. Ensure findmnt is installed and the path exists."
|
||||
failed=true
|
||||
elif [[ "$propagation" != shared* && "$propagation" != slave* ]]; then
|
||||
echo -e "${RED}#${RESET} Mount ${mount_path} uses '${propagation}' propagation; mount propagation must be shared or slave for the management containers."
|
||||
echo -e "${RED}#${RESET} Configure the host mount as shared, then rerun the installer."
|
||||
failed=true
|
||||
fi
|
||||
done < <(get_propagation_mount_paths "$compose_file")
|
||||
|
||||
if ! command -v ss &> /dev/null; then
|
||||
echo -e "${RED}#${RESET} Unable to inspect management ports because the 'ss' command is unavailable. Install iproute2 and rerun the installer."
|
||||
failed=true
|
||||
else
|
||||
while read -r port protocol; do
|
||||
[[ -z "$port" ]] && continue
|
||||
if is_port_in_use "$port" "$protocol"; then
|
||||
echo -e "${RED}#${RESET} Port ${port}/${protocol} is already in use. Stop the listening service or free the port, then rerun the installer."
|
||||
port_conflict=true
|
||||
failed=true
|
||||
fi
|
||||
done < <(get_published_management_ports "$compose_file")
|
||||
|
||||
if [[ "$port_conflict" == true ]]; then
|
||||
echo -e "${RED}#${RESET} If Project NOMAD is already installed on this host, these ports are expected to be in use. Rerunning the installer is not an update path and will reset the database."
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$failed" == true ]]; then
|
||||
echo -e "${RED}#${RESET} Project NOMAD host preflight failed before installation resources were created."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}#${RESET} Host mount propagation and management port checks passed.\\n"
|
||||
}
|
||||
|
||||
cleanup_preflight_compose_file() {
|
||||
if [[ -n "$preflight_compose_file" ]]; then
|
||||
rm -f "$preflight_compose_file"
|
||||
fi
|
||||
}
|
||||
|
||||
verify_gpu_setup() {
|
||||
# This function only displays GPU setup status and is completely non-blocking
|
||||
# It never exits or returns error codes - purely informational
|
||||
|
|
@ -630,28 +753,43 @@ success_message() {
|
|||
# #
|
||||
###################################################################################################################################################################################################
|
||||
|
||||
# Pre-flight checks
|
||||
check_is_debian_based
|
||||
check_is_x86_64
|
||||
check_is_bash
|
||||
check_has_sudo
|
||||
ensure_dependencies_installed
|
||||
check_is_debug_mode
|
||||
main() {
|
||||
# Pre-flight checks
|
||||
check_is_debian_based
|
||||
check_is_x86_64
|
||||
check_is_bash
|
||||
check_has_sudo
|
||||
ensure_dependencies_installed
|
||||
check_is_debug_mode
|
||||
|
||||
# Main install
|
||||
banner
|
||||
get_install_confirmation
|
||||
accept_terms
|
||||
ensure_docker_installed
|
||||
check_docker_compose
|
||||
setup_nvidia_container_toolkit
|
||||
get_local_ip
|
||||
create_nomad_directory
|
||||
download_helper_scripts
|
||||
download_management_compose_file
|
||||
start_management_containers
|
||||
verify_gpu_setup
|
||||
success_message
|
||||
# Main install
|
||||
banner
|
||||
get_install_confirmation
|
||||
accept_terms
|
||||
ensure_docker_installed
|
||||
check_docker_compose
|
||||
setup_nvidia_container_toolkit
|
||||
get_local_ip
|
||||
|
||||
preflight_compose_file=$(mktemp)
|
||||
trap cleanup_preflight_compose_file EXIT
|
||||
if ! curl -fsSL "$MANAGEMENT_COMPOSE_FILE_URL" -o "$preflight_compose_file"; then
|
||||
echo -e "${RED}#${RESET} Failed to download the management compose file for host preflight checks."
|
||||
exit 1
|
||||
fi
|
||||
run_host_preflight "$preflight_compose_file"
|
||||
|
||||
create_nomad_directory
|
||||
download_helper_scripts
|
||||
download_management_compose_file "$preflight_compose_file"
|
||||
start_management_containers
|
||||
verify_gpu_setup
|
||||
success_message
|
||||
}
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
||||
main "$@"
|
||||
fi
|
||||
|
||||
# free_space_check() {
|
||||
# if [[ "$(df -B1 / | awk 'NR==2{print $4}')" -le '5368709120' ]]; then
|
||||
|
|
|
|||
195
install/tests/install_nomad_preflight_test.sh
Normal file
195
install/tests/install_nomad_preflight_test.sh
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# install_nomad_preflight_test.sh
|
||||
#
|
||||
# Focused tests for the host preflight in install_nomad.sh: deriving the
|
||||
# propagation-sensitive mounts and published ports from the management Compose
|
||||
# file, inspecting them, and failing before any installation resource is
|
||||
# created. The installer is sourced, and its findmnt/ss wrappers are replaced
|
||||
# with stubs, so the tests need no root, Docker, or network.
|
||||
#
|
||||
# Usage: bash install/tests/install_nomad_preflight_test.sh
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - All assertions passed
|
||||
# 1 - One or more assertions failed (each is printed as "not ok - ...")
|
||||
|
||||
# shellcheck disable=SC2016,SC2329
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
INSTALLER="${SCRIPT_DIR}/../install_nomad.sh"
|
||||
COMPOSE_FILE="${SCRIPT_DIR}/../management_compose.yaml"
|
||||
failures=0
|
||||
|
||||
fail() {
|
||||
printf 'not ok - %s\n' "$1"
|
||||
failures=$((failures + 1))
|
||||
}
|
||||
|
||||
pass() {
|
||||
printf 'ok - %s\n' "$1"
|
||||
}
|
||||
|
||||
assert_success() {
|
||||
local description="$1"
|
||||
shift
|
||||
if ( "$@" ) >/tmp/nomad-preflight-test.out 2>&1; then
|
||||
pass "$description"
|
||||
else
|
||||
fail "$description"
|
||||
cat /tmp/nomad-preflight-test.out
|
||||
fi
|
||||
}
|
||||
|
||||
assert_failure_with() {
|
||||
local description="$1"
|
||||
local expected="$2"
|
||||
shift 2
|
||||
if ( "$@" ) >/tmp/nomad-preflight-test.out 2>&1; then
|
||||
fail "$description"
|
||||
elif grep -Fq "$expected" /tmp/nomad-preflight-test.out; then
|
||||
pass "$description"
|
||||
else
|
||||
fail "$description"
|
||||
cat /tmp/nomad-preflight-test.out
|
||||
fi
|
||||
}
|
||||
|
||||
if ! grep -Fq '[[ "${BASH_SOURCE[0]}" == "$0" ]]' "$INSTALLER"; then
|
||||
fail "installer can be sourced without running main"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# shellcheck source=install/install_nomad.sh
|
||||
source "$INSTALLER"
|
||||
|
||||
mapfile -t published_ports < <(get_published_management_ports "$COMPOSE_FILE")
|
||||
if [[ "${published_ports[*]}" == "8080 tcp 9999 tcp" ]]; then
|
||||
pass "derives every published management port"
|
||||
else
|
||||
fail "derives every published management port"
|
||||
fi
|
||||
|
||||
edge_compose=$(mktemp)
|
||||
cat > "$edge_compose" <<'YAML'
|
||||
services:
|
||||
a:
|
||||
ports:
|
||||
- "6333-6334:6333-6334"
|
||||
- "53:53/udp"
|
||||
- "127.0.0.1:7000:7000"
|
||||
- "3000"
|
||||
volumes:
|
||||
- named_vol:/data:rshared
|
||||
- /srv/data:/host/data:rslave
|
||||
YAML
|
||||
|
||||
mapfile -t edge_ports < <(get_published_management_ports "$edge_compose")
|
||||
if [[ "${edge_ports[*]}" == "53 udp 6333 tcp 6334 tcp 7000 tcp" ]]; then
|
||||
pass "expands port ranges and keeps the published protocol"
|
||||
else
|
||||
fail "expands port ranges and keeps the published protocol"
|
||||
printf ' got: %s\n' "${edge_ports[*]}"
|
||||
fi
|
||||
|
||||
mapfile -t edge_mounts < <(get_propagation_mount_paths "$edge_compose")
|
||||
if [[ "${edge_mounts[*]}" == "/srv/data" ]]; then
|
||||
pass "ignores named volumes when deriving propagation mounts"
|
||||
else
|
||||
fail "ignores named volumes when deriving propagation mounts"
|
||||
printf ' got: %s\n' "${edge_mounts[*]}"
|
||||
fi
|
||||
|
||||
mapfile -t propagation_mounts < <(get_propagation_mount_paths "$COMPOSE_FILE")
|
||||
if [[ "${propagation_mounts[*]}" == "/" ]]; then
|
||||
pass "derives every propagation-sensitive host mount"
|
||||
else
|
||||
fail "derives every propagation-sensitive host mount"
|
||||
fi
|
||||
|
||||
mount_propagation_for_path() { printf '%s\n' shared; }
|
||||
is_port_in_use() { return 1; }
|
||||
assert_success "compatible mounts and free ports pass" run_host_preflight "$COMPOSE_FILE"
|
||||
|
||||
mount_propagation_for_path() { printf '%s\n' slave; }
|
||||
assert_success "slave propagation passes" run_host_preflight "$COMPOSE_FILE"
|
||||
|
||||
mount_propagation_for_path() { printf '%s\n' private; }
|
||||
assert_failure_with "private mount reports remediation" "mount propagation must be shared or slave" run_host_preflight "$COMPOSE_FILE"
|
||||
|
||||
mount_propagation_for_path() { return 1; }
|
||||
assert_failure_with "unavailable propagation detection fails clearly" "Unable to inspect mount propagation" run_host_preflight "$COMPOSE_FILE"
|
||||
|
||||
mount_propagation_for_path() { printf '%s\n' shared; }
|
||||
command() {
|
||||
if [[ "$1" == "-v" && "$2" == "ss" ]]; then
|
||||
return 1
|
||||
fi
|
||||
builtin command "$@"
|
||||
}
|
||||
assert_failure_with "unavailable port detection fails clearly" "Unable to inspect management ports" run_host_preflight "$COMPOSE_FILE"
|
||||
unset -f command
|
||||
|
||||
is_port_in_use() { [[ "$1" == "9999" ]]; }
|
||||
assert_failure_with "occupied Dozzle port identifies the port" "Port 9999/tcp is already in use" run_host_preflight "$COMPOSE_FILE"
|
||||
|
||||
is_port_in_use() { [[ "$1" == "8080" ]]; }
|
||||
assert_failure_with "occupied Admin port identifies the port" "Port 8080/tcp is already in use" run_host_preflight "$COMPOSE_FILE"
|
||||
|
||||
assert_failure_with "occupied port warns that rerunning resets the database" "Rerunning the installer is not an update path" run_host_preflight "$COMPOSE_FILE"
|
||||
|
||||
preflight_line=$(grep -n '^[[:space:]]*run_host_preflight ' "$INSTALLER" | tail -n 1 | cut -d: -f1)
|
||||
create_line=$(grep -n '^[[:space:]]*create_nomad_directory$' "$INSTALLER" | tail -n 1 | cut -d: -f1)
|
||||
if [[ -n "$preflight_line" && -n "$create_line" && "$preflight_line" -lt "$create_line" ]]; then
|
||||
pass "preflight runs before installation directory creation"
|
||||
else
|
||||
fail "preflight runs before installation directory creation"
|
||||
fi
|
||||
|
||||
if grep -Fq 'download_management_compose_file "$preflight_compose_file"' "$INSTALLER"; then
|
||||
pass "installation reuses the preflighted compose file"
|
||||
else
|
||||
fail "installation reuses the preflighted compose file"
|
||||
fi
|
||||
|
||||
side_effect_dir=$(mktemp -d)
|
||||
rmdir "$side_effect_dir"
|
||||
(
|
||||
NOMAD_DIR="$side_effect_dir"
|
||||
check_is_debian_based() { :; }
|
||||
check_is_x86_64() { :; }
|
||||
check_is_bash() { :; }
|
||||
check_has_sudo() { :; }
|
||||
ensure_dependencies_installed() { :; }
|
||||
check_is_debug_mode() { :; }
|
||||
banner() { :; }
|
||||
get_install_confirmation() { :; }
|
||||
accept_terms() { :; }
|
||||
ensure_docker_installed() { :; }
|
||||
check_docker_compose() { :; }
|
||||
setup_nvidia_container_toolkit() { :; }
|
||||
get_local_ip() { :; }
|
||||
curl() { cp "$COMPOSE_FILE" "${@: -1}"; }
|
||||
mount_propagation_for_path() { printf '%s\n' private; }
|
||||
is_port_in_use() { return 1; }
|
||||
main
|
||||
) >/tmp/nomad-preflight-main-test.out 2>&1
|
||||
main_status=$?
|
||||
if [[ "$main_status" -ne 0 ]] &&
|
||||
[[ ! -e "$side_effect_dir" ]] &&
|
||||
grep -Fq "Project NOMAD host preflight failed" /tmp/nomad-preflight-main-test.out; then
|
||||
pass "main fails preflight before creating the installation directory"
|
||||
else
|
||||
fail "main fails preflight before creating the installation directory"
|
||||
cat /tmp/nomad-preflight-main-test.out
|
||||
fi
|
||||
|
||||
rm -f /tmp/nomad-preflight-test.out /tmp/nomad-preflight-main-test.out "$edge_compose"
|
||||
|
||||
if (( failures > 0 )); then
|
||||
printf '\n%d test(s) failed\n' "$failures"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '\nall tests passed\n'
|
||||
Loading…
Reference in New Issue
Block a user