#!/bin/bash # # Web Hosts Health Check Script # Checks all domains, restarts unhealthy containers, reloads nginx # set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" HOSTS_DIR="$(dirname "$SCRIPT_DIR")" DOMAINS_DIR="$HOSTS_DIR/domains" LOGS_DIR="$HOSTS_DIR/logs" # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' mkdir -p "$LOGS_DIR" LOG_FILE="$LOGS_DIR/healthcheck.log" log() { local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $1" echo -e "${BLUE}[INFO]${NC} $1" echo "$msg" >> "$LOG_FILE" } log_ok() { local msg="[$(date '+%Y-%m-%d %H:%M:%S')] OK: $1" echo -e "${GREEN}[OK]${NC} $1" echo "$msg" >> "$LOG_FILE" } log_warn() { local msg="[$(date '+%Y-%m-%d %H:%M:%S')] WARN: $1" echo -e "${YELLOW}[WARN]${NC} $1" echo "$msg" >> "$LOG_FILE" } log_error() { local msg="[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $1" echo -e "${RED}[ERROR]${NC} $1" echo "$msg" >> "$LOG_FILE" } # Domain configurations: domain:port:compose_project:compose_dir declare -A DOMAINS=( ["chuckie.coppertone.tech"]="9201:chuckiecoppertonetech:/docker/web-hosts/domains/chuckie.coppertone.tech" ["test.coppertone.tech"]="9100:testcoppertonetech:/docker/web-hosts/domains/test.coppertone.tech" ["coppertone.tech"]="8090:coppertonetech:/docker/web-hosts/domains/coppertone.tech" ["ci.coppertone.tech"]="8000:woodpecker:/docker/woodpecker" ) # Check if a URL returns 200 check_url() { local url="$1" local status=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 5 --max-time 10 "$url" 2>/dev/null) [ "$status" = "200" ] } # Check if a port is listening check_port() { local port="$1" nc -z 127.0.0.1 "$port" 2>/dev/null } # Check containers for a compose project check_containers() { local project="$1" local unhealthy=0 # Get all containers for this project local containers=$(podman ps -a --filter "label=io.podman.compose.project=$project" --format "{{.Names}}:{{.Status}}" 2>/dev/null) if [ -z "$containers" ]; then return 1 fi while IFS=: read -r name status; do if [[ "$status" != Up* ]]; then log_warn "Container $name is not running: $status" unhealthy=1 elif [[ "$status" == *"unhealthy"* ]]; then log_warn "Container $name is unhealthy" unhealthy=1 fi done <<< "$containers" return $unhealthy } # Restart containers for a domain restart_domain() { local domain="$1" local compose_dir="$2" if [ -z "$compose_dir" ]; then compose_dir="$DOMAINS_DIR/$domain" fi if [ ! -d "$compose_dir" ]; then log_error "Directory not found: $compose_dir" return 1 fi # Check for compose file (compose.yaml or podman-compose.yml) local compose_file="" if [ -f "$compose_dir/compose.yaml" ]; then compose_file="compose.yaml" elif [ -f "$compose_dir/podman-compose.yml" ]; then compose_file="podman-compose.yml" else log_error "No compose file found for $domain in $compose_dir" return 1 fi log "Restarting $domain containers..." cd "$compose_dir" # Stop and start (not just restart, to ensure clean state) podman-compose -f "$compose_file" down 2>/dev/null || true sleep 2 if podman-compose -f "$compose_file" up -d 2>&1 | tail -3; then log_ok "$domain containers restarted" return 0 else log_error "Failed to restart $domain" return 1 fi } # Reload nginx reload_nginx() { log "Reloading nginx..." # Try via startup.sh (has sudo) if sudo /docker/www/startup.sh reload 2>/dev/null; then log_ok "Nginx reloaded" return 0 fi # Fallback: direct command if sudo podman exec gitea-nginx nginx -s reload 2>/dev/null; then log_ok "Nginx reloaded (direct)" return 0 fi log_error "Failed to reload nginx" return 1 } # Check and fix a single domain check_domain() { local domain="$1" local config="${DOMAINS[$domain]}" if [ -z "$config" ]; then log_warn "Unknown domain: $domain" return 1 fi IFS=':' read -r port project compose_dir <<< "$config" echo "" log "Checking $domain..." local needs_restart=0 # Check 1: HTTPS endpoint if check_url "https://$domain/"; then log_ok "HTTPS endpoint responding" else log_warn "HTTPS endpoint not responding (502/503/timeout)" needs_restart=1 fi # Check 2: Local port if check_port "$port"; then log_ok "Port $port is listening" else log_warn "Port $port is not listening" needs_restart=1 fi # Check 3: Container health if check_containers "$project"; then log_ok "All containers healthy" else log_warn "Some containers unhealthy or stopped" needs_restart=1 fi # Restart if needed if [ $needs_restart -eq 1 ]; then log_warn "Issues detected, restarting $domain..." restart_domain "$domain" "$compose_dir" return 1 fi return 0 } # Main health check healthcheck() { log "==========================================" log "Web Hosts Health Check" log "==========================================" local issues=0 local restarted=0 for domain in "${!DOMAINS[@]}"; do if ! check_domain "$domain"; then ((restarted++)) fi done # If any domains were restarted, wait and reload nginx if [ $restarted -gt 0 ]; then echo "" log "Waiting for containers to initialize..." sleep 10 reload_nginx || true echo "" log "Re-checking domains after restart..." for domain in "${!DOMAINS[@]}"; do if check_url "https://$domain/"; then log_ok "$domain is now responding" else log_error "$domain still not responding" ((issues++)) fi done fi echo "" log "==========================================" if [ $issues -eq 0 ]; then log_ok "All domains healthy" else log_error "$issues domain(s) still have issues" fi log "==========================================" return $issues } # Quick fix - just restart everything fix_all() { log "==========================================" log "Restarting ALL domains" log "==========================================" for domain in "${!DOMAINS[@]}"; do config="${DOMAINS[$domain]}" IFS=':' read -r port project compose_dir <<< "$config" restart_domain "$domain" "$compose_dir" echo "" done log "Waiting for containers..." sleep 10 reload_nginx echo "" log "Verifying..." sleep 3 for domain in "${!DOMAINS[@]}"; do if check_url "https://$domain/"; then log_ok "$domain: OK" else log_error "$domain: FAILED" fi done } # Usage usage() { echo "Web Hosts Health Check Script" echo "" echo "Usage: $0 " echo "" echo "Commands:" echo " check Check all domains, restart if unhealthy, reload nginx" echo " fix Force restart all domains and reload nginx" echo " reload Just reload nginx" echo " status Show current status without fixing" echo "" } # Status only (no fixes) status_only() { log "==========================================" log "Web Hosts Status" log "==========================================" for domain in "${!DOMAINS[@]}"; do config="${DOMAINS[$domain]}" IFS=':' read -r port project compose_dir <<< "$config" echo "" echo -e "${BLUE}=== $domain ===${NC}" # URL check if check_url "https://$domain/"; then echo -e " HTTPS: ${GREEN}OK${NC}" else echo -e " HTTPS: ${RED}FAIL${NC}" fi # Port check if check_port "$port"; then echo -e " Port $port: ${GREEN}OK${NC}" else echo -e " Port $port: ${RED}FAIL${NC}" fi # Container status echo " Containers:" podman ps -a --filter "label=io.podman.compose.project=$project" --format " {{.Names}}: {{.Status}}" 2>/dev/null || echo " None found" done echo "" } case "${1:-check}" in check) healthcheck ;; fix) fix_all ;; reload) reload_nginx ;; status) status_only ;; *) usage exit 1 ;; esac