Harden webhook and health checks
This commit is contained in:
Administrator
344
scripts/health-check.sh
Executable file
344
scripts/health-check.sh
Executable file
@@ -0,0 +1,344 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Health Check Script for all web services
|
||||
# Checks containers, ports, and URLs
|
||||
#
|
||||
|
||||
# Don't exit on errors - we want to collect all results
|
||||
set +e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Counters
|
||||
PASS=0
|
||||
FAIL=0
|
||||
WARN=0
|
||||
|
||||
log_pass() {
|
||||
echo -e "${GREEN}[PASS]${NC} $1"
|
||||
((PASS++))
|
||||
}
|
||||
|
||||
log_fail() {
|
||||
echo -e "${RED}[FAIL]${NC} $1"
|
||||
((FAIL++))
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
echo -e "${YELLOW}[WARN]${NC} $1"
|
||||
((WARN++))
|
||||
}
|
||||
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
section() {
|
||||
echo ""
|
||||
echo -e "${BLUE}=== $1 ===${NC}"
|
||||
}
|
||||
|
||||
# Check if a container is running
|
||||
check_container() {
|
||||
local name="$1"
|
||||
local status=$(podman ps --filter "name=^${name}$" --format "{{.Status}}" 2>/dev/null | head -1)
|
||||
|
||||
if [ -z "$status" ]; then
|
||||
log_fail "Container '$name' is not running"
|
||||
return 1
|
||||
elif echo "$status" | grep -q "unhealthy"; then
|
||||
log_warn "Container '$name' is unhealthy: $status"
|
||||
return 2
|
||||
elif echo "$status" | grep -q "Up"; then
|
||||
log_pass "Container '$name' is running: $status"
|
||||
return 0
|
||||
else
|
||||
log_fail "Container '$name' status: $status"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check if a port is listening
|
||||
check_port() {
|
||||
local port="$1"
|
||||
local service="$2"
|
||||
|
||||
if nc -z 127.0.0.1 "$port" 2>/dev/null; then
|
||||
log_pass "Port $port ($service) is listening"
|
||||
return 0
|
||||
else
|
||||
log_fail "Port $port ($service) is not listening"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check URL returns expected status
|
||||
check_url() {
|
||||
local url="$1"
|
||||
local expected="${2:-200}"
|
||||
local timeout="${3:-10}"
|
||||
|
||||
local status=$(curl -sI -o /dev/null -w "%{http_code}" --max-time "$timeout" "$url" 2>/dev/null)
|
||||
|
||||
if [ "$status" = "$expected" ]; then
|
||||
log_pass "$url returned $status"
|
||||
return 0
|
||||
elif [ "$status" = "000" ]; then
|
||||
log_fail "$url - connection failed"
|
||||
return 1
|
||||
else
|
||||
log_fail "$url returned $status (expected $expected)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check URL returns content (for sites that may return different codes)
|
||||
check_url_content() {
|
||||
local url="$1"
|
||||
local search="$2"
|
||||
local timeout="${3:-10}"
|
||||
|
||||
local content=$(curl -sL --max-time "$timeout" "$url" 2>/dev/null)
|
||||
|
||||
if echo "$content" | grep -qi "$search"; then
|
||||
log_pass "$url contains expected content"
|
||||
return 0
|
||||
else
|
||||
log_fail "$url missing expected content '$search'"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check systemd service
|
||||
check_systemd_service() {
|
||||
local service="$1"
|
||||
local user="${2:-}"
|
||||
|
||||
if [ "$user" = "user" ]; then
|
||||
local status=$(systemctl --user is-active "$service" 2>/dev/null)
|
||||
else
|
||||
local status=$(systemctl is-active "$service" 2>/dev/null)
|
||||
fi
|
||||
|
||||
if [ "$status" = "active" ]; then
|
||||
log_pass "Service '$service' is active"
|
||||
return 0
|
||||
elif [ "$status" = "inactive" ]; then
|
||||
log_warn "Service '$service' is inactive"
|
||||
return 2
|
||||
else
|
||||
log_fail "Service '$service' status: $status"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Main health checks
|
||||
main() {
|
||||
echo ""
|
||||
echo -e "${BLUE}╔════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${BLUE}║ Web Services Health Check ║${NC}"
|
||||
echo -e "${BLUE}║ $(date '+%Y-%m-%d %H:%M:%S') ║${NC}"
|
||||
echo -e "${BLUE}╚════════════════════════════════════════════════════════════╝${NC}"
|
||||
|
||||
# =========================================
|
||||
section "Systemd Services (User)"
|
||||
# =========================================
|
||||
check_systemd_service "podman-compose-postgres.service" "user"
|
||||
check_systemd_service "podman-compose-gitea.service" "user"
|
||||
check_systemd_service "web-hosts.service" "user"
|
||||
check_systemd_service "test-coppertone-webhook.service" "user"
|
||||
check_systemd_service "web-hosts-webhook.service" "user"
|
||||
|
||||
# =========================================
|
||||
section "Core Infrastructure Containers"
|
||||
# =========================================
|
||||
check_container "postgres"
|
||||
check_container "gitea"
|
||||
# gitea-nginx runs under root podman
|
||||
if sudo -n podman ps --filter "name=gitea-nginx" --format "{{.Status}}" 2>/dev/null | grep -q "Up"; then
|
||||
log_pass "Container 'gitea-nginx' (root) is running"
|
||||
else
|
||||
if sudo -n podman ps --format "{{.ID}}" >/dev/null 2>&1; then
|
||||
log_fail "Container 'gitea-nginx' (root) is not running"
|
||||
else
|
||||
log_warn "Skipping gitea-nginx root container check (sudo required)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# =========================================
|
||||
section "Web Host Containers"
|
||||
# =========================================
|
||||
|
||||
# Chuckie (MarketManager)
|
||||
check_container "chuckie-redis"
|
||||
check_container "chuckie-api"
|
||||
check_container "chuckie-frontend"
|
||||
|
||||
# Games (Spades)
|
||||
check_container "games-spades-backend"
|
||||
check_container "games-spades-frontend"
|
||||
|
||||
# Test.coppertone.tech
|
||||
check_container "test-coppertone-tech-frontend"
|
||||
check_container "test-coppertone-tech-db"
|
||||
check_container "test-coppertone-tech-auth"
|
||||
check_container "test-coppertone-tech-work-mgmt"
|
||||
check_container "test-coppertone-tech-blog"
|
||||
|
||||
# Coppertone.tech (if running)
|
||||
check_container "coppertonetech_frontend_1" || true
|
||||
check_container "coppertonetech_auth-service_1" || true
|
||||
|
||||
# =========================================
|
||||
section "Port Connectivity"
|
||||
# =========================================
|
||||
check_port 80 "HTTP (nginx)"
|
||||
check_port 443 "HTTPS (nginx)"
|
||||
check_port 3000 "Gitea"
|
||||
check_port 5432 "PostgreSQL"
|
||||
check_port 2222 "Gitea SSH"
|
||||
check_port 9100 "test.coppertone.tech frontend"
|
||||
check_port 9102 "test.coppertone.tech auth"
|
||||
check_port 9200 "chuckie.coppertone.tech backend"
|
||||
check_port 9201 "chuckie.coppertone.tech frontend"
|
||||
check_port 9300 "games.coppertone.tech frontend"
|
||||
|
||||
# =========================================
|
||||
section "Website Accessibility (HTTPS)"
|
||||
# =========================================
|
||||
check_url "https://coppertone.tech" "200"
|
||||
check_url "https://test.coppertone.tech" "200"
|
||||
check_url "https://chuckie.coppertone.tech" "200"
|
||||
check_url "https://api.chuckie.coppertone.tech/health" "200"
|
||||
check_url "https://canva.chuckie.coppertone.tech" "200"
|
||||
check_url "https://games.coppertone.tech" "200"
|
||||
check_url "https://git.coppertone.tech" "200"
|
||||
|
||||
# =========================================
|
||||
section "Website Content Verification"
|
||||
# =========================================
|
||||
check_url_content "https://git.coppertone.tech" "Gitea"
|
||||
check_url_content "https://chuckie.coppertone.tech" "html"
|
||||
check_url_content "https://canva.chuckie.coppertone.tech" "html"
|
||||
check_url_content "https://games.coppertone.tech" "html"
|
||||
check_url_content "https://test.coppertone.tech" "html"
|
||||
|
||||
# =========================================
|
||||
section "API Health Endpoints"
|
||||
# =========================================
|
||||
# Gitea API
|
||||
local gitea_api=$(curl -s "https://git.coppertone.tech/api/v1/version" 2>/dev/null)
|
||||
if echo "$gitea_api" | grep -q "version"; then
|
||||
log_pass "Gitea API is responsive"
|
||||
else
|
||||
log_fail "Gitea API not responding"
|
||||
fi
|
||||
|
||||
# Test auth service health (fallback to root if /health is missing)
|
||||
local auth_health=$(curl -s "http://127.0.0.1:9102/health" 2>/dev/null)
|
||||
if [ "$auth_health" = "OK" ] || echo "$auth_health" | grep -qi "healthy\|ok"; then
|
||||
log_pass "test.coppertone.tech auth service healthy"
|
||||
else
|
||||
local auth_status=$(curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:9102/" 2>/dev/null)
|
||||
if [ "$auth_status" != "000" ]; then
|
||||
log_pass "test.coppertone.tech auth service responding (/) "
|
||||
else
|
||||
log_warn "test.coppertone.tech auth service health unknown"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Chuckie backend health
|
||||
local chuckie_health=$(curl -s "http://127.0.0.1:9200/health" 2>/dev/null)
|
||||
if echo "$chuckie_health" | grep -qi "ok\|healthy"; then
|
||||
log_pass "chuckie.coppertone.tech backend healthy"
|
||||
else
|
||||
log_warn "chuckie.coppertone.tech backend health unknown"
|
||||
fi
|
||||
|
||||
# API subdomain health (via HTTPS)
|
||||
local api_health=$(curl -s "https://api.chuckie.coppertone.tech/health" 2>/dev/null)
|
||||
if echo "$api_health" | grep -qi "ok\|healthy"; then
|
||||
log_pass "api.chuckie.coppertone.tech backend healthy"
|
||||
else
|
||||
log_warn "api.chuckie.coppertone.tech backend health unknown"
|
||||
fi
|
||||
|
||||
# =========================================
|
||||
section "Database Connectivity"
|
||||
# =========================================
|
||||
if podman exec postgres pg_isready -U gitea -d gitea >/dev/null 2>&1; then
|
||||
log_pass "PostgreSQL is accepting connections"
|
||||
else
|
||||
log_fail "PostgreSQL is not accepting connections"
|
||||
fi
|
||||
|
||||
# Check gitea database
|
||||
local gitea_tables=$(podman exec postgres psql -U gitea -d gitea -c "SELECT count(*) FROM repository;" -t 2>/dev/null | tr -d ' ')
|
||||
if [ -n "$gitea_tables" ] && [ "$gitea_tables" -gt 0 ]; then
|
||||
log_pass "Gitea database has $gitea_tables repositories"
|
||||
else
|
||||
log_warn "Gitea database may be empty or inaccessible"
|
||||
fi
|
||||
|
||||
# =========================================
|
||||
section "SSL Certificates"
|
||||
# =========================================
|
||||
for domain in coppertone.tech test.coppertone.tech chuckie.coppertone.tech api.chuckie.coppertone.tech canva.chuckie.coppertone.tech games.coppertone.tech git.coppertone.tech; do
|
||||
local expiry=$(echo | openssl s_client -servername "$domain" -connect "$domain:443" 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||||
if [ -n "$expiry" ]; then
|
||||
local expiry_epoch=$(date -d "$expiry" +%s 2>/dev/null)
|
||||
local now_epoch=$(date +%s)
|
||||
local days_left=$(( (expiry_epoch - now_epoch) / 86400 ))
|
||||
|
||||
if [ "$days_left" -lt 7 ]; then
|
||||
log_fail "$domain SSL expires in $days_left days"
|
||||
elif [ "$days_left" -lt 30 ]; then
|
||||
log_warn "$domain SSL expires in $days_left days"
|
||||
else
|
||||
log_pass "$domain SSL valid ($days_left days)"
|
||||
fi
|
||||
else
|
||||
log_fail "$domain SSL certificate check failed"
|
||||
fi
|
||||
done
|
||||
|
||||
# =========================================
|
||||
section "Summary"
|
||||
# =========================================
|
||||
echo ""
|
||||
echo -e "Results: ${GREEN}$PASS passed${NC}, ${RED}$FAIL failed${NC}, ${YELLOW}$WARN warnings${NC}"
|
||||
echo ""
|
||||
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo -e "${RED}Health check completed with failures!${NC}"
|
||||
return 1
|
||||
elif [ "$WARN" -gt 0 ]; then
|
||||
echo -e "${YELLOW}Health check completed with warnings.${NC}"
|
||||
return 0
|
||||
else
|
||||
echo -e "${GREEN}All health checks passed!${NC}"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# Run with optional flags
|
||||
case "${1:-}" in
|
||||
--quiet|-q)
|
||||
main 2>&1 | grep -E "^\[(FAIL|WARN)\]|^Results:"
|
||||
;;
|
||||
--json|-j)
|
||||
# JSON output for monitoring systems
|
||||
main >/dev/null 2>&1
|
||||
echo "{\"pass\": $PASS, \"fail\": $FAIL, \"warn\": $WARN, \"timestamp\": \"$(date -Iseconds)\"}"
|
||||
;;
|
||||
*)
|
||||
main
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user