Harden webhook and health checks

This commit is contained in:
Administrator
2025-12-26 13:17:19 +01:00
parent 69a28b0537
commit 152f507d9a
6 changed files with 736 additions and 2 deletions

View File

@@ -0,0 +1,69 @@
version: '3.8'
services:
api:
build:
context: ./repo
dockerfile: Dockerfile.api
container_name: chuckie-api
restart: unless-stopped
env_file:
- ./repo/.env
environment:
- PORT=8678
- SESSION_STORE_PATH=/app/data/sessions.json
volumes:
- api-data:/app/data
ports:
- 127.0.0.1:9200:8678
networks:
- chuckie
healthcheck:
test: [CMD, wget, -q, --spider, http://127.0.0.1:8678/health]
interval: 30s
timeout: 5s
retries: 5
frontend:
build:
context: ./repo
dockerfile: Dockerfile
args:
VITE_API_BASE:
VITE_CANVA_CLIENT_ID:
VITE_CANVA_DEFAULT_DESIGN_TYPE:
VITE_CANVA_PANEL_URL:
VITE_AUTH0_DOMAIN:
VITE_AUTH0_CLIENT_ID:
VITE_AUTH0_AUDIENCE:
VITE_AUTH0_REDIRECT_URI:
container_name: chuckie-frontend
restart: unless-stopped
depends_on:
- api
ports:
- 127.0.0.1:9201:8080
networks:
- chuckie
canva-app:
build:
context: ./repo
dockerfile: Dockerfile.canva-app
args:
VITE_API_BASE:
container_name: chuckie-canva-app
restart: unless-stopped
depends_on:
- api
ports:
- 127.0.0.1:9202:8080
networks:
- chuckie
networks:
chuckie:
driver: bridge
volumes:
api-data:

View File

@@ -20,7 +20,7 @@ services:
environment: environment:
- VITE_API_BASE_URL=https://test.coppertone.tech/api - VITE_API_BASE_URL=https://test.coppertone.tech/api
healthcheck: healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost/"] test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1/"]
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 3 retries: 3

View File

@@ -83,6 +83,15 @@ show_logs() {
podman-compose logs -f --tail 50 podman-compose logs -f --tail 50
} }
reload_nginx() {
log_info "Reloading nginx..."
if /docker/www/scripts/nginx-reload.sh 2>/dev/null; then
log_success "Nginx reloaded"
else
log_warn "Nginx reload failed (non-fatal)"
fi
}
full_deploy() { full_deploy() {
echo "==========================================" echo "=========================================="
echo " Deploying test.coppertone.tech" echo " Deploying test.coppertone.tech"
@@ -98,6 +107,8 @@ full_deploy() {
echo "" echo ""
start_containers start_containers
echo "" echo ""
reload_nginx
echo ""
log_success "==========================================" log_success "=========================================="
log_success " Deployment complete!" log_success " Deployment complete!"
@@ -132,8 +143,11 @@ case "${1:-deploy}" in
logs) logs)
show_logs show_logs
;; ;;
reload-nginx)
reload_nginx
;;
*) *)
echo "Usage: $0 {deploy|pull|build|restart|stop|start|status|logs}" echo "Usage: $0 {deploy|pull|build|restart|stop|start|status|logs|reload-nginx}"
exit 1 exit 1
;; ;;
esac esac

344
scripts/health-check.sh Executable file
View File

@@ -0,0 +1,344 @@
#!/bin/bash
#
# Health Check Script for all web services
# Checks containers, ports, and URLs
#
# Don't exit on errors - we want to collect all results
set +e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'
# Counters
PASS=0
FAIL=0
WARN=0
log_pass() {
echo -e "${GREEN}[PASS]${NC} $1"
((PASS++))
}
log_fail() {
echo -e "${RED}[FAIL]${NC} $1"
((FAIL++))
}
log_warn() {
echo -e "${YELLOW}[WARN]${NC} $1"
((WARN++))
}
log_info() {
echo -e "${BLUE}[INFO]${NC} $1"
}
section() {
echo ""
echo -e "${BLUE}=== $1 ===${NC}"
}
# Check if a container is running
check_container() {
local name="$1"
local status=$(podman ps --filter "name=^${name}$" --format "{{.Status}}" 2>/dev/null | head -1)
if [ -z "$status" ]; then
log_fail "Container '$name' is not running"
return 1
elif echo "$status" | grep -q "unhealthy"; then
log_warn "Container '$name' is unhealthy: $status"
return 2
elif echo "$status" | grep -q "Up"; then
log_pass "Container '$name' is running: $status"
return 0
else
log_fail "Container '$name' status: $status"
return 1
fi
}
# Check if a port is listening
check_port() {
local port="$1"
local service="$2"
if nc -z 127.0.0.1 "$port" 2>/dev/null; then
log_pass "Port $port ($service) is listening"
return 0
else
log_fail "Port $port ($service) is not listening"
return 1
fi
}
# Check URL returns expected status
check_url() {
local url="$1"
local expected="${2:-200}"
local timeout="${3:-10}"
local status=$(curl -sI -o /dev/null -w "%{http_code}" --max-time "$timeout" "$url" 2>/dev/null)
if [ "$status" = "$expected" ]; then
log_pass "$url returned $status"
return 0
elif [ "$status" = "000" ]; then
log_fail "$url - connection failed"
return 1
else
log_fail "$url returned $status (expected $expected)"
return 1
fi
}
# Check URL returns content (for sites that may return different codes)
check_url_content() {
local url="$1"
local search="$2"
local timeout="${3:-10}"
local content=$(curl -sL --max-time "$timeout" "$url" 2>/dev/null)
if echo "$content" | grep -qi "$search"; then
log_pass "$url contains expected content"
return 0
else
log_fail "$url missing expected content '$search'"
return 1
fi
}
# Check systemd service
check_systemd_service() {
local service="$1"
local user="${2:-}"
if [ "$user" = "user" ]; then
local status=$(systemctl --user is-active "$service" 2>/dev/null)
else
local status=$(systemctl is-active "$service" 2>/dev/null)
fi
if [ "$status" = "active" ]; then
log_pass "Service '$service' is active"
return 0
elif [ "$status" = "inactive" ]; then
log_warn "Service '$service' is inactive"
return 2
else
log_fail "Service '$service' status: $status"
return 1
fi
}
# Main health checks
main() {
echo ""
echo -e "${BLUE}╔════════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║ Web Services Health Check ║${NC}"
echo -e "${BLUE}$(date '+%Y-%m-%d %H:%M:%S')${NC}"
echo -e "${BLUE}╚════════════════════════════════════════════════════════════╝${NC}"
# =========================================
section "Systemd Services (User)"
# =========================================
check_systemd_service "podman-compose-postgres.service" "user"
check_systemd_service "podman-compose-gitea.service" "user"
check_systemd_service "web-hosts.service" "user"
check_systemd_service "test-coppertone-webhook.service" "user"
check_systemd_service "web-hosts-webhook.service" "user"
# =========================================
section "Core Infrastructure Containers"
# =========================================
check_container "postgres"
check_container "gitea"
# gitea-nginx runs under root podman
if sudo -n podman ps --filter "name=gitea-nginx" --format "{{.Status}}" 2>/dev/null | grep -q "Up"; then
log_pass "Container 'gitea-nginx' (root) is running"
else
if sudo -n podman ps --format "{{.ID}}" >/dev/null 2>&1; then
log_fail "Container 'gitea-nginx' (root) is not running"
else
log_warn "Skipping gitea-nginx root container check (sudo required)"
fi
fi
# =========================================
section "Web Host Containers"
# =========================================
# Chuckie (MarketManager)
check_container "chuckie-redis"
check_container "chuckie-api"
check_container "chuckie-frontend"
# Games (Spades)
check_container "games-spades-backend"
check_container "games-spades-frontend"
# Test.coppertone.tech
check_container "test-coppertone-tech-frontend"
check_container "test-coppertone-tech-db"
check_container "test-coppertone-tech-auth"
check_container "test-coppertone-tech-work-mgmt"
check_container "test-coppertone-tech-blog"
# Coppertone.tech (if running)
check_container "coppertonetech_frontend_1" || true
check_container "coppertonetech_auth-service_1" || true
# =========================================
section "Port Connectivity"
# =========================================
check_port 80 "HTTP (nginx)"
check_port 443 "HTTPS (nginx)"
check_port 3000 "Gitea"
check_port 5432 "PostgreSQL"
check_port 2222 "Gitea SSH"
check_port 9100 "test.coppertone.tech frontend"
check_port 9102 "test.coppertone.tech auth"
check_port 9200 "chuckie.coppertone.tech backend"
check_port 9201 "chuckie.coppertone.tech frontend"
check_port 9300 "games.coppertone.tech frontend"
# =========================================
section "Website Accessibility (HTTPS)"
# =========================================
check_url "https://coppertone.tech" "200"
check_url "https://test.coppertone.tech" "200"
check_url "https://chuckie.coppertone.tech" "200"
check_url "https://api.chuckie.coppertone.tech/health" "200"
check_url "https://canva.chuckie.coppertone.tech" "200"
check_url "https://games.coppertone.tech" "200"
check_url "https://git.coppertone.tech" "200"
# =========================================
section "Website Content Verification"
# =========================================
check_url_content "https://git.coppertone.tech" "Gitea"
check_url_content "https://chuckie.coppertone.tech" "html"
check_url_content "https://canva.chuckie.coppertone.tech" "html"
check_url_content "https://games.coppertone.tech" "html"
check_url_content "https://test.coppertone.tech" "html"
# =========================================
section "API Health Endpoints"
# =========================================
# Gitea API
local gitea_api=$(curl -s "https://git.coppertone.tech/api/v1/version" 2>/dev/null)
if echo "$gitea_api" | grep -q "version"; then
log_pass "Gitea API is responsive"
else
log_fail "Gitea API not responding"
fi
# Test auth service health (fallback to root if /health is missing)
local auth_health=$(curl -s "http://127.0.0.1:9102/health" 2>/dev/null)
if [ "$auth_health" = "OK" ] || echo "$auth_health" | grep -qi "healthy\|ok"; then
log_pass "test.coppertone.tech auth service healthy"
else
local auth_status=$(curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:9102/" 2>/dev/null)
if [ "$auth_status" != "000" ]; then
log_pass "test.coppertone.tech auth service responding (/) "
else
log_warn "test.coppertone.tech auth service health unknown"
fi
fi
# Chuckie backend health
local chuckie_health=$(curl -s "http://127.0.0.1:9200/health" 2>/dev/null)
if echo "$chuckie_health" | grep -qi "ok\|healthy"; then
log_pass "chuckie.coppertone.tech backend healthy"
else
log_warn "chuckie.coppertone.tech backend health unknown"
fi
# API subdomain health (via HTTPS)
local api_health=$(curl -s "https://api.chuckie.coppertone.tech/health" 2>/dev/null)
if echo "$api_health" | grep -qi "ok\|healthy"; then
log_pass "api.chuckie.coppertone.tech backend healthy"
else
log_warn "api.chuckie.coppertone.tech backend health unknown"
fi
# =========================================
section "Database Connectivity"
# =========================================
if podman exec postgres pg_isready -U gitea -d gitea >/dev/null 2>&1; then
log_pass "PostgreSQL is accepting connections"
else
log_fail "PostgreSQL is not accepting connections"
fi
# Check gitea database
local gitea_tables=$(podman exec postgres psql -U gitea -d gitea -c "SELECT count(*) FROM repository;" -t 2>/dev/null | tr -d ' ')
if [ -n "$gitea_tables" ] && [ "$gitea_tables" -gt 0 ]; then
log_pass "Gitea database has $gitea_tables repositories"
else
log_warn "Gitea database may be empty or inaccessible"
fi
# =========================================
section "SSL Certificates"
# =========================================
for domain in coppertone.tech test.coppertone.tech chuckie.coppertone.tech api.chuckie.coppertone.tech canva.chuckie.coppertone.tech games.coppertone.tech git.coppertone.tech; do
local expiry=$(echo | openssl s_client -servername "$domain" -connect "$domain:443" 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
if [ -n "$expiry" ]; then
local expiry_epoch=$(date -d "$expiry" +%s 2>/dev/null)
local now_epoch=$(date +%s)
local days_left=$(( (expiry_epoch - now_epoch) / 86400 ))
if [ "$days_left" -lt 7 ]; then
log_fail "$domain SSL expires in $days_left days"
elif [ "$days_left" -lt 30 ]; then
log_warn "$domain SSL expires in $days_left days"
else
log_pass "$domain SSL valid ($days_left days)"
fi
else
log_fail "$domain SSL certificate check failed"
fi
done
# =========================================
section "Summary"
# =========================================
echo ""
echo -e "Results: ${GREEN}$PASS passed${NC}, ${RED}$FAIL failed${NC}, ${YELLOW}$WARN warnings${NC}"
echo ""
if [ "$FAIL" -gt 0 ]; then
echo -e "${RED}Health check completed with failures!${NC}"
return 1
elif [ "$WARN" -gt 0 ]; then
echo -e "${YELLOW}Health check completed with warnings.${NC}"
return 0
else
echo -e "${GREEN}All health checks passed!${NC}"
return 0
fi
}
# Run with optional flags
case "${1:-}" in
--quiet|-q)
main 2>&1 | grep -E "^\[(FAIL|WARN)\]|^Results:"
;;
--json|-j)
# JSON output for monitoring systems
main >/dev/null 2>&1
echo "{\"pass\": $PASS, \"fail\": $FAIL, \"warn\": $WARN, \"timestamp\": \"$(date -Iseconds)\"}"
;;
*)
main
;;
esac

269
webhook/main.go Normal file
View File

@@ -0,0 +1,269 @@
package main
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
)
const (
domainsDir = "/docker/web-hosts/domains"
logsDir = "/docker/web-hosts/logs"
)
var (
deployMutex sync.Mutex
lastDeploys = make(map[string]time.Time)
domainConfig = make(map[string]DomainConfig)
)
type DomainConfig struct {
Secret string
TargetBranch string
DeployScript string
}
type GiteaPayload struct {
Ref string `json:"ref"`
Repository struct {
Name string `json:"name"`
FullName string `json:"full_name"`
} `json:"repository"`
Pusher struct {
Username string `json:"username"`
} `json:"pusher"`
}
func loadConfig() {
// Load domain configurations from environment
// Format: WEBHOOK_<DOMAIN_UNDERSCORE>_SECRET, WEBHOOK_<DOMAIN_UNDERSCORE>_BRANCH
// e.g., WEBHOOK_TEST_COPPERTONE_TECH_SECRET
domains := []string{
"test.coppertone.tech",
"chuckie.coppertone.tech",
"dev.coppertone.tech",
"games.coppertone.tech",
"coppertone.tech",
}
globalSecret := os.Getenv("WEBHOOK_SECRET")
for _, domain := range domains {
envKey := strings.ReplaceAll(strings.ReplaceAll(domain, ".", "_"), "-", "_")
envKey = strings.ToUpper(envKey)
secret := os.Getenv("WEBHOOK_" + envKey + "_SECRET")
if secret == "" {
secret = globalSecret
}
branch := os.Getenv("WEBHOOK_" + envKey + "_BRANCH")
if branch == "" {
branch = "main"
}
deployScript := filepath.Join(domainsDir, domain, "deploy.sh")
if _, err := os.Stat(deployScript); os.IsNotExist(err) {
altDeploy := filepath.Join(domainsDir, domain, "scripts", "deploy.sh")
if _, altErr := os.Stat(altDeploy); altErr == nil {
deployScript = altDeploy
}
}
domainConfig[domain] = DomainConfig{
Secret: secret,
TargetBranch: branch,
DeployScript: deployScript,
}
log.Printf("Configured domain: %s (branch: %s, secret: %v)", domain, branch, secret != "")
}
}
func main() {
port := os.Getenv("WEBHOOK_PORT")
if port == "" {
port = "9090"
}
loadConfig()
http.HandleFunc("/health", healthHandler)
http.HandleFunc("/deploy/", deployHandler)
http.HandleFunc("/", rootHandler)
log.Printf("Webhook service listening on :%s", port)
log.Fatal(http.ListenAndServe("127.0.0.1:"+port, nil))
}
func rootHandler(w http.ResponseWriter, r *http.Request) {
log.Printf("[REQUEST] %s %s from %s", r.Method, r.URL.Path, r.RemoteAddr)
if r.URL.Path == "/" {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"service": "web-hosts-webhook",
"status": "running",
"domains": getDomainList(),
})
return
}
http.NotFound(w, r)
}
func healthHandler(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("OK"))
}
func deployHandler(w http.ResponseWriter, r *http.Request) {
// Extract domain from path: /deploy/<domain>
path := strings.TrimPrefix(r.URL.Path, "/deploy/")
domain := strings.TrimSuffix(path, "/")
log.Printf("[DEPLOY] Request for domain: %s from %s", domain, r.RemoteAddr)
log.Printf("[HEADERS] %v", r.Header)
config, exists := domainConfig[domain]
if !exists {
log.Printf("[ERROR] Unknown domain: %s", domain)
http.Error(w, "Unknown domain", http.StatusNotFound)
return
}
// Check deploy script exists
if _, err := os.Stat(config.DeployScript); os.IsNotExist(err) {
log.Printf("[ERROR] Deploy script not found: %s", config.DeployScript)
http.Error(w, "Deploy script not found", http.StatusNotFound)
return
}
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
body, err := io.ReadAll(r.Body)
if err != nil {
log.Printf("[ERROR] Failed to read body: %v", err)
http.Error(w, "Failed to read body", http.StatusBadRequest)
return
}
log.Printf("[PAYLOAD] %s", string(body))
// Verify signature if secret is configured
if config.Secret != "" {
sig := r.Header.Get("X-Gitea-Signature")
if sig == "" {
sig = r.Header.Get("X-Hub-Signature-256")
if sig != "" {
sig = strings.TrimPrefix(sig, "sha256=")
}
}
if !verifySignature(body, sig, config.Secret) {
log.Printf("[REJECTED] Invalid signature for %s", domain)
http.Error(w, "Invalid signature", http.StatusUnauthorized)
return
}
}
// Parse payload
var payload GiteaPayload
if len(body) > 0 {
if err := json.Unmarshal(body, &payload); err != nil {
log.Printf("[WARN] Failed to parse payload: %v", err)
}
}
// Check branch
force := r.URL.Query().Get("force") == "true"
expectedRef := "refs/heads/" + config.TargetBranch
if payload.Ref != "" && payload.Ref != expectedRef && !force {
log.Printf("[SKIP] Branch %s != %s for %s", payload.Ref, expectedRef, domain)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]string{
"status": "skipped",
"message": fmt.Sprintf("Not %s branch", config.TargetBranch),
})
return
}
// Rate limit per domain
deployMutex.Lock()
if last, ok := lastDeploys[domain]; ok && time.Since(last) < 30*time.Second {
deployMutex.Unlock()
log.Printf("[RATE_LIMITED] %s - last deploy %v ago", domain, time.Since(last))
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]string{
"status": "rate_limited",
"message": "Please wait before deploying again",
})
return
}
lastDeploys[domain] = time.Now()
deployMutex.Unlock()
log.Printf("[DEPLOY] Triggering deploy for %s (repo: %s, pusher: %s)",
domain, payload.Repository.FullName, payload.Pusher.Username)
// Run deploy in background
go runDeploy(domain, config)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]string{
"status": "success",
"message": fmt.Sprintf("Deployment triggered for %s", domain),
})
}
func runDeploy(domain string, config DomainConfig) {
logFile := filepath.Join(logsDir, domain+"-webhook.log")
cmd := exec.Command(config.DeployScript)
cmd.Dir = filepath.Dir(config.DeployScript)
output, err := cmd.CombinedOutput()
if err != nil {
log.Printf("[DEPLOY_FAILED] %s: %v\n%s", domain, err, output)
} else {
log.Printf("[DEPLOY_SUCCESS] %s:\n%s", domain, output)
}
// Append to log file
f, _ := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if f != nil {
fmt.Fprintf(f, "\n=== Deploy %s ===\n%s\n", time.Now().Format(time.RFC3339), output)
f.Close()
}
}
func verifySignature(payload []byte, signature, secret string) bool {
if signature == "" {
return false
}
mac := hmac.New(sha256.New, []byte(secret))
mac.Write(payload)
expected := hex.EncodeToString(mac.Sum(nil))
return hmac.Equal([]byte(expected), []byte(signature))
}
func getDomainList() []string {
domains := make([]string, 0, len(domainConfig))
for d := range domainConfig {
domains = append(domains, d)
}
return domains
}

38
webhook/start.sh Executable file
View File

@@ -0,0 +1,38 @@
#!/bin/bash
# Start the webhook service
# This runs on the host (not in a container) to execute deploy scripts
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"
# Ensure Go is available in non-interactive environments (systemd)
GO_CMD="$(command -v go || true)"
if [ -z "$GO_CMD" ] && [ -x /usr/local/go/bin/go ]; then
GO_CMD="/usr/local/go/bin/go"
fi
if [ -z "$GO_CMD" ]; then
echo "Go not found in PATH and /usr/local/go/bin/go is missing" >&2
exit 1
fi
# Build if needed
if [ ! -f webhook-service ] || [ main.go -nt webhook-service ]; then
echo "Building webhook service..."
"$GO_CMD" build -o webhook-service main.go || exit 1
fi
# Load environment from .env if exists
if [ -f .env ]; then
export $(grep -v '^#' .env | xargs)
fi
# Default config
export WEBHOOK_PORT="${WEBHOOK_PORT:-9090}"
export WEBHOOK_TEST_COPPERTONE_TECH_BRANCH="${WEBHOOK_TEST_COPPERTONE_TECH_BRANCH:-testing}"
export WEBHOOK_CHUCKIE_COPPERTONE_TECH_BRANCH="${WEBHOOK_CHUCKIE_COPPERTONE_TECH_BRANCH:-main}"
export WEBHOOK_DEV_COPPERTONE_TECH_BRANCH="${WEBHOOK_DEV_COPPERTONE_TECH_BRANCH:-develop}"
export WEBHOOK_GAMES_COPPERTONE_TECH_BRANCH="${WEBHOOK_GAMES_COPPERTONE_TECH_BRANCH:-main}"
export WEBHOOK_COPPERTONE_TECH_BRANCH="${WEBHOOK_COPPERTONE_TECH_BRANCH:-main}"
echo "Starting webhook service on port $WEBHOOK_PORT..."
exec ./webhook-service