c54c569f30
Completed clean root directory structure: - Root now contains only: .git, .env, docs/, orig/ - Moved all remaining files and directories to orig/: - Config files (.claude, .dockerignore, .drone.yml, etc.) - All .env variants (except active .env) - Git config (.gitconfig, .github, .gitignore, etc.) - Tool configs (.golangci.yml, .revive.toml, etc.) - Documentation (*.md files, @prompts) - Build files (Dockerfiles, Makefile, go.mod, go.sum) - Docker compose files - All source directories (scripts, tests, tools, etc.) - Runtime directories (logs, monitoring, reports) - Dependency files (node_modules, lib, cache) - Special files (--delete) - Removed empty runtime directories (bin/, data/) V2 structure is now clean: - docs/planning/ - V2 planning documents - orig/ - Complete V1 codebase preserved - .env - Active environment config (not in git) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
834 lines
30 KiB
Bash
Executable File
834 lines
30 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# MEV Bot Production Log Manager
|
|
# Comprehensive log management with real-time monitoring, alerting, and analytics
|
|
|
|
set -euo pipefail
|
|
|
|
# Production Configuration
|
|
PROJECT_ROOT="/home/administrator/projects/mev-beta"
|
|
LOGS_DIR="$PROJECT_ROOT/logs"
|
|
ARCHIVE_DIR="$PROJECT_ROOT/logs/archives"
|
|
ANALYTICS_DIR="$PROJECT_ROOT/logs/analytics"
|
|
ALERTS_DIR="$PROJECT_ROOT/logs/alerts"
|
|
CONFIG_FILE="$PROJECT_ROOT/config/log-manager.conf"
|
|
|
|
# Default Configuration
|
|
DEFAULT_RETENTION_DAYS=30
|
|
DEFAULT_ARCHIVE_SIZE_LIMIT="10G"
|
|
DEFAULT_LOG_SIZE_LIMIT="1G"
|
|
DEFAULT_ERROR_THRESHOLD=100
|
|
DEFAULT_ALERT_EMAIL=""
|
|
DEFAULT_SLACK_WEBHOOK=""
|
|
DEFAULT_MONITORING_INTERVAL=60
|
|
|
|
# Colors and formatting
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
PURPLE='\033[0;35m'
|
|
CYAN='\033[0;36m'
|
|
BOLD='\033[1m'
|
|
NC='\033[0m'
|
|
|
|
# Performance metrics
|
|
declare -A METRICS=(
|
|
["archives_created"]=0
|
|
["logs_rotated"]=0
|
|
["alerts_sent"]=0
|
|
["errors_detected"]=0
|
|
["corruption_found"]=0
|
|
["performance_issues"]=0
|
|
)
|
|
|
|
# Initialize configuration
|
|
init_config() {
|
|
if [[ ! -f "$CONFIG_FILE" ]]; then
|
|
mkdir -p "$(dirname "$CONFIG_FILE")"
|
|
cat > "$CONFIG_FILE" << EOF
|
|
# MEV Bot Log Manager Configuration
|
|
RETENTION_DAYS=${DEFAULT_RETENTION_DAYS}
|
|
ARCHIVE_SIZE_LIMIT=${DEFAULT_ARCHIVE_SIZE_LIMIT}
|
|
LOG_SIZE_LIMIT=${DEFAULT_LOG_SIZE_LIMIT}
|
|
ERROR_THRESHOLD=${DEFAULT_ERROR_THRESHOLD}
|
|
ALERT_EMAIL=${DEFAULT_ALERT_EMAIL}
|
|
SLACK_WEBHOOK=${DEFAULT_SLACK_WEBHOOK}
|
|
MONITORING_INTERVAL=${DEFAULT_MONITORING_INTERVAL}
|
|
AUTO_ROTATE=true
|
|
AUTO_ANALYZE=true
|
|
AUTO_ALERT=true
|
|
COMPRESS_LEVEL=9
|
|
HEALTH_CHECK_ENABLED=true
|
|
PERFORMANCE_TRACKING=true
|
|
EOF
|
|
log "Created default configuration: $CONFIG_FILE"
|
|
fi
|
|
source "$CONFIG_FILE"
|
|
}
|
|
|
|
# Logging functions with levels
|
|
log() {
|
|
echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] INFO:${NC} $1" | tee -a "$LOGS_DIR/log-manager.log"
|
|
}
|
|
|
|
warn() {
|
|
echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARN:${NC} $1" | tee -a "$LOGS_DIR/log-manager.log"
|
|
}
|
|
|
|
error() {
|
|
echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR:${NC} $1" | tee -a "$LOGS_DIR/log-manager.log"
|
|
((METRICS["errors_detected"]++))
|
|
}
|
|
|
|
success() {
|
|
echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] SUCCESS:${NC} $1" | tee -a "$LOGS_DIR/log-manager.log"
|
|
}
|
|
|
|
debug() {
|
|
if [[ "${DEBUG:-false}" == "true" ]]; then
|
|
echo -e "${CYAN}[$(date +'%Y-%m-%d %H:%M:%S')] DEBUG:${NC} $1" | tee -a "$LOGS_DIR/log-manager.log"
|
|
fi
|
|
}
|
|
|
|
# Create directory structure
|
|
setup_directories() {
|
|
local dirs=("$ARCHIVE_DIR" "$ANALYTICS_DIR" "$ALERTS_DIR" "$LOGS_DIR/rotated" "$LOGS_DIR/health")
|
|
for dir in "${dirs[@]}"; do
|
|
if [[ ! -d "$dir" ]]; then
|
|
mkdir -p "$dir"
|
|
debug "Created directory: $dir"
|
|
fi
|
|
done
|
|
}
|
|
|
|
# Enhanced log rotation with size and time-based triggers
|
|
rotate_logs() {
|
|
log "Starting intelligent log rotation..."
|
|
|
|
local rotated_count=0
|
|
local timestamp=$(date +"%Y%m%d_%H%M%S")
|
|
|
|
# Find logs that need rotation
|
|
while IFS= read -r -d '' logfile; do
|
|
local filename=$(basename "$logfile")
|
|
local size=$(stat -c%s "$logfile" 2>/dev/null || echo 0)
|
|
local size_mb=$((size / 1024 / 1024))
|
|
|
|
# Check if rotation is needed (size > limit or age > 24h)
|
|
local needs_rotation=false
|
|
|
|
if [[ $size -gt $(numfmt --from=iec "${LOG_SIZE_LIMIT}") ]]; then
|
|
needs_rotation=true
|
|
debug "Log $filename needs rotation: size ${size_mb}MB exceeds limit"
|
|
fi
|
|
|
|
if [[ $(find "$logfile" -mtime +0 -print 2>/dev/null) ]]; then
|
|
needs_rotation=true
|
|
debug "Log $filename needs rotation: older than 24 hours"
|
|
fi
|
|
|
|
if [[ "$needs_rotation" == "true" ]]; then
|
|
local rotated_name="${filename%.log}_${timestamp}.log"
|
|
mv "$logfile" "$LOGS_DIR/rotated/$rotated_name"
|
|
gzip "$LOGS_DIR/rotated/$rotated_name"
|
|
touch "$logfile" # Create fresh log file
|
|
((rotated_count++))
|
|
log "Rotated $filename -> ${rotated_name}.gz (${size_mb}MB)"
|
|
fi
|
|
done < <(find "$LOGS_DIR" -maxdepth 1 -name "*.log" -type f -print0)
|
|
|
|
METRICS["logs_rotated"]=$rotated_count
|
|
success "Log rotation completed: $rotated_count files rotated"
|
|
}
|
|
|
|
# Real-time log analysis with pattern detection
|
|
analyze_logs() {
|
|
log "Starting comprehensive log analysis..."
|
|
|
|
local analysis_file="$ANALYTICS_DIR/analysis_$(date +%Y%m%d_%H%M%S).json"
|
|
local main_log="$LOGS_DIR/mev_bot.log"
|
|
|
|
if [[ ! -f "$main_log" ]]; then
|
|
warn "Main log file not found: $main_log"
|
|
return 1
|
|
fi
|
|
|
|
# Performance metrics extraction
|
|
local total_lines=$(wc -l < "$main_log")
|
|
local error_lines=$(grep -c "ERROR" "$main_log" || echo 0)
|
|
local warn_lines=$(grep -c "WARN" "$main_log" || echo 0)
|
|
local success_lines=$(grep -c "SUCCESS\|✅" "$main_log" || echo 0)
|
|
|
|
# MEV-specific metrics
|
|
local opportunities=$(grep -c "opportunity" "$main_log" || echo 0)
|
|
local rejections=$(grep -c "REJECTED" "$main_log" || echo 0)
|
|
local parsing_failures=$(grep -c "PARSING FAILED" "$main_log" || echo 0)
|
|
local direct_parsing=$(grep -c "DIRECT PARSING" "$main_log" || echo 0)
|
|
|
|
# Transaction processing metrics
|
|
local blocks_processed=$(grep -c "Block.*Processing.*transactions" "$main_log" || echo 0)
|
|
local dex_transactions=$(grep -c "DEX transactions" "$main_log" || echo 0)
|
|
|
|
# Error pattern analysis
|
|
local zero_address_issues=$(grep -c "zero.*address" "$main_log" || echo 0)
|
|
local connection_errors=$(grep -c "connection.*failed\|context.*canceled" "$main_log" || echo 0)
|
|
local timeout_errors=$(grep -c "timeout\|deadline exceeded" "$main_log" || echo 0)
|
|
|
|
# Performance trending (last 1000 lines for recent activity)
|
|
local recent_errors=$(tail -1000 "$main_log" | grep -c "ERROR" || echo 0)
|
|
local recent_success=$(tail -1000 "$main_log" | grep -c "SUCCESS" || echo 0)
|
|
|
|
# Calculate rates and health scores
|
|
local error_rate=$(echo "scale=2; $error_lines * 100 / $total_lines" | bc -l 2>/dev/null || echo 0)
|
|
local success_rate=$(echo "scale=2; $success_lines * 100 / $total_lines" | bc -l 2>/dev/null || echo 0)
|
|
local health_score=$(echo "scale=0; 100 - $error_rate" | bc -l 2>/dev/null || echo 100)
|
|
|
|
# Generate comprehensive analysis
|
|
cat > "$analysis_file" << EOF
|
|
{
|
|
"analysis_timestamp": "$(date -Iseconds)",
|
|
"log_file": "$main_log",
|
|
"system_info": {
|
|
"hostname": "$(hostname)",
|
|
"uptime": "$(uptime -p 2>/dev/null || echo 'unknown')",
|
|
"load_average": "$(uptime | awk -F'load average:' '{print $2}' | xargs)"
|
|
},
|
|
"log_statistics": {
|
|
"total_lines": $total_lines,
|
|
"file_size_mb": $(echo "scale=2; $(stat -c%s "$main_log") / 1024 / 1024" | bc -l),
|
|
"error_lines": $error_lines,
|
|
"warning_lines": $warn_lines,
|
|
"success_lines": $success_lines,
|
|
"error_rate_percent": $error_rate,
|
|
"success_rate_percent": $success_rate,
|
|
"health_score": $health_score
|
|
},
|
|
"mev_metrics": {
|
|
"opportunities_detected": $opportunities,
|
|
"events_rejected": $rejections,
|
|
"parsing_failures": $parsing_failures,
|
|
"direct_parsing_attempts": $direct_parsing,
|
|
"blocks_processed": $blocks_processed,
|
|
"dex_transactions": $dex_transactions
|
|
},
|
|
"error_patterns": {
|
|
"zero_address_issues": $zero_address_issues,
|
|
"connection_errors": $connection_errors,
|
|
"timeout_errors": $timeout_errors
|
|
},
|
|
"recent_activity": {
|
|
"recent_errors": $recent_errors,
|
|
"recent_success": $recent_success,
|
|
"recent_health_trend": "$([ -n "${recent_errors}" ] && [ "${recent_errors}" -lt 10 ] 2>/dev/null && echo good || echo concerning)"
|
|
},
|
|
"alerts_triggered": []
|
|
}
|
|
EOF
|
|
|
|
# Check for alert conditions
|
|
check_alert_conditions "$analysis_file"
|
|
|
|
success "Log analysis completed: $analysis_file"
|
|
echo -e "${BLUE}Health Score: $health_score/100${NC} | Error Rate: ${error_rate}% | Success Rate: ${success_rate}%"
|
|
}
|
|
|
|
# Alert system with multiple notification channels
|
|
check_alert_conditions() {
|
|
local analysis_file="$1"
|
|
local alerts_triggered=()
|
|
|
|
# Read analysis data
|
|
local error_rate=$(jq -r '.log_statistics.error_rate_percent' "$analysis_file" 2>/dev/null || echo 0)
|
|
local health_score=$(jq -r '.log_statistics.health_score' "$analysis_file" 2>/dev/null || echo 100)
|
|
local parsing_failures=$(jq -r '.mev_metrics.parsing_failures' "$analysis_file" 2>/dev/null || echo 0)
|
|
local zero_address_issues=$(jq -r '.error_patterns.zero_address_issues' "$analysis_file" 2>/dev/null || echo 0)
|
|
|
|
# Define alert conditions
|
|
if (( $(echo "$error_rate > 10" | bc -l) )); then
|
|
alerts_triggered+=("HIGH_ERROR_RATE:$error_rate%")
|
|
send_alert "High Error Rate" "Error rate is $error_rate%, exceeding 10% threshold"
|
|
fi
|
|
|
|
if (( $(echo "$health_score < 80" | bc -l) )); then
|
|
alerts_triggered+=("LOW_HEALTH_SCORE:$health_score")
|
|
send_alert "Low Health Score" "System health score is $health_score/100, below 80 threshold"
|
|
fi
|
|
|
|
if (( parsing_failures > 50 )); then
|
|
alerts_triggered+=("PARSING_FAILURES:$parsing_failures")
|
|
send_alert "High Parsing Failures" "$parsing_failures parsing failures detected"
|
|
fi
|
|
|
|
if (( zero_address_issues > 100 )); then
|
|
alerts_triggered+=("ZERO_ADDRESS_CORRUPTION:$zero_address_issues")
|
|
send_alert "Address Corruption" "$zero_address_issues zero address issues detected"
|
|
fi
|
|
|
|
# Update analysis file with alerts
|
|
if [[ ${#alerts_triggered[@]} -gt 0 ]]; then
|
|
local alerts_json=$(printf '%s\n' "${alerts_triggered[@]}" | jq -R . | jq -s .)
|
|
jq ".alerts_triggered = $alerts_json" "$analysis_file" > "${analysis_file}.tmp" && mv "${analysis_file}.tmp" "$analysis_file"
|
|
METRICS["alerts_sent"]=${#alerts_triggered[@]}
|
|
fi
|
|
}
|
|
|
|
# Multi-channel alert delivery
|
|
send_alert() {
|
|
local title="$1"
|
|
local message="$2"
|
|
local timestamp=$(date -Iseconds)
|
|
local alert_file="$ALERTS_DIR/alert_$(date +%Y%m%d_%H%M%S).json"
|
|
|
|
# Create alert record
|
|
cat > "$alert_file" << EOF
|
|
{
|
|
"timestamp": "$timestamp",
|
|
"title": "$title",
|
|
"message": "$message",
|
|
"hostname": "$(hostname)",
|
|
"severity": "warning",
|
|
"system_load": "$(uptime | awk -F'load average:' '{print $2}' | xargs)",
|
|
"disk_usage": "$(df -h $LOGS_DIR | tail -1 | awk '{print $5}')"
|
|
}
|
|
EOF
|
|
|
|
error "ALERT: $title - $message"
|
|
|
|
# Email notification
|
|
if [[ -n "${ALERT_EMAIL:-}" ]] && command -v mail >/dev/null 2>&1; then
|
|
echo "MEV Bot Alert: $title - $message ($(hostname) at $timestamp)" | mail -s "MEV Bot Alert: $title" "$ALERT_EMAIL"
|
|
fi
|
|
|
|
# Slack notification
|
|
if [[ -n "${SLACK_WEBHOOK:-}" ]] && command -v curl >/dev/null 2>&1; then
|
|
curl -X POST -H 'Content-type: application/json' \
|
|
--data "{\"text\":\"🚨 MEV Bot Alert: $title\n$message\nHost: $(hostname)\nTime: $timestamp\"}" \
|
|
"$SLACK_WEBHOOK" >/dev/null 2>&1 || true
|
|
fi
|
|
}
|
|
|
|
# Log corruption detection and health checks
|
|
health_check() {
|
|
log "Running comprehensive health checks..."
|
|
|
|
local health_report="$LOGS_DIR/health/health_$(date +%Y%m%d_%H%M%S).json"
|
|
local issues=()
|
|
|
|
# Check log file integrity
|
|
while IFS= read -r -d '' logfile; do
|
|
if [[ ! -r "$logfile" ]]; then
|
|
issues+=("UNREADABLE_LOG:$(basename "$logfile")")
|
|
continue
|
|
fi
|
|
|
|
# Check for truncated logs
|
|
if [[ $(tail -c 1 "$logfile" | wc -l) -eq 0 ]]; then
|
|
issues+=("TRUNCATED_LOG:$(basename "$logfile")")
|
|
fi
|
|
|
|
# Check for corruption patterns
|
|
if grep -q "\x00" "$logfile" 2>/dev/null; then
|
|
issues+=("NULL_BYTES:$(basename "$logfile")")
|
|
((METRICS["corruption_found"]++))
|
|
fi
|
|
|
|
# Check for encoding issues
|
|
if ! file "$logfile" | grep -q "text"; then
|
|
issues+=("ENCODING_ISSUE:$(basename "$logfile")")
|
|
fi
|
|
|
|
done < <(find "$LOGS_DIR" -maxdepth 1 -name "*.log" -type f -print0)
|
|
|
|
# Check disk space
|
|
local disk_usage=$(df "$LOGS_DIR" | tail -1 | awk '{print $5}' | sed 's/%//')
|
|
if (( disk_usage > 90 )); then
|
|
issues+=("HIGH_DISK_USAGE:${disk_usage}%")
|
|
send_alert "High Disk Usage" "Log directory is ${disk_usage}% full"
|
|
fi
|
|
|
|
# Check archive integrity
|
|
while IFS= read -r -d '' archive; do
|
|
if ! tar -tzf "$archive" >/dev/null 2>&1; then
|
|
issues+=("CORRUPTED_ARCHIVE:$(basename "$archive")")
|
|
((METRICS["corruption_found"]++))
|
|
fi
|
|
done < <(find "$ARCHIVE_DIR" -name "*.tar.gz" -type f -print0 2>/dev/null)
|
|
|
|
# Generate health report
|
|
local health_status="healthy"
|
|
if [[ ${#issues[@]} -gt 0 ]]; then
|
|
health_status="issues_detected"
|
|
fi
|
|
|
|
cat > "$health_report" << EOF
|
|
{
|
|
"timestamp": "$(date -Iseconds)",
|
|
"status": "$health_status",
|
|
"issues_count": ${#issues[@]},
|
|
"issues": $(printf '%s\n' "${issues[@]}" | jq -R . | jq -s . 2>/dev/null || echo '[]'),
|
|
"disk_usage_percent": $disk_usage,
|
|
"log_files_count": $(find "$LOGS_DIR" -maxdepth 1 -name "*.log" -type f | wc -l),
|
|
"archive_files_count": $(find "$ARCHIVE_DIR" -name "*.tar.gz" -type f 2>/dev/null | wc -l),
|
|
"total_log_size_mb": $(du -sm "$LOGS_DIR" | cut -f1),
|
|
"system_load": "$(uptime | awk -F'load average:' '{print $2}' | xargs)"
|
|
}
|
|
EOF
|
|
|
|
if [[ ${#issues[@]} -eq 0 ]]; then
|
|
success "Health check passed: No issues detected"
|
|
else
|
|
warn "Health check found ${#issues[@]} issues: ${issues[*]}"
|
|
fi
|
|
|
|
echo "$health_report"
|
|
}
|
|
|
|
# Performance monitoring with trending
|
|
monitor_performance() {
|
|
log "Monitoring system performance..."
|
|
|
|
local perf_file="$ANALYTICS_DIR/performance_$(date +%Y%m%d_%H%M%S).json"
|
|
|
|
# System metrics
|
|
local cpu_usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | sed 's/%us,//')
|
|
local memory_usage=$(free | grep Mem | awk '{printf("%.1f", $3/$2 * 100.0)}')
|
|
local load_avg=$(uptime | awk -F'load average:' '{print $2}' | awk '{print $1}' | sed 's/,//')
|
|
|
|
# MEV bot specific metrics
|
|
local mev_processes=$(pgrep -f mev-bot | wc -l)
|
|
local mev_memory=0
|
|
if [[ $mev_processes -gt 0 ]]; then
|
|
mev_memory=$(pgrep -f mev-bot | xargs ps -o pid,rss --no-headers | awk '{sum+=$2} END {print sum/1024}' 2>/dev/null || echo 0)
|
|
fi
|
|
|
|
# Log processing rate
|
|
local log_lines_per_min=0
|
|
if [[ -f "$LOGS_DIR/mev_bot.log" ]]; then
|
|
log_lines_per_min=$(tail -100 "$LOGS_DIR/mev_bot.log" | grep "$(date '+%Y/%m/%d %H:%M')" | wc -l || echo 0)
|
|
fi
|
|
|
|
cat > "$perf_file" << EOF
|
|
{
|
|
"timestamp": "$(date -Iseconds)",
|
|
"system_metrics": {
|
|
"cpu_usage_percent": $cpu_usage,
|
|
"memory_usage_percent": $memory_usage,
|
|
"load_average": $load_avg,
|
|
"uptime_seconds": $(awk '{print int($1)}' /proc/uptime)
|
|
},
|
|
"mev_bot_metrics": {
|
|
"process_count": $mev_processes,
|
|
"memory_usage_mb": $mev_memory,
|
|
"log_rate_lines_per_min": $log_lines_per_min
|
|
},
|
|
"log_metrics": {
|
|
"total_log_size_mb": $(du -sm "$LOGS_DIR" | cut -f1),
|
|
"archive_size_mb": $(du -sm "$ARCHIVE_DIR" 2>/dev/null | cut -f1 || echo 0),
|
|
"active_log_files": $(find "$LOGS_DIR" -maxdepth 1 -name "*.log" -type f | wc -l)
|
|
}
|
|
}
|
|
EOF
|
|
|
|
# Check for performance issues
|
|
if (( $(echo "$cpu_usage > 80" | bc -l) )); then
|
|
((METRICS["performance_issues"]++))
|
|
send_alert "High CPU Usage" "CPU usage is ${cpu_usage}%"
|
|
fi
|
|
|
|
if (( $(echo "$memory_usage > 85" | bc -l) )); then
|
|
((METRICS["performance_issues"]++))
|
|
send_alert "High Memory Usage" "Memory usage is ${memory_usage}%"
|
|
fi
|
|
|
|
debug "Performance monitoring completed: $perf_file"
|
|
}
|
|
|
|
# Advanced archiving with compression optimization
|
|
advanced_archive() {
|
|
log "Starting advanced archive process..."
|
|
|
|
local timestamp=$(date +"%Y%m%d_%H%M%S")
|
|
local archive_name="mev_logs_${timestamp}"
|
|
local temp_dir="$ARCHIVE_DIR/.tmp_$archive_name"
|
|
|
|
mkdir -p "$temp_dir"
|
|
|
|
# Copy logs with metadata preservation
|
|
find "$LOGS_DIR" -maxdepth 1 -name "*.log" -type f -exec cp -p {} "$temp_dir/" \;
|
|
|
|
# Copy rotated logs
|
|
if [[ -d "$LOGS_DIR/rotated" ]]; then
|
|
cp -r "$LOGS_DIR/rotated" "$temp_dir/"
|
|
fi
|
|
|
|
# Copy analytics and health data
|
|
if [[ -d "$ANALYTICS_DIR" ]]; then
|
|
cp -r "$ANALYTICS_DIR" "$temp_dir/"
|
|
fi
|
|
|
|
if [[ -d "$ALERTS_DIR" ]]; then
|
|
cp -r "$ALERTS_DIR" "$temp_dir/"
|
|
fi
|
|
|
|
# Generate comprehensive metadata
|
|
cat > "$temp_dir/archive_metadata.json" << EOF
|
|
{
|
|
"archive_info": {
|
|
"timestamp": "$(date -Iseconds)",
|
|
"archive_name": "$archive_name",
|
|
"created_by": "$(whoami)",
|
|
"hostname": "$(hostname)",
|
|
"mev_bot_version": "$(git rev-parse HEAD 2>/dev/null || echo 'unknown')",
|
|
"git_branch": "$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo 'unknown')",
|
|
"compression_level": ${COMPRESS_LEVEL:-9}
|
|
},
|
|
"system_snapshot": {
|
|
"os": "$(uname -s)",
|
|
"kernel": "$(uname -r)",
|
|
"architecture": "$(uname -m)",
|
|
"uptime": "$(uptime -p 2>/dev/null || echo 'unknown')",
|
|
"load_average": "$(uptime | awk -F'load average:' '{print $2}' | xargs)",
|
|
"memory_total_gb": $(echo "scale=2; $(grep MemTotal /proc/meminfo | awk '{print $2}') / 1024 / 1024" | bc -l),
|
|
"disk_space_logs": "$(df -h $LOGS_DIR | tail -1 | awk '{print $4}')"
|
|
},
|
|
"content_summary": {
|
|
"total_files": $(find "$temp_dir" -type f | wc -l),
|
|
"total_size_bytes": $(find "$temp_dir" -type f -exec stat -c%s {} + | awk '{sum+=$1} END {print sum+0}'),
|
|
"log_files": $(find "$temp_dir" -name "*.log" | wc -l),
|
|
"compressed_files": $(find "$temp_dir" -name "*.gz" | wc -l)
|
|
},
|
|
"metrics": $(echo "${METRICS[@]}" | tr ' ' '\n' | awk -F= '{print "\"" $1 "\":" $2}' | paste -sd, | sed 's/^/{/' | sed 's/$/}/')
|
|
}
|
|
EOF
|
|
|
|
# Create optimized archive
|
|
cd "$ARCHIVE_DIR"
|
|
tar -cf "${archive_name}.tar.gz" --use-compress-program="gzip -${COMPRESS_LEVEL:-9}" -C "$(dirname "$temp_dir")" "$(basename "$temp_dir")"
|
|
|
|
# Verify archive integrity
|
|
if tar -tzf "${archive_name}.tar.gz" >/dev/null 2>&1; then
|
|
local archive_size=$(stat -c%s "${archive_name}.tar.gz" | numfmt --to=iec)
|
|
success "Archive created successfully: ${archive_name}.tar.gz ($archive_size)"
|
|
|
|
# Update symlink
|
|
ln -sf "${archive_name}.tar.gz" "latest_archive.tar.gz"
|
|
|
|
# Cleanup temp directory
|
|
rm -rf "$temp_dir"
|
|
|
|
((METRICS["archives_created"]++))
|
|
else
|
|
error "Archive verification failed: ${archive_name}.tar.gz"
|
|
rm -f "${archive_name}.tar.gz"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Cleanup with advanced retention policies
|
|
intelligent_cleanup() {
|
|
log "Starting intelligent cleanup with retention policies..."
|
|
|
|
local deleted_archives=0
|
|
local deleted_size=0
|
|
|
|
# Archive retention by age
|
|
while IFS= read -r -d '' archive; do
|
|
local size=$(stat -c%s "$archive")
|
|
rm "$archive"
|
|
((deleted_archives++))
|
|
deleted_size=$((deleted_size + size))
|
|
debug "Deleted old archive: $(basename "$archive")"
|
|
done < <(find "$ARCHIVE_DIR" -name "mev_logs_*.tar.gz" -mtime +${RETENTION_DAYS} -print0 2>/dev/null)
|
|
|
|
# Size-based cleanup if total exceeds limit
|
|
local total_size=$(du -sb "$ARCHIVE_DIR" 2>/dev/null | cut -f1 || echo 0)
|
|
local size_limit=$(numfmt --from=iec "${ARCHIVE_SIZE_LIMIT}")
|
|
|
|
if [[ $total_size -gt $size_limit ]]; then
|
|
warn "Archive directory exceeds size limit, cleaning oldest archives..."
|
|
while [[ $total_size -gt $size_limit ]] && [[ $(find "$ARCHIVE_DIR" -name "mev_logs_*.tar.gz" | wc -l) -gt 1 ]]; do
|
|
local oldest=$(find "$ARCHIVE_DIR" -name "mev_logs_*.tar.gz" -printf '%T+ %p\n' | sort | head -1 | cut -d' ' -f2)
|
|
if [[ -f "$oldest" ]]; then
|
|
local size=$(stat -c%s "$oldest")
|
|
rm "$oldest"
|
|
((deleted_archives++))
|
|
deleted_size=$((deleted_size + size))
|
|
total_size=$((total_size - size))
|
|
debug "Deleted for size limit: $(basename "$oldest")"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# Cleanup analytics and alerts older than retention period
|
|
find "$ANALYTICS_DIR" -name "*.json" -mtime +${RETENTION_DAYS} -delete 2>/dev/null || true
|
|
find "$ALERTS_DIR" -name "*.json" -mtime +${RETENTION_DAYS} -delete 2>/dev/null || true
|
|
find "$LOGS_DIR/health" -name "*.json" -mtime +${RETENTION_DAYS} -delete 2>/dev/null || true
|
|
|
|
if [[ $deleted_archives -gt 0 ]]; then
|
|
local deleted_size_human=$(echo $deleted_size | numfmt --to=iec)
|
|
success "Cleanup completed: $deleted_archives archives deleted ($deleted_size_human freed)"
|
|
else
|
|
log "Cleanup completed: No files needed deletion"
|
|
fi
|
|
}
|
|
|
|
# Real-time monitoring daemon
|
|
start_monitoring() {
|
|
log "Starting real-time monitoring daemon..."
|
|
|
|
local monitor_pid_file="$LOGS_DIR/.monitor.pid"
|
|
|
|
if [[ -f "$monitor_pid_file" ]] && kill -0 $(cat "$monitor_pid_file") 2>/dev/null; then
|
|
warn "Monitoring daemon already running (PID: $(cat "$monitor_pid_file"))"
|
|
return 1
|
|
fi
|
|
|
|
# Background monitoring loop
|
|
(
|
|
echo $$ > "$monitor_pid_file"
|
|
while true; do
|
|
sleep "${MONITORING_INTERVAL}"
|
|
|
|
# Quick health check
|
|
if [[ "${HEALTH_CHECK_ENABLED}" == "true" ]]; then
|
|
health_check >/dev/null 2>&1
|
|
fi
|
|
|
|
# Performance monitoring
|
|
if [[ "${PERFORMANCE_TRACKING}" == "true" ]]; then
|
|
monitor_performance >/dev/null 2>&1
|
|
fi
|
|
|
|
# Auto-rotation check
|
|
if [[ "${AUTO_ROTATE}" == "true" ]]; then
|
|
local needs_rotation=$(find "$LOGS_DIR" -maxdepth 1 -name "*.log" -size +${LOG_SIZE_LIMIT} | wc -l)
|
|
if [[ $needs_rotation -gt 0 ]]; then
|
|
rotate_logs >/dev/null 2>&1
|
|
fi
|
|
fi
|
|
|
|
# Auto-analysis
|
|
if [[ "${AUTO_ANALYZE}" == "true" ]]; then
|
|
analyze_logs >/dev/null 2>&1
|
|
fi
|
|
|
|
done
|
|
) &
|
|
|
|
local daemon_pid=$!
|
|
echo "$daemon_pid" > "$monitor_pid_file"
|
|
success "Monitoring daemon started (PID: $daemon_pid, interval: ${MONITORING_INTERVAL}s)"
|
|
}
|
|
|
|
# Stop monitoring daemon
|
|
stop_monitoring() {
|
|
local monitor_pid_file="$LOGS_DIR/.monitor.pid"
|
|
|
|
if [[ -f "$monitor_pid_file" ]]; then
|
|
local pid=$(cat "$monitor_pid_file")
|
|
if kill -0 "$pid" 2>/dev/null; then
|
|
kill "$pid"
|
|
rm "$monitor_pid_file"
|
|
success "Monitoring daemon stopped (PID: $pid)"
|
|
else
|
|
warn "Monitoring daemon not running (stale PID file)"
|
|
rm "$monitor_pid_file"
|
|
fi
|
|
else
|
|
warn "Monitoring daemon not running"
|
|
fi
|
|
}
|
|
|
|
# Dashboard generation
|
|
generate_dashboard() {
|
|
log "Generating operational dashboard..."
|
|
|
|
local dashboard_file="$ANALYTICS_DIR/dashboard_$(date +%Y%m%d_%H%M%S).html"
|
|
local latest_analysis=$(find "$ANALYTICS_DIR" -name "analysis_*.json" -type f | sort | tail -1)
|
|
local latest_health=$(find "$LOGS_DIR/health" -name "health_*.json" -type f | sort | tail -1)
|
|
local latest_performance=$(find "$ANALYTICS_DIR" -name "performance_*.json" -type f | sort | tail -1)
|
|
|
|
cat > "$dashboard_file" << 'EOF'
|
|
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>MEV Bot Operations Dashboard</title>
|
|
<style>
|
|
body { font-family: 'Segoe UI', Arial, sans-serif; margin: 0; padding: 20px; background: #f5f5f5; }
|
|
.dashboard { max-width: 1200px; margin: 0 auto; }
|
|
.header { background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; padding: 20px; border-radius: 10px; margin-bottom: 20px; }
|
|
.metrics-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); gap: 20px; margin-bottom: 20px; }
|
|
.metric-card { background: white; border-radius: 10px; padding: 20px; box-shadow: 0 2px 10px rgba(0,0,0,0.1); }
|
|
.metric-value { font-size: 2em; font-weight: bold; margin: 10px 0; }
|
|
.metric-label { color: #666; font-size: 0.9em; }
|
|
.status-good { color: #28a745; }
|
|
.status-warning { color: #ffc107; }
|
|
.status-error { color: #dc3545; }
|
|
.chart-container { background: white; border-radius: 10px; padding: 20px; margin-bottom: 20px; }
|
|
.log-preview { background: #1e1e1e; color: #fff; padding: 15px; border-radius: 5px; font-family: 'Courier New', monospace; font-size: 0.8em; max-height: 300px; overflow-y: auto; }
|
|
.timestamp { color: #888; font-size: 0.8em; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="dashboard">
|
|
<div class="header">
|
|
<h1>MEV Bot Operations Dashboard</h1>
|
|
<p class="timestamp">Generated: $(date)</p>
|
|
</div>
|
|
EOF
|
|
|
|
# Add metrics if analysis file exists
|
|
if [[ -f "$latest_analysis" ]]; then
|
|
local health_score=$(jq -r '.log_statistics.health_score' "$latest_analysis" 2>/dev/null || echo 0)
|
|
local error_rate=$(jq -r '.log_statistics.error_rate_percent' "$latest_analysis" 2>/dev/null || echo 0)
|
|
local opportunities=$(jq -r '.mev_metrics.opportunities_detected' "$latest_analysis" 2>/dev/null || echo 0)
|
|
|
|
cat >> "$dashboard_file" << EOF
|
|
<div class="metrics-grid">
|
|
<div class="metric-card">
|
|
<div class="metric-label">System Health Score</div>
|
|
<div class="metric-value status-$([ $(echo "$health_score > 80" | bc -l) -eq 1 ] && echo 'good' || echo 'warning')">${health_score}/100</div>
|
|
</div>
|
|
<div class="metric-card">
|
|
<div class="metric-label">Error Rate</div>
|
|
<div class="metric-value status-$([ $(echo "$error_rate < 5" | bc -l) -eq 1 ] && echo 'good' || echo 'warning')">${error_rate}%</div>
|
|
</div>
|
|
<div class="metric-card">
|
|
<div class="metric-label">MEV Opportunities</div>
|
|
<div class="metric-value status-good">${opportunities}</div>
|
|
</div>
|
|
</div>
|
|
EOF
|
|
fi
|
|
|
|
# Add recent log entries
|
|
cat >> "$dashboard_file" << EOF
|
|
<div class="chart-container">
|
|
<h3>Recent Log Activity</h3>
|
|
<div class="log-preview">$(tail -20 "$LOGS_DIR/mev_bot.log" 2>/dev/null | sed 's/&/\&/g; s/</\</g; s/>/\>/g' || echo 'No recent log activity')</div>
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
EOF
|
|
|
|
success "Dashboard generated: $dashboard_file"
|
|
echo "$dashboard_file"
|
|
}
|
|
|
|
# Main command dispatcher
|
|
main() {
|
|
case "${1:-help}" in
|
|
"init")
|
|
setup_directories
|
|
init_config
|
|
success "Log manager initialized"
|
|
;;
|
|
"rotate")
|
|
init_config
|
|
setup_directories
|
|
rotate_logs
|
|
;;
|
|
"analyze")
|
|
init_config
|
|
setup_directories
|
|
analyze_logs
|
|
;;
|
|
"archive")
|
|
init_config
|
|
setup_directories
|
|
advanced_archive
|
|
;;
|
|
"health")
|
|
init_config
|
|
setup_directories
|
|
health_check
|
|
;;
|
|
"monitor")
|
|
init_config
|
|
setup_directories
|
|
monitor_performance
|
|
;;
|
|
"cleanup")
|
|
init_config
|
|
setup_directories
|
|
intelligent_cleanup
|
|
;;
|
|
"start-daemon")
|
|
init_config
|
|
setup_directories
|
|
start_monitoring
|
|
;;
|
|
"stop-daemon")
|
|
stop_monitoring
|
|
;;
|
|
"dashboard")
|
|
init_config
|
|
setup_directories
|
|
generate_dashboard
|
|
;;
|
|
"full")
|
|
init_config
|
|
setup_directories
|
|
rotate_logs
|
|
analyze_logs
|
|
health_check
|
|
monitor_performance
|
|
advanced_archive
|
|
intelligent_cleanup
|
|
generate_dashboard
|
|
;;
|
|
"status")
|
|
init_config
|
|
echo -e "${BOLD}MEV Bot Log Manager Status${NC}"
|
|
echo "Configuration: $CONFIG_FILE"
|
|
echo "Monitoring: $([ -f "$LOGS_DIR/.monitor.pid" ] && echo "Running (PID: $(cat "$LOGS_DIR/.monitor.pid"))" || echo "Stopped")"
|
|
echo "Archives: $(find "$ARCHIVE_DIR" -name "*.tar.gz" 2>/dev/null | wc -l) files"
|
|
echo "Total archive size: $(du -sh "$ARCHIVE_DIR" 2>/dev/null | cut -f1 || echo "0")"
|
|
echo "Log directory size: $(du -sh "$LOGS_DIR" | cut -f1)"
|
|
;;
|
|
*)
|
|
cat << EOF
|
|
MEV Bot Production Log Manager
|
|
|
|
USAGE:
|
|
$0 <command> [options]
|
|
|
|
COMMANDS:
|
|
init Initialize log manager with directories and config
|
|
rotate Rotate large log files
|
|
analyze Perform comprehensive log analysis
|
|
archive Create compressed archive with metadata
|
|
health Run health checks and corruption detection
|
|
monitor Generate performance monitoring report
|
|
cleanup Clean old archives based on retention policy
|
|
start-daemon Start real-time monitoring daemon
|
|
stop-daemon Stop monitoring daemon
|
|
dashboard Generate HTML operations dashboard
|
|
full Run complete log management cycle
|
|
status Show current system status
|
|
|
|
EXAMPLES:
|
|
$0 init # First-time setup
|
|
$0 full # Complete log management cycle
|
|
$0 start-daemon # Start background monitoring
|
|
$0 dashboard # Generate operations dashboard
|
|
|
|
CONFIGURATION:
|
|
Edit $CONFIG_FILE to customize behavior
|
|
|
|
MONITORING:
|
|
The daemon provides real-time monitoring with configurable intervals,
|
|
automatic rotation, health checks, and alerting via email/Slack.
|
|
EOF
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Initialize and run
|
|
cd "$PROJECT_ROOT" 2>/dev/null || { error "Invalid project root: $PROJECT_ROOT"; exit 1; }
|
|
main "$@"
|