fix(critical): complete execution pipeline - all blockers fixed and operational

2025-11-04 10:24:34 -06:00
parent 0b1c7bbc86
commit 52d555ccdf
410 changed files with 99504 additions and 28488 deletions
--- a/scripts/deprecated/README.md
+++ b/scripts/deprecated/README.md
@@ -0,0 +1,68 @@
+# Deprecated Scripts
+
+These scripts have been moved here because their functionality is now handled by more comprehensive tools.
+
+## Log Management Scripts (Superseded by log-manager.sh)
+
+All of these scripts have been replaced by `scripts/log-manager.sh`, which provides:
+
+- Real-time analysis with health scoring
+- Performance monitoring with MEV-specific metrics
+- Corruption detection and integrity validation
+- Multi-channel alerting (email, Slack)
+- Background monitoring daemon
+- Operations dashboard generation
+- Intelligent rotation (size and time-based)
+- Advanced archiving with metadata
+
+### Deprecated Scripts
+
+1. **archive-logs.sh** → Use `./scripts/log-manager.sh archive`
+2. **quick-archive.sh** → Use `./scripts/log-manager.sh full`
+3. **view-latest-archive.sh** → Use `./scripts/log-manager.sh status`
+4. **rotate-logs.sh** → Use `./scripts/log-manager.sh rotate`
+5. **setup-log-rotation.sh** → Use `./scripts/log-manager.sh init`
+
+## Migration Guide
+
+**Instead of:**
+```bash
+./scripts/archive-logs.sh
+```
+
+**Use:**
+```bash
+./scripts/log-manager.sh archive
+```
+
+**Instead of:**
+```bash
+./scripts/quick-archive.sh --clear-logs
+```
+
+**Use:**
+```bash
+./scripts/log-manager.sh full
+```
+
+## Why These Were Deprecated
+
+The individual log management scripts were created before the comprehensive `log-manager.sh` system was implemented. The new system provides:
+
+- **Unified Interface**: Single command with multiple subcommands
+- **Production Grade**: Health monitoring, alerting, and metrics
+- **Better Maintenance**: One script to maintain instead of five
+- **More Features**: Dashboard generation, daemon mode, performance tracking
+- **Safer Operations**: Validation and corruption detection
+
+## Can I Still Use These?
+
+Yes, these scripts still work and are kept for backwards compatibility. However, it's recommended to migrate to `log-manager.sh` for better functionality and ongoing support.
+
+## When Will These Be Removed?
+
+These scripts will be kept for at least one major version release to allow for migration. They may be removed in a future version once all users have migrated to `log-manager.sh`.
+
+---
+
+**See:** `docs/SCRIPT_ANALYSIS_REPORT.md` for the full analysis
--- a/scripts/deprecated/archive-logs.sh
+++ b/scripts/deprecated/archive-logs.sh
@@ -0,0 +1,334 @@
+#!/bin/bash
+
+# MEV Bot Log Archiving Script
+# Automatically archives and compresses logs with timestamp and metadata
+
+set -euo pipefail
+
+# Configuration
+PROJECT_ROOT="/home/administrator/projects/mev-beta"
+LOGS_DIR="$PROJECT_ROOT/logs"
+ARCHIVE_DIR="$PROJECT_ROOT/logs/archives"
+TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
+ARCHIVE_NAME="mev_logs_${TIMESTAMP}"
+RETENTION_DAYS=30
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Logging function
+log() {
+    echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
+}
+
+warn() {
+    echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARNING:${NC} $1"
+}
+
+error() {
+    echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR:${NC} $1"
+}
+
+# Create archive directory if it doesn't exist
+create_archive_dir() {
+    if [[ ! -d "$ARCHIVE_DIR" ]]; then
+        log "Creating archive directory: $ARCHIVE_DIR"
+        mkdir -p "$ARCHIVE_DIR"
+    fi
+}
+
+# Generate archive metadata
+generate_metadata() {
+    local archive_path="$1"
+    local metadata_file="$archive_path/archive_metadata.json"
+
+    log "Generating archive metadata..."
+
+    cat > "$metadata_file" << EOF
+{
+    "archive_info": {
+        "timestamp": "$(date -Iseconds)",
+        "archive_name": "$ARCHIVE_NAME",
+        "created_by": "$(whoami)",
+        "hostname": "$(hostname)",
+        "mev_bot_version": "$(git rev-parse HEAD 2>/dev/null || echo 'unknown')",
+        "git_branch": "$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo 'unknown')"
+    },
+    "system_info": {
+        "os": "$(uname -s)",
+        "kernel": "$(uname -r)",
+        "architecture": "$(uname -m)",
+        "uptime": "$(uptime -p 2>/dev/null || echo 'unknown')"
+    },
+    "log_summary": {
+        "total_files": $(find "$LOGS_DIR" -type f -name "*.log" | wc -l),
+        "total_size_bytes": $(find "$LOGS_DIR" -type f -name "*.log" -exec stat -c%s {} + | awk '{sum+=$1} END {print sum+0}'),
+        "date_range": {
+            "oldest_file": "$(find "$LOGS_DIR" -type f -name "*.log" -printf '%T+ %p\n' | sort | head -1 | cut -d' ' -f1 || echo 'none')",
+            "newest_file": "$(find "$LOGS_DIR" -type f -name "*.log" -printf '%T+ %p\n' | sort | tail -1 | cut -d' ' -f1 || echo 'none')"
+        }
+    },
+    "archive_contents": [
+$(find "$LOGS_DIR" -type f -name "*.log" -printf '        "%f",\n' | sed '$s/,$//')
+    ]
+}
+EOF
+}
+
+# Archive logs with compression
+archive_logs() {
+    local temp_archive_dir="$ARCHIVE_DIR/$ARCHIVE_NAME"
+
+    log "Creating temporary archive directory: $temp_archive_dir"
+    mkdir -p "$temp_archive_dir"
+
+    # Copy all log files
+    log "Copying log files..."
+    if ls "$LOGS_DIR"/*.log 1> /dev/null 2>&1; then
+        cp "$LOGS_DIR"/*.log "$temp_archive_dir/"
+        log "Copied $(ls "$LOGS_DIR"/*.log | wc -l) log files"
+    else
+        warn "No .log files found in $LOGS_DIR"
+    fi
+
+    # Copy diagnostic logs if they exist
+    if [[ -d "$LOGS_DIR/diagnostics" ]]; then
+        log "Copying diagnostics directory..."
+        cp -r "$LOGS_DIR/diagnostics" "$temp_archive_dir/"
+    fi
+
+    # Copy any other relevant log directories
+    for subdir in debug test performance audit; do
+        if [[ -d "$LOGS_DIR/$subdir" ]]; then
+            log "Copying $subdir directory..."
+            cp -r "$LOGS_DIR/$subdir" "$temp_archive_dir/"
+        fi
+    done
+
+    # Generate metadata
+    generate_metadata "$temp_archive_dir"
+
+    # Create compressed archive
+    log "Creating compressed archive..."
+    cd "$ARCHIVE_DIR"
+    tar -czf "${ARCHIVE_NAME}.tar.gz" "$ARCHIVE_NAME"
+
+    # Calculate archive size
+    local archive_size=$(stat -c%s "${ARCHIVE_NAME}.tar.gz" | numfmt --to=iec)
+    log "Archive created: ${ARCHIVE_NAME}.tar.gz (${archive_size})"
+
+    # Remove temporary directory
+    rm -rf "$temp_archive_dir"
+
+    # Create symlink to latest archive
+    ln -sf "${ARCHIVE_NAME}.tar.gz" "latest_archive.tar.gz"
+    log "Created symlink: latest_archive.tar.gz"
+}
+
+# Generate archive report
+generate_report() {
+    local report_file="$ARCHIVE_DIR/archive_report_${TIMESTAMP}.txt"
+
+    log "Generating archive report..."
+
+    cat > "$report_file" << EOF
+MEV Bot Log Archive Report
+==========================
+Generated: $(date)
+Archive: ${ARCHIVE_NAME}.tar.gz
+
+System Information:
+- Hostname: $(hostname)
+- User: $(whoami)
+- OS: $(uname -s) $(uname -r)
+- Architecture: $(uname -m)
+
+Archive Contents:
+$(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | head -20)
+$([ $(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | wc -l) -gt 20 ] && echo "... and $(($(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | wc -l) - 20)) more files")
+
+Archive Statistics:
+- Compressed size: $(stat -c%s "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | numfmt --to=iec)
+- Files archived: $(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | grep -c '\.log$' || echo '0')
+
+Git Information:
+- Branch: $(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo 'unknown')
+- Commit: $(git rev-parse HEAD 2>/dev/null || echo 'unknown')
+- Status: $(git status --porcelain 2>/dev/null | wc -l) uncommitted changes
+
+Recent Log Activity:
+$(tail -10 "$LOGS_DIR/mev_bot.log" 2>/dev/null | head -5 || echo "No recent activity found")
+
+Archive Location: $ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz
+EOF
+
+    log "Report generated: $report_file"
+}
+
+# Clean old archives based on retention policy
+cleanup_old_archives() {
+    log "Cleaning up archives older than $RETENTION_DAYS days..."
+
+    local deleted_count=0
+    while IFS= read -r -d '' archive; do
+        if [[ -f "$archive" ]]; then
+            rm "$archive"
+            ((deleted_count++))
+            log "Deleted old archive: $(basename "$archive")"
+        fi
+    done < <(find "$ARCHIVE_DIR" -name "mev_logs_*.tar.gz" -mtime +$RETENTION_DAYS -print0 2>/dev/null)
+
+    # Also clean old report files
+    find "$ARCHIVE_DIR" -name "archive_report_*.txt" -mtime +$RETENTION_DAYS -delete 2>/dev/null || true
+
+    if [[ $deleted_count -gt 0 ]]; then
+        log "Cleaned up $deleted_count old archives"
+    else
+        log "No old archives to clean up"
+    fi
+}
+
+# Clear current logs (optional)
+clear_current_logs() {
+    if [[ "${1:-}" == "--clear-logs" ]]; then
+        log "Clearing current log files..."
+
+        # Backup current running processes
+        local running_processes=$(ps aux | grep mev-bot | grep -v grep | wc -l)
+        if [[ $running_processes -gt 0 ]]; then
+            warn "MEV bot processes are still running. Stopping them first..."
+            pkill -f mev-bot || true
+            sleep 2
+        fi
+
+        # Clear main log files but keep directory structure
+        if ls "$LOGS_DIR"/*.log 1> /dev/null 2>&1; then
+            rm "$LOGS_DIR"/*.log
+            log "Cleared current log files"
+        fi
+
+        # Clear diagnostic logs
+        if [[ -d "$LOGS_DIR/diagnostics" ]]; then
+            rm -rf "$LOGS_DIR/diagnostics"/*
+            log "Cleared diagnostics directory"
+        fi
+
+        # Create fresh main log file
+        touch "$LOGS_DIR/mev_bot.log"
+        log "Created fresh log file"
+    fi
+}
+
+# Display archive information
+show_archive_info() {
+    if [[ "${1:-}" == "--info" ]]; then
+        echo -e "${BLUE}Archive Information:${NC}"
+        echo "Archive directory: $ARCHIVE_DIR"
+        echo "Retention policy: $RETENTION_DAYS days"
+        echo
+
+        if [[ -d "$ARCHIVE_DIR" ]]; then
+            echo -e "${BLUE}Existing archives:${NC}"
+            ls -lah "$ARCHIVE_DIR"/*.tar.gz 2>/dev/null | while read -r line; do
+                echo "  $line"
+            done
+
+            echo
+            echo -e "${BLUE}Total archive space used:${NC}"
+            du -sh "$ARCHIVE_DIR" 2>/dev/null || echo "  Archive directory not found"
+        else
+            echo "No archives found (directory doesn't exist yet)"
+        fi
+        exit 0
+    fi
+}
+
+# Display help
+show_help() {
+    if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
+        cat << EOF
+MEV Bot Log Archiving Script
+
+USAGE:
+    $0 [OPTIONS]
+
+OPTIONS:
+    --clear-logs    Archive logs and then clear current log files
+    --info          Show information about existing archives
+    --help, -h      Show this help message
+
+DESCRIPTION:
+    Archives all MEV bot log files with timestamp, compression, and metadata.
+    Creates organized archives in logs/archives/ directory with automatic cleanup.
+
+EXAMPLES:
+    $0                    # Archive logs (keep current logs)
+    $0 --clear-logs      # Archive and clear current logs
+    $0 --info            # Show archive information
+
+ARCHIVE LOCATION:
+    $ARCHIVE_DIR
+
+RETENTION POLICY:
+    Archives older than $RETENTION_DAYS days are automatically deleted.
+EOF
+        exit 0
+    fi
+}
+
+# Main execution
+main() {
+    log "Starting MEV Bot log archiving process..."
+
+    # Check if we're in the right directory
+    if [[ ! -d "$PROJECT_ROOT" ]]; then
+        error "Project root not found: $PROJECT_ROOT"
+        exit 1
+    fi
+
+    cd "$PROJECT_ROOT"
+
+    # Check for help or info flags
+    show_help "$@"
+    show_archive_info "$@"
+
+    # Check if logs directory exists
+    if [[ ! -d "$LOGS_DIR" ]]; then
+        error "Logs directory not found: $LOGS_DIR"
+        exit 1
+    fi
+
+    # Create archive directory
+    create_archive_dir
+
+    # Archive logs
+    archive_logs
+
+    # Generate report
+    generate_report
+
+    # Clean up old archives
+    cleanup_old_archives
+
+    # Clear current logs if requested
+    clear_current_logs "$@"
+
+    log "Archive process completed successfully!"
+    log "Archive location: $ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz"
+
+    # Show final summary
+    echo
+    echo -e "${GREEN}=== ARCHIVE SUMMARY ===${NC}"
+    echo "Archive: ${ARCHIVE_NAME}.tar.gz"
+    echo "Location: $ARCHIVE_DIR"
+    echo "Size: $(stat -c%s "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | numfmt --to=iec)"
+    echo "Files: $(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | grep -c '\.log$' || echo '0') log files"
+    echo "Latest archive symlink: $ARCHIVE_DIR/latest_archive.tar.gz"
+}
+
+# Run main function with all arguments
+main "$@"
--- a/scripts/deprecated/quick-archive.sh
+++ b/scripts/deprecated/quick-archive.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Quick Archive - Archive logs and clear current logs for fresh start
+# Usage: ./scripts/quick-archive.sh
+
+echo "🗂️  Quick Archive: Creating archive and clearing logs for fresh start..."
+echo
+
+# Archive with clear logs option
+./scripts/archive-logs.sh --clear-logs
+
+echo
+echo "✅ Quick archive complete! Ready for fresh MEV bot run."
+echo "📁 Archived logs location: logs/archives/latest_archive.tar.gz"
+echo "🆕 Fresh log files created and ready"
--- a/scripts/deprecated/rotate-logs.sh
+++ b/scripts/deprecated/rotate-logs.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Log rotation script for MEV Bot
+
+# Configuration
+LOG_DIR="/home/administrator/projects/mev-beta/logs"
+MAX_SIZE_MB=100
+RETENTION_DAYS=30
+
+# Rotate event logs when they exceed MAX_SIZE_MB
+rotate_large_logs() {
+    echo "Checking for large logs to rotate..."
+    
+    # Find log files larger than MAX_SIZE_MB
+    find "$LOG_DIR/events" -name "*.jsonl" -size +${MAX_SIZE_MB}M | while read logfile; do
+        echo "Rotating large log: $logfile"
+        
+        # Compress the log file
+        gzip "$logfile"
+        
+        # Move to archived directory
+        mv "${logfile}.gz" "$LOG_DIR/archived/"
+    done
+}
+
+# Clean up old archived logs
+cleanup_old_logs() {
+    echo "Cleaning up archived logs older than $RETENTION_DAYS days..."
+    
+    find "$LOG_DIR/archived" -name "*.gz" -mtime +$RETENTION_DAYS -delete
+}
+
+# Main execution
+echo "Starting log rotation for MEV Bot..."
+rotate_large_logs
+cleanup_old_logs
+echo "Log rotation completed."
--- a/scripts/deprecated/setup-log-rotation.sh
+++ b/scripts/deprecated/setup-log-rotation.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Add this line to your crontab to run log rotation daily at 2 AM:
+# 0 2 * * * /home/administrator/projects/mev-beta/scripts/rotate-logs.sh
+
+# This script is meant to be run as a cron job for automatic log rotation
+echo "Setting up daily log rotation for MEV Bot..."
+
+# Get the current crontab
+crontab -l > /tmp/mev_cron
+
+# Check if our job is already in the crontab
+if ! grep -q "rotate-logs.sh" /tmp/mev_cron; then
+    # Add the log rotation job to run daily at 2 AM
+    echo "0 2 * * * cd /home/administrator/projects/mev-beta && /home/administrator/projects/mev-beta/scripts/rotate-logs.sh" >> /tmp/mev_cron
+    # Install the new crontab
+    crontab /tmp/mev_cron
+    echo "Log rotation job added to crontab. Will run daily at 2 AM."
+else
+    echo "Log rotation job already exists in crontab."
+fi
+
+# Clean up
+rm /tmp/mev_cron
--- a/scripts/deprecated/view-latest-archive.sh
+++ b/scripts/deprecated/view-latest-archive.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# View Latest Archive - Extract and browse the most recent log archive
+# Usage: ./scripts/view-latest-archive.sh [pattern]
+
+ARCHIVE_DIR="logs/archives"
+TEMP_DIR="/tmp/mev_archive_view"
+PATTERN="${1:-}"
+
+if [[ ! -f "$ARCHIVE_DIR/latest_archive.tar.gz" ]]; then
+    echo "❌ No archive found. Run ./scripts/archive-logs.sh first."
+    exit 1
+fi
+
+echo "📂 Extracting latest archive for viewing..."
+rm -rf "$TEMP_DIR"
+mkdir -p "$TEMP_DIR"
+cd "$TEMP_DIR"
+
+# Extract archive
+tar -xzf "$OLDPWD/$ARCHIVE_DIR/latest_archive.tar.gz"
+
+ARCHIVE_NAME=$(ls | head -1)
+cd "$ARCHIVE_NAME"
+
+echo "✅ Archive extracted to: $TEMP_DIR/$ARCHIVE_NAME"
+echo
+
+if [[ -n "$PATTERN" ]]; then
+    echo "🔍 Searching for pattern: $PATTERN"
+    echo "================================================"
+    grep -r "$PATTERN" . --color=always | head -20
+    echo
+    echo "📊 Pattern summary:"
+    grep -r "$PATTERN" . | wc -l | xargs echo "Total matches:"
+else
+    echo "📋 Archive contents:"
+    ls -la
+    echo
+    echo "📊 Archive summary:"
+    echo "- Log files: $(ls *.log 2>/dev/null | wc -l)"
+    echo "- Total size: $(du -sh . | cut -f1)"
+
+    if [[ -f "archive_metadata.json" ]]; then
+        echo
+        echo "📈 Metadata excerpt:"
+        cat archive_metadata.json | head -20
+    fi
+fi
+
+echo
+echo "💡 Tips:"
+echo "  View specific log: cat $TEMP_DIR/$ARCHIVE_NAME/mev_bot.log"
+echo "  Search pattern: $0 'DIRECT PARSING'"
+echo "  Cleanup: rm -rf $TEMP_DIR"