#!/bin/bash # MEV Bot Log Archiving Script # Automatically archives and compresses logs with timestamp and metadata set -euo pipefail # Configuration PROJECT_ROOT="/home/administrator/projects/mev-beta" LOGS_DIR="$PROJECT_ROOT/logs" ARCHIVE_DIR="$PROJECT_ROOT/logs/archives" TIMESTAMP=$(date +"%Y%m%d_%H%M%S") ARCHIVE_NAME="mev_logs_${TIMESTAMP}" RETENTION_DAYS=30 # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Logging function log() { echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1" } warn() { echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARNING:${NC} $1" } error() { echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR:${NC} $1" } # Create archive directory if it doesn't exist create_archive_dir() { if [[ ! -d "$ARCHIVE_DIR" ]]; then log "Creating archive directory: $ARCHIVE_DIR" mkdir -p "$ARCHIVE_DIR" fi } # Generate archive metadata generate_metadata() { local archive_path="$1" local metadata_file="$archive_path/archive_metadata.json" log "Generating archive metadata..." cat > "$metadata_file" << EOF { "archive_info": { "timestamp": "$(date -Iseconds)", "archive_name": "$ARCHIVE_NAME", "created_by": "$(whoami)", "hostname": "$(hostname)", "mev_bot_version": "$(git rev-parse HEAD 2>/dev/null || echo 'unknown')", "git_branch": "$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo 'unknown')" }, "system_info": { "os": "$(uname -s)", "kernel": "$(uname -r)", "architecture": "$(uname -m)", "uptime": "$(uptime -p 2>/dev/null || echo 'unknown')" }, "log_summary": { "total_files": $(find "$LOGS_DIR" -type f -name "*.log" | wc -l), "total_size_bytes": $(find "$LOGS_DIR" -type f -name "*.log" -exec stat -c%s {} + | awk '{sum+=$1} END {print sum+0}'), "date_range": { "oldest_file": "$(find "$LOGS_DIR" -type f -name "*.log" -printf '%T+ %p\n' | sort | head -1 | cut -d' ' -f1 || echo 'none')", "newest_file": "$(find "$LOGS_DIR" -type f -name "*.log" -printf '%T+ %p\n' | sort | tail -1 | cut -d' ' -f1 || echo 'none')" } }, "archive_contents": [ $(find "$LOGS_DIR" -type f -name "*.log" -printf ' "%f",\n' | sed '$s/,$//') ] } EOF } # Archive logs with compression archive_logs() { local temp_archive_dir="$ARCHIVE_DIR/$ARCHIVE_NAME" log "Creating temporary archive directory: $temp_archive_dir" mkdir -p "$temp_archive_dir" # Copy all log files log "Copying log files..." if ls "$LOGS_DIR"/*.log 1> /dev/null 2>&1; then cp "$LOGS_DIR"/*.log "$temp_archive_dir/" log "Copied $(ls "$LOGS_DIR"/*.log | wc -l) log files" else warn "No .log files found in $LOGS_DIR" fi # Copy diagnostic logs if they exist if [[ -d "$LOGS_DIR/diagnostics" ]]; then log "Copying diagnostics directory..." cp -r "$LOGS_DIR/diagnostics" "$temp_archive_dir/" fi # Copy any other relevant log directories for subdir in debug test performance audit; do if [[ -d "$LOGS_DIR/$subdir" ]]; then log "Copying $subdir directory..." cp -r "$LOGS_DIR/$subdir" "$temp_archive_dir/" fi done # Generate metadata generate_metadata "$temp_archive_dir" # Create compressed archive log "Creating compressed archive..." cd "$ARCHIVE_DIR" tar -czf "${ARCHIVE_NAME}.tar.gz" "$ARCHIVE_NAME" # Calculate archive size local archive_size=$(stat -c%s "${ARCHIVE_NAME}.tar.gz" | numfmt --to=iec) log "Archive created: ${ARCHIVE_NAME}.tar.gz (${archive_size})" # Remove temporary directory rm -rf "$temp_archive_dir" # Create symlink to latest archive ln -sf "${ARCHIVE_NAME}.tar.gz" "latest_archive.tar.gz" log "Created symlink: latest_archive.tar.gz" } # Generate archive report generate_report() { local report_file="$ARCHIVE_DIR/archive_report_${TIMESTAMP}.txt" log "Generating archive report..." cat > "$report_file" << EOF MEV Bot Log Archive Report ========================== Generated: $(date) Archive: ${ARCHIVE_NAME}.tar.gz System Information: - Hostname: $(hostname) - User: $(whoami) - OS: $(uname -s) $(uname -r) - Architecture: $(uname -m) Archive Contents: $(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | head -20) $([ $(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | wc -l) -gt 20 ] && echo "... and $(($(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | wc -l) - 20)) more files") Archive Statistics: - Compressed size: $(stat -c%s "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | numfmt --to=iec) - Files archived: $(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | grep -c '\.log$' || echo '0') Git Information: - Branch: $(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo 'unknown') - Commit: $(git rev-parse HEAD 2>/dev/null || echo 'unknown') - Status: $(git status --porcelain 2>/dev/null | wc -l) uncommitted changes Recent Log Activity: $(tail -10 "$LOGS_DIR/mev_bot.log" 2>/dev/null | head -5 || echo "No recent activity found") Archive Location: $ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz EOF log "Report generated: $report_file" } # Clean old archives based on retention policy cleanup_old_archives() { log "Cleaning up archives older than $RETENTION_DAYS days..." local deleted_count=0 while IFS= read -r -d '' archive; do if [[ -f "$archive" ]]; then rm "$archive" ((deleted_count++)) log "Deleted old archive: $(basename "$archive")" fi done < <(find "$ARCHIVE_DIR" -name "mev_logs_*.tar.gz" -mtime +$RETENTION_DAYS -print0 2>/dev/null) # Also clean old report files find "$ARCHIVE_DIR" -name "archive_report_*.txt" -mtime +$RETENTION_DAYS -delete 2>/dev/null || true if [[ $deleted_count -gt 0 ]]; then log "Cleaned up $deleted_count old archives" else log "No old archives to clean up" fi } # Clear current logs (optional) clear_current_logs() { if [[ "${1:-}" == "--clear-logs" ]]; then log "Clearing current log files..." # Backup current running processes local running_processes=$(ps aux | grep mev-bot | grep -v grep | wc -l) if [[ $running_processes -gt 0 ]]; then warn "MEV bot processes are still running. Stopping them first..." pkill -f mev-bot || true sleep 2 fi # Clear main log files but keep directory structure if ls "$LOGS_DIR"/*.log 1> /dev/null 2>&1; then rm "$LOGS_DIR"/*.log log "Cleared current log files" fi # Clear diagnostic logs if [[ -d "$LOGS_DIR/diagnostics" ]]; then rm -rf "$LOGS_DIR/diagnostics"/* log "Cleared diagnostics directory" fi # Create fresh main log file touch "$LOGS_DIR/mev_bot.log" log "Created fresh log file" fi } # Display archive information show_archive_info() { if [[ "${1:-}" == "--info" ]]; then echo -e "${BLUE}Archive Information:${NC}" echo "Archive directory: $ARCHIVE_DIR" echo "Retention policy: $RETENTION_DAYS days" echo if [[ -d "$ARCHIVE_DIR" ]]; then echo -e "${BLUE}Existing archives:${NC}" ls -lah "$ARCHIVE_DIR"/*.tar.gz 2>/dev/null | while read -r line; do echo " $line" done echo echo -e "${BLUE}Total archive space used:${NC}" du -sh "$ARCHIVE_DIR" 2>/dev/null || echo " Archive directory not found" else echo "No archives found (directory doesn't exist yet)" fi exit 0 fi } # Display help show_help() { if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then cat << EOF MEV Bot Log Archiving Script USAGE: $0 [OPTIONS] OPTIONS: --clear-logs Archive logs and then clear current log files --info Show information about existing archives --help, -h Show this help message DESCRIPTION: Archives all MEV bot log files with timestamp, compression, and metadata. Creates organized archives in logs/archives/ directory with automatic cleanup. EXAMPLES: $0 # Archive logs (keep current logs) $0 --clear-logs # Archive and clear current logs $0 --info # Show archive information ARCHIVE LOCATION: $ARCHIVE_DIR RETENTION POLICY: Archives older than $RETENTION_DAYS days are automatically deleted. EOF exit 0 fi } # Main execution main() { log "Starting MEV Bot log archiving process..." # Check if we're in the right directory if [[ ! -d "$PROJECT_ROOT" ]]; then error "Project root not found: $PROJECT_ROOT" exit 1 fi cd "$PROJECT_ROOT" # Check for help or info flags show_help "$@" show_archive_info "$@" # Check if logs directory exists if [[ ! -d "$LOGS_DIR" ]]; then error "Logs directory not found: $LOGS_DIR" exit 1 fi # Create archive directory create_archive_dir # Archive logs archive_logs # Generate report generate_report # Clean up old archives cleanup_old_archives # Clear current logs if requested clear_current_logs "$@" log "Archive process completed successfully!" log "Archive location: $ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" # Show final summary echo echo -e "${GREEN}=== ARCHIVE SUMMARY ===${NC}" echo "Archive: ${ARCHIVE_NAME}.tar.gz" echo "Location: $ARCHIVE_DIR" echo "Size: $(stat -c%s "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | numfmt --to=iec)" echo "Files: $(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | grep -c '\.log$' || echo '0') log files" echo "Latest archive symlink: $ARCHIVE_DIR/latest_archive.tar.gz" } # Run main function with all arguments main "$@"