refactor: move all remaining files to orig/ directory

Completed clean root directory structure:
- Root now contains only: .git, .env, docs/, orig/
- Moved all remaining files and directories to orig/:
  - Config files (.claude, .dockerignore, .drone.yml, etc.)
  - All .env variants (except active .env)
  - Git config (.gitconfig, .github, .gitignore, etc.)
  - Tool configs (.golangci.yml, .revive.toml, etc.)
  - Documentation (*.md files, @prompts)
  - Build files (Dockerfiles, Makefile, go.mod, go.sum)
  - Docker compose files
  - All source directories (scripts, tests, tools, etc.)
  - Runtime directories (logs, monitoring, reports)
  - Dependency files (node_modules, lib, cache)
  - Special files (--delete)

- Removed empty runtime directories (bin/, data/)

V2 structure is now clean:
- docs/planning/ - V2 planning documents
- orig/ - Complete V1 codebase preserved
- .env - Active environment config (not in git)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Administrator
2025-11-10 10:53:05 +01:00
parent 803de231ba
commit c54c569f30
718 changed files with 8304 additions and 8281 deletions

View File

@@ -0,0 +1,68 @@
# Deprecated Scripts
These scripts have been moved here because their functionality is now handled by more comprehensive tools.
## Log Management Scripts (Superseded by log-manager.sh)
All of these scripts have been replaced by `scripts/log-manager.sh`, which provides:
- Real-time analysis with health scoring
- Performance monitoring with MEV-specific metrics
- Corruption detection and integrity validation
- Multi-channel alerting (email, Slack)
- Background monitoring daemon
- Operations dashboard generation
- Intelligent rotation (size and time-based)
- Advanced archiving with metadata
### Deprecated Scripts
1. **archive-logs.sh** → Use `./scripts/log-manager.sh archive`
2. **quick-archive.sh** → Use `./scripts/log-manager.sh full`
3. **view-latest-archive.sh** → Use `./scripts/log-manager.sh status`
4. **rotate-logs.sh** → Use `./scripts/log-manager.sh rotate`
5. **setup-log-rotation.sh** → Use `./scripts/log-manager.sh init`
## Migration Guide
**Instead of:**
```bash
./scripts/archive-logs.sh
```
**Use:**
```bash
./scripts/log-manager.sh archive
```
**Instead of:**
```bash
./scripts/quick-archive.sh --clear-logs
```
**Use:**
```bash
./scripts/log-manager.sh full
```
## Why These Were Deprecated
The individual log management scripts were created before the comprehensive `log-manager.sh` system was implemented. The new system provides:
- **Unified Interface**: Single command with multiple subcommands
- **Production Grade**: Health monitoring, alerting, and metrics
- **Better Maintenance**: One script to maintain instead of five
- **More Features**: Dashboard generation, daemon mode, performance tracking
- **Safer Operations**: Validation and corruption detection
## Can I Still Use These?
Yes, these scripts still work and are kept for backwards compatibility. However, it's recommended to migrate to `log-manager.sh` for better functionality and ongoing support.
## When Will These Be Removed?
These scripts will be kept for at least one major version release to allow for migration. They may be removed in a future version once all users have migrated to `log-manager.sh`.
---
**See:** `docs/SCRIPT_ANALYSIS_REPORT.md` for the full analysis

View File

@@ -0,0 +1,334 @@
#!/bin/bash
# MEV Bot Log Archiving Script
# Automatically archives and compresses logs with timestamp and metadata
set -euo pipefail
# Configuration
PROJECT_ROOT="/home/administrator/projects/mev-beta"
LOGS_DIR="$PROJECT_ROOT/logs"
ARCHIVE_DIR="$PROJECT_ROOT/logs/archives"
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
ARCHIVE_NAME="mev_logs_${TIMESTAMP}"
RETENTION_DAYS=30
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Logging function
log() {
echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
}
warn() {
echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARNING:${NC} $1"
}
error() {
echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR:${NC} $1"
}
# Create archive directory if it doesn't exist
create_archive_dir() {
if [[ ! -d "$ARCHIVE_DIR" ]]; then
log "Creating archive directory: $ARCHIVE_DIR"
mkdir -p "$ARCHIVE_DIR"
fi
}
# Generate archive metadata
generate_metadata() {
local archive_path="$1"
local metadata_file="$archive_path/archive_metadata.json"
log "Generating archive metadata..."
cat > "$metadata_file" << EOF
{
"archive_info": {
"timestamp": "$(date -Iseconds)",
"archive_name": "$ARCHIVE_NAME",
"created_by": "$(whoami)",
"hostname": "$(hostname)",
"mev_bot_version": "$(git rev-parse HEAD 2>/dev/null || echo 'unknown')",
"git_branch": "$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo 'unknown')"
},
"system_info": {
"os": "$(uname -s)",
"kernel": "$(uname -r)",
"architecture": "$(uname -m)",
"uptime": "$(uptime -p 2>/dev/null || echo 'unknown')"
},
"log_summary": {
"total_files": $(find "$LOGS_DIR" -type f -name "*.log" | wc -l),
"total_size_bytes": $(find "$LOGS_DIR" -type f -name "*.log" -exec stat -c%s {} + | awk '{sum+=$1} END {print sum+0}'),
"date_range": {
"oldest_file": "$(find "$LOGS_DIR" -type f -name "*.log" -printf '%T+ %p\n' | sort | head -1 | cut -d' ' -f1 || echo 'none')",
"newest_file": "$(find "$LOGS_DIR" -type f -name "*.log" -printf '%T+ %p\n' | sort | tail -1 | cut -d' ' -f1 || echo 'none')"
}
},
"archive_contents": [
$(find "$LOGS_DIR" -type f -name "*.log" -printf ' "%f",\n' | sed '$s/,$//')
]
}
EOF
}
# Archive logs with compression
archive_logs() {
local temp_archive_dir="$ARCHIVE_DIR/$ARCHIVE_NAME"
log "Creating temporary archive directory: $temp_archive_dir"
mkdir -p "$temp_archive_dir"
# Copy all log files
log "Copying log files..."
if ls "$LOGS_DIR"/*.log 1> /dev/null 2>&1; then
cp "$LOGS_DIR"/*.log "$temp_archive_dir/"
log "Copied $(ls "$LOGS_DIR"/*.log | wc -l) log files"
else
warn "No .log files found in $LOGS_DIR"
fi
# Copy diagnostic logs if they exist
if [[ -d "$LOGS_DIR/diagnostics" ]]; then
log "Copying diagnostics directory..."
cp -r "$LOGS_DIR/diagnostics" "$temp_archive_dir/"
fi
# Copy any other relevant log directories
for subdir in debug test performance audit; do
if [[ -d "$LOGS_DIR/$subdir" ]]; then
log "Copying $subdir directory..."
cp -r "$LOGS_DIR/$subdir" "$temp_archive_dir/"
fi
done
# Generate metadata
generate_metadata "$temp_archive_dir"
# Create compressed archive
log "Creating compressed archive..."
cd "$ARCHIVE_DIR"
tar -czf "${ARCHIVE_NAME}.tar.gz" "$ARCHIVE_NAME"
# Calculate archive size
local archive_size=$(stat -c%s "${ARCHIVE_NAME}.tar.gz" | numfmt --to=iec)
log "Archive created: ${ARCHIVE_NAME}.tar.gz (${archive_size})"
# Remove temporary directory
rm -rf "$temp_archive_dir"
# Create symlink to latest archive
ln -sf "${ARCHIVE_NAME}.tar.gz" "latest_archive.tar.gz"
log "Created symlink: latest_archive.tar.gz"
}
# Generate archive report
generate_report() {
local report_file="$ARCHIVE_DIR/archive_report_${TIMESTAMP}.txt"
log "Generating archive report..."
cat > "$report_file" << EOF
MEV Bot Log Archive Report
==========================
Generated: $(date)
Archive: ${ARCHIVE_NAME}.tar.gz
System Information:
- Hostname: $(hostname)
- User: $(whoami)
- OS: $(uname -s) $(uname -r)
- Architecture: $(uname -m)
Archive Contents:
$(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | head -20)
$([ $(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | wc -l) -gt 20 ] && echo "... and $(($(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | wc -l) - 20)) more files")
Archive Statistics:
- Compressed size: $(stat -c%s "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | numfmt --to=iec)
- Files archived: $(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | grep -c '\.log$' || echo '0')
Git Information:
- Branch: $(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo 'unknown')
- Commit: $(git rev-parse HEAD 2>/dev/null || echo 'unknown')
- Status: $(git status --porcelain 2>/dev/null | wc -l) uncommitted changes
Recent Log Activity:
$(tail -10 "$LOGS_DIR/mev_bot.log" 2>/dev/null | head -5 || echo "No recent activity found")
Archive Location: $ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz
EOF
log "Report generated: $report_file"
}
# Clean old archives based on retention policy
cleanup_old_archives() {
log "Cleaning up archives older than $RETENTION_DAYS days..."
local deleted_count=0
while IFS= read -r -d '' archive; do
if [[ -f "$archive" ]]; then
rm "$archive"
((deleted_count++))
log "Deleted old archive: $(basename "$archive")"
fi
done < <(find "$ARCHIVE_DIR" -name "mev_logs_*.tar.gz" -mtime +$RETENTION_DAYS -print0 2>/dev/null)
# Also clean old report files
find "$ARCHIVE_DIR" -name "archive_report_*.txt" -mtime +$RETENTION_DAYS -delete 2>/dev/null || true
if [[ $deleted_count -gt 0 ]]; then
log "Cleaned up $deleted_count old archives"
else
log "No old archives to clean up"
fi
}
# Clear current logs (optional)
clear_current_logs() {
if [[ "${1:-}" == "--clear-logs" ]]; then
log "Clearing current log files..."
# Backup current running processes
local running_processes=$(ps aux | grep mev-bot | grep -v grep | wc -l)
if [[ $running_processes -gt 0 ]]; then
warn "MEV bot processes are still running. Stopping them first..."
pkill -f mev-bot || true
sleep 2
fi
# Clear main log files but keep directory structure
if ls "$LOGS_DIR"/*.log 1> /dev/null 2>&1; then
rm "$LOGS_DIR"/*.log
log "Cleared current log files"
fi
# Clear diagnostic logs
if [[ -d "$LOGS_DIR/diagnostics" ]]; then
rm -rf "$LOGS_DIR/diagnostics"/*
log "Cleared diagnostics directory"
fi
# Create fresh main log file
touch "$LOGS_DIR/mev_bot.log"
log "Created fresh log file"
fi
}
# Display archive information
show_archive_info() {
if [[ "${1:-}" == "--info" ]]; then
echo -e "${BLUE}Archive Information:${NC}"
echo "Archive directory: $ARCHIVE_DIR"
echo "Retention policy: $RETENTION_DAYS days"
echo
if [[ -d "$ARCHIVE_DIR" ]]; then
echo -e "${BLUE}Existing archives:${NC}"
ls -lah "$ARCHIVE_DIR"/*.tar.gz 2>/dev/null | while read -r line; do
echo " $line"
done
echo
echo -e "${BLUE}Total archive space used:${NC}"
du -sh "$ARCHIVE_DIR" 2>/dev/null || echo " Archive directory not found"
else
echo "No archives found (directory doesn't exist yet)"
fi
exit 0
fi
}
# Display help
show_help() {
if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
cat << EOF
MEV Bot Log Archiving Script
USAGE:
$0 [OPTIONS]
OPTIONS:
--clear-logs Archive logs and then clear current log files
--info Show information about existing archives
--help, -h Show this help message
DESCRIPTION:
Archives all MEV bot log files with timestamp, compression, and metadata.
Creates organized archives in logs/archives/ directory with automatic cleanup.
EXAMPLES:
$0 # Archive logs (keep current logs)
$0 --clear-logs # Archive and clear current logs
$0 --info # Show archive information
ARCHIVE LOCATION:
$ARCHIVE_DIR
RETENTION POLICY:
Archives older than $RETENTION_DAYS days are automatically deleted.
EOF
exit 0
fi
}
# Main execution
main() {
log "Starting MEV Bot log archiving process..."
# Check if we're in the right directory
if [[ ! -d "$PROJECT_ROOT" ]]; then
error "Project root not found: $PROJECT_ROOT"
exit 1
fi
cd "$PROJECT_ROOT"
# Check for help or info flags
show_help "$@"
show_archive_info "$@"
# Check if logs directory exists
if [[ ! -d "$LOGS_DIR" ]]; then
error "Logs directory not found: $LOGS_DIR"
exit 1
fi
# Create archive directory
create_archive_dir
# Archive logs
archive_logs
# Generate report
generate_report
# Clean up old archives
cleanup_old_archives
# Clear current logs if requested
clear_current_logs "$@"
log "Archive process completed successfully!"
log "Archive location: $ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz"
# Show final summary
echo
echo -e "${GREEN}=== ARCHIVE SUMMARY ===${NC}"
echo "Archive: ${ARCHIVE_NAME}.tar.gz"
echo "Location: $ARCHIVE_DIR"
echo "Size: $(stat -c%s "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | numfmt --to=iec)"
echo "Files: $(tar -tzf "$ARCHIVE_DIR/${ARCHIVE_NAME}.tar.gz" | grep -c '\.log$' || echo '0') log files"
echo "Latest archive symlink: $ARCHIVE_DIR/latest_archive.tar.gz"
}
# Run main function with all arguments
main "$@"

View File

@@ -0,0 +1,15 @@
#!/bin/bash
# Quick Archive - Archive logs and clear current logs for fresh start
# Usage: ./scripts/quick-archive.sh
echo "🗂️ Quick Archive: Creating archive and clearing logs for fresh start..."
echo
# Archive with clear logs option
./scripts/archive-logs.sh --clear-logs
echo
echo "✅ Quick archive complete! Ready for fresh MEV bot run."
echo "📁 Archived logs location: logs/archives/latest_archive.tar.gz"
echo "🆕 Fresh log files created and ready"

View File

@@ -0,0 +1,37 @@
#!/bin/bash
# Log rotation script for MEV Bot
# Configuration
LOG_DIR="/home/administrator/projects/mev-beta/logs"
MAX_SIZE_MB=100
RETENTION_DAYS=30
# Rotate event logs when they exceed MAX_SIZE_MB
rotate_large_logs() {
echo "Checking for large logs to rotate..."
# Find log files larger than MAX_SIZE_MB
find "$LOG_DIR/events" -name "*.jsonl" -size +${MAX_SIZE_MB}M | while read logfile; do
echo "Rotating large log: $logfile"
# Compress the log file
gzip "$logfile"
# Move to archived directory
mv "${logfile}.gz" "$LOG_DIR/archived/"
done
}
# Clean up old archived logs
cleanup_old_logs() {
echo "Cleaning up archived logs older than $RETENTION_DAYS days..."
find "$LOG_DIR/archived" -name "*.gz" -mtime +$RETENTION_DAYS -delete
}
# Main execution
echo "Starting log rotation for MEV Bot..."
rotate_large_logs
cleanup_old_logs
echo "Log rotation completed."

View File

@@ -0,0 +1,24 @@
#!/bin/bash
# Add this line to your crontab to run log rotation daily at 2 AM:
# 0 2 * * * /home/administrator/projects/mev-beta/scripts/rotate-logs.sh
# This script is meant to be run as a cron job for automatic log rotation
echo "Setting up daily log rotation for MEV Bot..."
# Get the current crontab
crontab -l > /tmp/mev_cron
# Check if our job is already in the crontab
if ! grep -q "rotate-logs.sh" /tmp/mev_cron; then
# Add the log rotation job to run daily at 2 AM
echo "0 2 * * * cd /home/administrator/projects/mev-beta && /home/administrator/projects/mev-beta/scripts/rotate-logs.sh" >> /tmp/mev_cron
# Install the new crontab
crontab /tmp/mev_cron
echo "Log rotation job added to crontab. Will run daily at 2 AM."
else
echo "Log rotation job already exists in crontab."
fi
# Clean up
rm /tmp/mev_cron

View File

@@ -0,0 +1,55 @@
#!/bin/bash
# View Latest Archive - Extract and browse the most recent log archive
# Usage: ./scripts/view-latest-archive.sh [pattern]
ARCHIVE_DIR="logs/archives"
TEMP_DIR="/tmp/mev_archive_view"
PATTERN="${1:-}"
if [[ ! -f "$ARCHIVE_DIR/latest_archive.tar.gz" ]]; then
echo "❌ No archive found. Run ./scripts/archive-logs.sh first."
exit 1
fi
echo "📂 Extracting latest archive for viewing..."
rm -rf "$TEMP_DIR"
mkdir -p "$TEMP_DIR"
cd "$TEMP_DIR"
# Extract archive
tar -xzf "$OLDPWD/$ARCHIVE_DIR/latest_archive.tar.gz"
ARCHIVE_NAME=$(ls | head -1)
cd "$ARCHIVE_NAME"
echo "✅ Archive extracted to: $TEMP_DIR/$ARCHIVE_NAME"
echo
if [[ -n "$PATTERN" ]]; then
echo "🔍 Searching for pattern: $PATTERN"
echo "================================================"
grep -r "$PATTERN" . --color=always | head -20
echo
echo "📊 Pattern summary:"
grep -r "$PATTERN" . | wc -l | xargs echo "Total matches:"
else
echo "📋 Archive contents:"
ls -la
echo
echo "📊 Archive summary:"
echo "- Log files: $(ls *.log 2>/dev/null | wc -l)"
echo "- Total size: $(du -sh . | cut -f1)"
if [[ -f "archive_metadata.json" ]]; then
echo
echo "📈 Metadata excerpt:"
cat archive_metadata.json | head -20
fi
fi
echo
echo "💡 Tips:"
echo " View specific log: cat $TEMP_DIR/$ARCHIVE_NAME/mev_bot.log"
echo " Search pattern: $0 'DIRECT PARSING'"
echo " Cleanup: rm -rf $TEMP_DIR"