feat: create v2-prep branch with comprehensive planning

Restructured project for V2 refactor:

**Structure Changes:**
- Moved all V1 code to orig/ folder (preserved with git mv)
- Created docs/planning/ directory
- Added orig/README_V1.md explaining V1 preservation

**Planning Documents:**
- 00_V2_MASTER_PLAN.md: Complete architecture overview
  - Executive summary of critical V1 issues
  - High-level component architecture diagrams
  - 5-phase implementation roadmap
  - Success metrics and risk mitigation

- 07_TASK_BREAKDOWN.md: Atomic task breakdown
  - 99+ hours of detailed tasks
  - Every task < 2 hours (atomic)
  - Clear dependencies and success criteria
  - Organized by implementation phase

**V2 Key Improvements:**
- Per-exchange parsers (factory pattern)
- Multi-layer strict validation
- Multi-index pool cache
- Background validation pipeline
- Comprehensive observability

**Critical Issues Addressed:**
- Zero address tokens (strict validation + cache enrichment)
- Parsing accuracy (protocol-specific parsers)
- No audit trail (background validation channel)
- Inefficient lookups (multi-index cache)
- Stats disconnection (event-driven metrics)

Next Steps:
1. Review planning documents
2. Begin Phase 1: Foundation (P1-001 through P1-010)
3. Implement parsers in Phase 2
4. Build cache system in Phase 3
5. Add validation pipeline in Phase 4
6. Migrate and test in Phase 5

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Administrator
2025-11-10 10:14:26 +01:00
parent 1773daffe7
commit 803de231ba
411 changed files with 20390 additions and 8680 deletions

144
scripts/auto-update.sh Executable file
View File

@@ -0,0 +1,144 @@
#!/bin/bash
# MEV Bot Auto-Update Script
# Checks for updates on master branch and automatically pulls, rebuilds, and restarts
set -e
# Configuration
BRANCH="${GIT_BRANCH:-master}"
REMOTE="${GIT_REMOTE:-origin}"
PROJECT_DIR="${PROJECT_DIR:-$(cd "$(dirname "$0")/.." && pwd)}"
LOG_FILE="${PROJECT_DIR}/logs/auto-update.log"
# Color codes
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Ensure log directory exists
mkdir -p "${PROJECT_DIR}/logs"
# Logging function
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}
log_color() {
echo -e "$1" | tee -a "$LOG_FILE"
}
# Change to project directory
cd "$PROJECT_DIR"
log_color "${BLUE}========================================="
log "MEV Bot Auto-Update Check"
log_color "${BLUE}=========================================${NC}"
log "Project: $PROJECT_DIR"
log "Branch: $BRANCH"
log "Remote: $REMOTE"
log ""
# Fetch latest changes from remote
log_color "${YELLOW}Fetching latest changes from $REMOTE...${NC}"
if git fetch "$REMOTE" >> "$LOG_FILE" 2>&1; then
log_color "${GREEN}✓ Fetch successful${NC}"
else
log_color "${RED}✗ Fetch failed${NC}"
exit 1
fi
# Get current and remote commit hashes
LOCAL_COMMIT=$(git rev-parse HEAD)
REMOTE_COMMIT=$(git rev-parse "$REMOTE/$BRANCH")
log "Local commit: $LOCAL_COMMIT"
log "Remote commit: $REMOTE_COMMIT"
# Check if update is available
if [ "$LOCAL_COMMIT" = "$REMOTE_COMMIT" ]; then
log_color "${GREEN}✓ Already up to date${NC}"
log ""
exit 0
fi
# Update available
log_color "${YELLOW}⚠ Update available!${NC}"
log ""
# Show what's new
log_color "${BLUE}New commits:${NC}"
git log --oneline "$LOCAL_COMMIT..$REMOTE_COMMIT" | tee -a "$LOG_FILE"
log ""
# Check if we're on the correct branch
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
if [ "$CURRENT_BRANCH" != "$BRANCH" ]; then
log_color "${RED}✗ Not on $BRANCH branch (currently on $CURRENT_BRANCH)${NC}"
log "Skipping auto-update. Please switch to $BRANCH manually."
exit 1
fi
# Check for uncommitted changes
if ! git diff-index --quiet HEAD --; then
log_color "${RED}✗ Uncommitted changes detected${NC}"
log "Skipping auto-update. Please commit or stash changes first."
git status --short | tee -a "$LOG_FILE"
exit 1
fi
# Pull the changes
log_color "${YELLOW}Pulling changes from $REMOTE/$BRANCH...${NC}"
if git pull "$REMOTE" "$BRANCH" >> "$LOG_FILE" 2>&1; then
log_color "${GREEN}✓ Pull successful${NC}"
else
log_color "${RED}✗ Pull failed${NC}"
exit 1
fi
# The post-merge hook will handle rebuild and restart
log ""
log_color "${YELLOW}Post-merge hook will handle rebuild and restart...${NC}"
log ""
# Wait for post-merge hook to complete
sleep 2
# Verify container is running
log_color "${YELLOW}Verifying container status...${NC}"
if docker compose ps | grep -q "mev-bot.*running"; then
log_color "${GREEN}✓ Container is running${NC}"
else
log_color "${RED}✗ Container is not running${NC}"
docker compose ps | tee -a "$LOG_FILE"
exit 1
fi
# Show container status
log ""
log_color "${GREEN}========================================="
log "Update Complete!"
log_color "${GREEN}=========================================${NC}"
log "Updated from: $(echo $LOCAL_COMMIT | cut -c1-7)"
log "Updated to: $(echo $REMOTE_COMMIT | cut -c1-7)"
log ""
# Send notification if configured
if command -v curl &> /dev/null && [ -n "$WEBHOOK_URL" ]; then
NEW_COMMITS=$(git log --oneline "$LOCAL_COMMIT..$REMOTE_COMMIT" | wc -l)
MESSAGE="MEV Bot auto-updated: $NEW_COMMITS new commit(s) on $BRANCH"
curl -X POST "$WEBHOOK_URL" \
-H "Content-Type: application/json" \
-d "{\"text\":\"$MESSAGE\"}" \
>> "$LOG_FILE" 2>&1 || true
log "Notification sent to webhook"
fi
log "View logs: tail -f $LOG_FILE"
log "View container logs: docker compose logs -f mev-bot"
log ""
exit 0

273
scripts/dev-env.sh Executable file
View File

@@ -0,0 +1,273 @@
#!/bin/bash
# MEV Bot Development Environment Manager
# Supports branch selection and podman-in-podman development
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Script directory
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
# Default values
DEFAULT_BRANCH="master-dev"
COMPOSE_FILE="docker-compose.dev.yml"
# Function to print colored messages
print_info() {
echo -e "${BLUE}${NC} $1"
}
print_success() {
echo -e "${GREEN}${NC} $1"
}
print_warning() {
echo -e "${YELLOW}${NC} $1"
}
print_error() {
echo -e "${RED}${NC} $1"
}
# Function to show usage
usage() {
cat << EOF
${BLUE}MEV Bot Development Environment Manager${NC}
Usage: $0 [COMMAND] [OPTIONS]
Commands:
start [BRANCH] Start development environment with specified branch (default: master-dev)
stop Stop development environment
restart [BRANCH] Restart development environment with specified branch
rebuild [BRANCH] Rebuild and restart with specified branch
logs [OPTIONS] Show logs (pass options like -f for follow)
shell Open shell in running container
status Show container status
clean Clean all containers and volumes
branches List available git branches
switch BRANCH Switch to a different branch and rebuild
Options:
-h, --help Show this help message
Examples:
$0 start master-dev # Start with master-dev branch
$0 start fix-critical-arbitrage-bugs # Start with feature branch
$0 rebuild master # Rebuild with master branch
$0 logs -f # Follow logs
$0 switch feat-new-feature # Switch to new feature branch
Environment Variables:
GIT_BRANCH Git branch to use (default: master-dev)
LOG_LEVEL Logging level (default: debug)
METRICS_ENABLED Enable metrics (default: true)
EOF
}
# Function to check dependencies
check_dependencies() {
if ! command -v podman-compose &> /dev/null; then
print_error "podman-compose is not installed"
print_info "Install with: pip3 install podman-compose"
exit 1
fi
}
# Function to list available branches
list_branches() {
print_info "Available branches:"
cd "$PROJECT_ROOT"
git branch -a | grep -v HEAD | sed 's/^..../ /'
}
# Function to start development environment
start_dev() {
local branch="${1:-$DEFAULT_BRANCH}"
print_info "Starting MEV Bot development environment on branch: $branch"
cd "$PROJECT_ROOT"
# Export branch for docker-compose
export GIT_BRANCH="$branch"
# Build and start
print_info "Building image for branch: $branch"
podman-compose -f "$COMPOSE_FILE" build --build-arg GIT_BRANCH="$branch"
print_info "Starting container..."
podman-compose -f "$COMPOSE_FILE" up -d
print_success "Development environment started on branch: $branch"
print_info "View logs with: $0 logs -f"
print_info "Access shell with: $0 shell"
}
# Function to stop development environment
stop_dev() {
print_info "Stopping development environment..."
cd "$PROJECT_ROOT"
podman-compose -f "$COMPOSE_FILE" down
print_success "Development environment stopped"
}
# Function to restart development environment
restart_dev() {
local branch="${1:-$DEFAULT_BRANCH}"
stop_dev
start_dev "$branch"
}
# Function to rebuild development environment
rebuild_dev() {
local branch="${1:-$DEFAULT_BRANCH}"
print_info "Rebuilding development environment on branch: $branch"
cd "$PROJECT_ROOT"
export GIT_BRANCH="$branch"
# Stop existing containers
podman-compose -f "$COMPOSE_FILE" down
# Rebuild with no cache
print_info "Building image (no cache) for branch: $branch"
podman-compose -f "$COMPOSE_FILE" build --no-cache --build-arg GIT_BRANCH="$branch"
# Start
print_info "Starting container..."
podman-compose -f "$COMPOSE_FILE" up -d
print_success "Development environment rebuilt and started on branch: $branch"
}
# Function to show logs
show_logs() {
cd "$PROJECT_ROOT"
podman-compose -f "$COMPOSE_FILE" logs "$@"
}
# Function to open shell
open_shell() {
cd "$PROJECT_ROOT"
local container=$(podman-compose -f "$COMPOSE_FILE" ps -q | head -1)
if [ -z "$container" ]; then
print_error "No running container found"
exit 1
fi
print_info "Opening shell in container..."
podman exec -it "$container" /bin/bash || podman exec -it "$container" /bin/sh
}
# Function to show status
show_status() {
cd "$PROJECT_ROOT"
print_info "Development environment status:"
podman-compose -f "$COMPOSE_FILE" ps
}
# Function to clean everything
clean_all() {
print_warning "This will remove all containers, images, and volumes. Continue? (y/N)"
read -r response
if [[ "$response" =~ ^[Yy]$ ]]; then
cd "$PROJECT_ROOT"
podman-compose -f "$COMPOSE_FILE" down -v
podman images | grep mev-bot | awk '{print $3}' | xargs -r podman rmi -f
print_success "Cleaned all containers, images, and volumes"
else
print_info "Cancelled"
fi
}
# Function to switch branches
switch_branch() {
local branch="$1"
if [ -z "$branch" ]; then
print_error "Please specify a branch"
usage
exit 1
fi
print_info "Switching to branch: $branch"
cd "$PROJECT_ROOT"
# Check if branch exists
if ! git rev-parse --verify "$branch" &> /dev/null; then
print_error "Branch '$branch' does not exist"
list_branches
exit 1
fi
# Stop current environment
stop_dev
# Checkout branch
print_info "Checking out branch: $branch"
git checkout "$branch"
# Rebuild and start
rebuild_dev "$branch"
}
# Main script logic
main() {
check_dependencies
case "${1:-}" in
start)
start_dev "${2:-$DEFAULT_BRANCH}"
;;
stop)
stop_dev
;;
restart)
restart_dev "${2:-$DEFAULT_BRANCH}"
;;
rebuild)
rebuild_dev "${2:-$DEFAULT_BRANCH}"
;;
logs)
shift
show_logs "$@"
;;
shell)
open_shell
;;
status)
show_status
;;
clean)
clean_all
;;
branches)
list_branches
;;
switch)
switch_branch "$2"
;;
-h|--help|help)
usage
;;
*)
print_error "Unknown command: ${1:-}"
echo ""
usage
exit 1
;;
esac
}
# Run main function
main "$@"

96
scripts/git-hooks/post-merge Executable file
View File

@@ -0,0 +1,96 @@
#!/bin/bash
# Git post-merge hook
# This hook runs after 'git pull' or 'git merge'
# Automatically rebuilds and restarts the MEV bot
set -e
HOOK_DIR=$(dirname "$0")
PROJECT_DIR=$(cd "$HOOK_DIR/../.." && pwd)
echo "========================================="
echo "Post-Merge Hook: Auto-Rebuild & Restart"
echo "========================================="
echo "Project: $PROJECT_DIR"
echo ""
# Color codes
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m' # No Color
# Change to project directory
cd "$PROJECT_DIR"
# Check if Docker Compose is available
if ! command -v docker &> /dev/null; then
echo -e "${RED}Error: Docker not found${NC}"
exit 1
fi
# Log the update
LOG_FILE="$PROJECT_DIR/logs/auto-update.log"
mkdir -p "$PROJECT_DIR/logs"
echo "[$(date)] Post-merge hook triggered" >> "$LOG_FILE"
# Get the latest commit info
LATEST_COMMIT=$(git log -1 --pretty=format:"%h - %s")
echo -e "${YELLOW}Latest commit: $LATEST_COMMIT${NC}"
echo "[$(date)] Latest commit: $LATEST_COMMIT" >> "$LOG_FILE"
# Rebuild the Docker image
echo ""
echo -e "${YELLOW}Rebuilding Docker image...${NC}"
if docker compose build --no-cache >> "$LOG_FILE" 2>&1; then
echo -e "${GREEN}✓ Docker image rebuilt${NC}"
echo "[$(date)] Docker image rebuilt successfully" >> "$LOG_FILE"
else
echo -e "${RED}✗ Docker build failed - check logs${NC}"
echo "[$(date)] ERROR: Docker build failed" >> "$LOG_FILE"
exit 1
fi
# Restart the container
echo ""
echo -e "${YELLOW}Restarting container...${NC}"
if docker compose up -d >> "$LOG_FILE" 2>&1; then
echo -e "${GREEN}✓ Container restarted${NC}"
echo "[$(date)] Container restarted successfully" >> "$LOG_FILE"
else
echo -e "${RED}✗ Container restart failed - check logs${NC}"
echo "[$(date)] ERROR: Container restart failed" >> "$LOG_FILE"
exit 1
fi
# Wait a few seconds for the container to start
sleep 5
# Check container status
echo ""
echo -e "${YELLOW}Container status:${NC}"
docker compose ps
# Show recent logs
echo ""
echo -e "${YELLOW}Recent logs:${NC}"
docker compose logs --tail=20 mev-bot
echo ""
echo -e "${GREEN}========================================="
echo "Auto-update complete!"
echo "=========================================${NC}"
echo ""
echo "View full logs: tail -f $LOG_FILE"
echo "View container logs: docker compose logs -f mev-bot"
echo ""
# Send notification if curl is available (optional)
if command -v curl &> /dev/null && [ -n "$WEBHOOK_URL" ]; then
curl -X POST "$WEBHOOK_URL" \
-H "Content-Type: application/json" \
-d "{\"text\":\"MEV Bot updated to: $LATEST_COMMIT\"}" \
>> "$LOG_FILE" 2>&1 || true
fi
exit 0

46
scripts/install-git-hooks.sh Executable file
View File

@@ -0,0 +1,46 @@
#!/bin/bash
# Install Git Hooks for Auto-Update
set -e
PROJECT_DIR=$(cd "$(dirname "$0")/.." && pwd)
HOOKS_SRC_DIR="$PROJECT_DIR/scripts/git-hooks"
HOOKS_DEST_DIR="$PROJECT_DIR/.git/hooks"
echo "========================================="
echo "Installing Git Hooks"
echo "========================================="
echo ""
# Check if .git directory exists
if [ ! -d "$PROJECT_DIR/.git" ]; then
echo "Error: Not a git repository"
exit 1
fi
# Create hooks directory if it doesn't exist
mkdir -p "$HOOKS_DEST_DIR"
# Install post-merge hook
if [ -f "$HOOKS_SRC_DIR/post-merge" ]; then
echo "Installing post-merge hook..."
cp "$HOOKS_SRC_DIR/post-merge" "$HOOKS_DEST_DIR/post-merge"
chmod +x "$HOOKS_DEST_DIR/post-merge"
echo "✓ post-merge hook installed"
else
echo "✗ post-merge hook not found at $HOOKS_SRC_DIR/post-merge"
exit 1
fi
echo ""
echo "========================================="
echo "Git Hooks Installed Successfully!"
echo "========================================="
echo ""
echo "The following hooks are now active:"
echo " - post-merge: Auto-rebuild and restart after 'git pull'"
echo ""
echo "Hooks will run automatically when you:"
echo " - Run 'git pull'"
echo " - Run './scripts/auto-update.sh'"
echo ""

View File

@@ -0,0 +1,27 @@
[Unit]
Description=MEV Bot Auto-Update Service
After=network-online.target docker.service
Wants=network-online.target
[Service]
Type=oneshot
WorkingDirectory=/docker/mev-beta
ExecStart=/docker/mev-beta/scripts/auto-update.sh
StandardOutput=append:/docker/mev-beta/logs/auto-update.log
StandardError=append:/docker/mev-beta/logs/auto-update.log
# Environment variables (optional)
# Environment="WEBHOOK_URL=https://your-webhook-url"
# Environment="GIT_BRANCH=master"
# Environment="GIT_REMOTE=origin"
# Security settings
User=root
Group=docker
# Prevent service from failing if update fails
# This ensures the timer continues to run
SuccessExitStatus=0 1
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,14 @@
[Unit]
Description=MEV Bot Auto-Update Timer
Requires=mev-bot-auto-update.service
[Timer]
# Check for updates every 5 minutes
OnBootSec=2min
OnUnitActiveSec=5min
# Run timer even if system was off when it should have run
Persistent=true
[Install]
WantedBy=timers.target

149
scripts/setup-auto-update.sh Executable file
View File

@@ -0,0 +1,149 @@
#!/bin/bash
# Setup Auto-Update System for MEV Bot
# This script installs git hooks and optionally systemd timers for automatic updates
set -e
PROJECT_DIR=$(cd "$(dirname "$0")/.." && pwd)
# Color codes
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
echo -e "${BLUE}========================================="
echo "MEV Bot Auto-Update Setup"
echo "=========================================${NC}"
echo ""
# Check if running as root
IS_ROOT=false
if [ "$EUID" -eq 0 ]; then
IS_ROOT=true
echo -e "${YELLOW}Running as root - will setup systemd timer${NC}"
else
echo -e "${YELLOW}Not running as root - will skip systemd timer${NC}"
echo -e "${YELLOW}Run with sudo to enable automatic periodic updates${NC}"
fi
echo ""
# Step 1: Install git hooks
echo -e "${BLUE}Step 1: Installing Git Hooks${NC}"
echo "-------------------------------------------"
cd "$PROJECT_DIR"
if [ -f "$PROJECT_DIR/scripts/install-git-hooks.sh" ]; then
"$PROJECT_DIR/scripts/install-git-hooks.sh"
echo -e "${GREEN}✓ Git hooks installed${NC}"
else
echo -e "${RED}✗ Git hooks installation script not found${NC}"
exit 1
fi
echo ""
# Step 2: Setup systemd timer (if root)
if [ "$IS_ROOT" = true ]; then
echo -e "${BLUE}Step 2: Setting up Systemd Timer${NC}"
echo "-------------------------------------------"
# Update WorkingDirectory in service files
sed "s|WorkingDirectory=/docker/mev-beta|WorkingDirectory=$PROJECT_DIR|g" \
"$PROJECT_DIR/scripts/mev-bot-auto-update.service" > /tmp/mev-bot-auto-update.service
sed "s|/docker/mev-beta|$PROJECT_DIR|g" \
/tmp/mev-bot-auto-update.service > /tmp/mev-bot-auto-update.service.tmp
mv /tmp/mev-bot-auto-update.service.tmp /tmp/mev-bot-auto-update.service
# Copy service and timer files
cp /tmp/mev-bot-auto-update.service /etc/systemd/system/
cp "$PROJECT_DIR/scripts/mev-bot-auto-update.timer" /etc/systemd/system/
# Reload systemd
systemctl daemon-reload
# Enable and start the timer
systemctl enable mev-bot-auto-update.timer
systemctl start mev-bot-auto-update.timer
echo -e "${GREEN}✓ Systemd timer enabled and started${NC}"
echo ""
# Show timer status
echo -e "${YELLOW}Timer status:${NC}"
systemctl status mev-bot-auto-update.timer --no-pager | head -10
echo ""
# Show next scheduled run
echo -e "${YELLOW}Next scheduled update check:${NC}"
systemctl list-timers mev-bot-auto-update.timer --no-pager
echo ""
else
echo -e "${BLUE}Step 2: Systemd Timer (Skipped)${NC}"
echo "-------------------------------------------"
echo -e "${YELLOW}Run 'sudo ./scripts/setup-auto-update.sh' to enable automatic updates${NC}"
echo ""
fi
# Step 3: Create logs directory
echo -e "${BLUE}Step 3: Creating Logs Directory${NC}"
echo "-------------------------------------------"
mkdir -p "$PROJECT_DIR/logs"
echo -e "${GREEN}✓ Logs directory created${NC}"
echo ""
# Summary
echo -e "${GREEN}========================================="
echo "Auto-Update Setup Complete!"
echo "=========================================${NC}"
echo ""
echo "What's been configured:"
echo ""
echo "1. Git Hooks:"
echo " ✓ post-merge hook installed"
echo " → Triggers auto-rebuild after 'git pull'"
echo ""
if [ "$IS_ROOT" = true ]; then
echo "2. Systemd Timer:"
echo " ✓ Checks for updates every 5 minutes"
echo " ✓ Starts automatically on boot"
echo " ✓ Pulls and rebuilds when updates detected"
echo ""
echo "Manage the timer:"
echo " sudo systemctl status mev-bot-auto-update.timer"
echo " sudo systemctl stop mev-bot-auto-update.timer"
echo " sudo systemctl start mev-bot-auto-update.timer"
echo " sudo journalctl -u mev-bot-auto-update -f"
echo ""
else
echo "2. Manual Updates:"
echo " Run: ./scripts/auto-update.sh"
echo " Or: git pull (hooks will auto-rebuild)"
echo ""
fi
echo "3. Logs:"
echo " Auto-update log: tail -f logs/auto-update.log"
echo " Container log: docker compose logs -f mev-bot"
echo ""
echo -e "${BLUE}Optional: Webhook Receiver${NC}"
echo "-------------------------------------------"
echo "For immediate updates via GitHub/GitLab webhooks:"
echo ""
echo "1. Configure webhook in your Git provider:"
echo " URL: http://your-server:9000/webhook"
echo " Events: Push events (master branch)"
echo ""
echo "2. Start the webhook receiver:"
echo " ./scripts/webhook-receiver.sh"
echo ""
echo "Or run as systemd service (advanced setup needed)"
echo ""
echo -e "${GREEN}Your MEV bot will now auto-update! 🚀${NC}"
echo ""

200
scripts/test-calculations.sh Executable file
View File

@@ -0,0 +1,200 @@
#!/bin/bash
# Test Calculations Framework - Extract and validate arbitrage calculations from logs
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
TEST_DIR="$PROJECT_ROOT/tests/calculation-validation"
LOG_FILE="${1:-/tmp/mev_full_logs.txt}"
# Colors for output
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════${NC}"
echo -e "${BLUE} MEV Bot - Arbitrage Calculation Validation Framework${NC}"
echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════${NC}"
echo ""
# Create test directory
mkdir -p "$TEST_DIR/extracted"
mkdir -p "$TEST_DIR/reports"
echo -e "${YELLOW}Step 1: Extracting arbitrage opportunities from logs...${NC}"
# Check if log file exists
if [ ! -f "$LOG_FILE" ]; then
echo -e "${YELLOW}Log file not found at $LOG_FILE${NC}"
echo -e "${YELLOW}Extracting from running container...${NC}"
podman logs mev-bot-dev-master-dev 2>&1 > "$LOG_FILE"
fi
# Extract executable opportunities
echo -e "${GREEN}Extracting executable opportunities...${NC}"
grep "EXECUTABLE OPPORTUNITY" "$LOG_FILE" > "$TEST_DIR/extracted/executable_opportunities.log" || true
EXEC_COUNT=$(wc -l < "$TEST_DIR/extracted/executable_opportunities.log" || echo "0")
echo -e " ✓ Found $EXEC_COUNT executable opportunities"
# Extract opportunity details
echo -e "${GREEN}Extracting opportunity details...${NC}"
grep "ARBITRAGE OPPORTUNITY DETECTED" "$LOG_FILE" -A 20 > "$TEST_DIR/extracted/opportunity_details.log" || true
DETAIL_COUNT=$(grep -c "ARBITRAGE OPPORTUNITY DETECTED" "$TEST_DIR/extracted/opportunity_details.log" || echo "0")
echo -e " ✓ Found $DETAIL_COUNT opportunity records"
# Extract V3 swap calculations
echo -e "${GREEN}Extracting V3 swap calculations...${NC}"
grep "V3 calculation:" "$LOG_FILE" > "$TEST_DIR/extracted/v3_calculations.log" || true
V3_COUNT=$(wc -l < "$TEST_DIR/extracted/v3_calculations.log" || echo "0")
echo -e " ✓ Found $V3_COUNT V3 swap calculations"
# Extract profit threshold checks
echo -e "${GREEN}Extracting profit threshold checks...${NC}"
grep "Profit threshold check:" "$LOG_FILE" > "$TEST_DIR/extracted/threshold_checks.log" || true
THRESHOLD_COUNT=$(wc -l < "$TEST_DIR/extracted/threshold_checks.log" || echo "0")
echo -e " ✓ Found $THRESHOLD_COUNT threshold validation checks"
# Extract rejected opportunities
echo -e "${GREEN}Extracting rejected opportunities...${NC}"
grep "rejectReason:" "$LOG_FILE" | grep -v "rejectReason: " | grep -v "rejectReason:$" > "$TEST_DIR/extracted/rejections.log" || true
REJECT_COUNT=$(wc -l < "$TEST_DIR/extracted/rejections.log" || echo "0")
echo -e " ✓ Found $REJECT_COUNT rejections"
echo ""
echo -e "${YELLOW}Step 2: Generating summary report...${NC}"
REPORT_FILE="$TEST_DIR/reports/validation_report_$(date +%Y%m%d_%H%M%S).md"
cat > "$REPORT_FILE" <<EOF
# Arbitrage Calculation Validation Report
**Generated:** $(date)
**Log File:** $LOG_FILE
## Summary Statistics
- **Executable Opportunities:** $EXEC_COUNT
- **Total Opportunity Records:** $DETAIL_COUNT
- **V3 Calculations:** $V3_COUNT
- **Threshold Checks:** $THRESHOLD_COUNT
- **Rejections:** $REJECT_COUNT
## Executable Opportunities Analysis
EOF
if [ $EXEC_COUNT -gt 0 ]; then
echo "### Top 10 Executable Opportunities" >> "$REPORT_FILE"
echo '```' >> "$REPORT_FILE"
head -10 "$TEST_DIR/extracted/executable_opportunities.log" >> "$REPORT_FILE"
echo '```' >> "$REPORT_FILE"
echo "" >> "$REPORT_FILE"
fi
echo "## Profit Calculation Validation" >> "$REPORT_FILE"
echo "" >> "$REPORT_FILE"
if [ $THRESHOLD_COUNT -gt 0 ]; then
echo "### Sample Threshold Checks (First 5)" >> "$REPORT_FILE"
echo '```' >> "$REPORT_FILE"
head -5 "$TEST_DIR/extracted/threshold_checks.log" >> "$REPORT_FILE"
echo '```' >> "$REPORT_FILE"
echo "" >> "$REPORT_FILE"
fi
echo "## Rejection Analysis" >> "$REPORT_FILE"
echo "" >> "$REPORT_FILE"
if [ $REJECT_COUNT -gt 0 ]; then
echo "### Rejection Reasons Breakdown" >> "$REPORT_FILE"
echo '```' >> "$REPORT_FILE"
grep -oP 'rejectReason:\K[^,}]+' "$TEST_DIR/extracted/rejections.log" | sort | uniq -c | sort -rn >> "$REPORT_FILE"
echo '```' >> "$REPORT_FILE"
echo "" >> "$REPORT_FILE"
fi
echo "## V3 Calculation Samples" >> "$REPORT_FILE"
echo "" >> "$REPORT_FILE"
if [ $V3_COUNT -gt 0 ]; then
echo "### Recent V3 Calculations (Last 10)" >> "$REPORT_FILE"
echo '```' >> "$REPORT_FILE"
tail -10 "$TEST_DIR/extracted/v3_calculations.log" >> "$REPORT_FILE"
echo '```' >> "$REPORT_FILE"
echo "" >> "$REPORT_FILE"
fi
echo "---" >> "$REPORT_FILE"
echo "**Report saved to:** $REPORT_FILE" >> "$REPORT_FILE"
echo -e "${GREEN}✓ Report generated: $REPORT_FILE${NC}"
echo ""
echo -e "${YELLOW}Step 3: Analyzing calculation accuracy...${NC}"
# Parse and validate calculations
if [ $EXEC_COUNT -gt 0 ]; then
echo -e "${GREEN}Validating executable opportunity calculations...${NC}"
# Extract profit values
grep -oP 'Profit=\K[0-9.]+' "$TEST_DIR/extracted/executable_opportunities.log" > "$TEST_DIR/extracted/profit_values.txt" || true
# Calculate statistics
if [ -f "$TEST_DIR/extracted/profit_values.txt" ] && [ -s "$TEST_DIR/extracted/profit_values.txt" ]; then
TOTAL_PROFIT=$(awk '{sum+=$1} END {print sum}' "$TEST_DIR/extracted/profit_values.txt")
AVG_PROFIT=$(awk '{sum+=$1} END {print sum/NR}' "$TEST_DIR/extracted/profit_values.txt")
MAX_PROFIT=$(sort -n "$TEST_DIR/extracted/profit_values.txt" | tail -1)
MIN_PROFIT=$(sort -n "$TEST_DIR/extracted/profit_values.txt" | head -1)
echo -e " ${BLUE}Total Profit Detected:${NC} $TOTAL_PROFIT ETH"
echo -e " ${BLUE}Average Profit:${NC} $AVG_PROFIT ETH"
echo -e " ${BLUE}Max Profit:${NC} $MAX_PROFIT ETH"
echo -e " ${BLUE}Min Profit:${NC} $MIN_PROFIT ETH"
fi
fi
echo ""
echo -e "${YELLOW}Step 4: Extracting test data for replay...${NC}"
# Create JSON test data for replay
TEST_DATA_FILE="$TEST_DIR/extracted/test_data.json"
echo "{" > "$TEST_DATA_FILE"
echo " \"timestamp\": \"$(date -Iseconds)\"," >> "$TEST_DATA_FILE"
echo " \"executableOpportunities\": $EXEC_COUNT," >> "$TEST_DATA_FILE"
echo " \"totalOpportunities\": $DETAIL_COUNT," >> "$TEST_DATA_FILE"
echo " \"v3Calculations\": $V3_COUNT," >> "$TEST_DATA_FILE"
echo " \"thresholdChecks\": $THRESHOLD_COUNT," >> "$TEST_DATA_FILE"
echo " \"rejections\": $REJECT_COUNT," >> "$TEST_DATA_FILE"
echo " \"logFile\": \"$LOG_FILE\"," >> "$TEST_DATA_FILE"
echo " \"extractedFiles\": {" >> "$TEST_DATA_FILE"
echo " \"executable\": \"$TEST_DIR/extracted/executable_opportunities.log\"," >> "$TEST_DATA_FILE"
echo " \"details\": \"$TEST_DIR/extracted/opportunity_details.log\"," >> "$TEST_DATA_FILE"
echo " \"v3Calcs\": \"$TEST_DIR/extracted/v3_calculations.log\"," >> "$TEST_DATA_FILE"
echo " \"thresholds\": \"$TEST_DIR/extracted/threshold_checks.log\"," >> "$TEST_DATA_FILE"
echo " \"rejections\": \"$TEST_DIR/extracted/rejections.log\"" >> "$TEST_DATA_FILE"
echo " }" >> "$TEST_DATA_FILE"
echo "}" >> "$TEST_DATA_FILE"
echo -e "${GREEN}✓ Test data saved: $TEST_DATA_FILE${NC}"
echo ""
echo -e "${GREEN}═══════════════════════════════════════════════════════════════════════${NC}"
echo -e "${GREEN} Extraction Complete!${NC}"
echo -e "${GREEN}═══════════════════════════════════════════════════════════════════════${NC}"
echo ""
echo -e "${BLUE}Next Steps:${NC}"
echo -e " 1. Review report: cat $REPORT_FILE"
echo -e " 2. Run Go validation tests: go test ./tests/calculation-validation/..."
echo -e " 3. Replay calculations: go run ./tests/calculation-validation/replay.go"
echo ""
echo -e "${YELLOW}Files Created:${NC}"
echo -e " - Executable opportunities: $TEST_DIR/extracted/executable_opportunities.log"
echo -e " - Opportunity details: $TEST_DIR/extracted/opportunity_details.log"
echo -e " - V3 calculations: $TEST_DIR/extracted/v3_calculations.log"
echo -e " - Threshold checks: $TEST_DIR/extracted/threshold_checks.log"
echo -e " - Rejections: $TEST_DIR/extracted/rejections.log"
echo -e " - Test data JSON: $TEST_DATA_FILE"
echo -e " - Validation report: $REPORT_FILE"
echo ""

53
scripts/webhook-receiver.sh Executable file
View File

@@ -0,0 +1,53 @@
#!/bin/bash
# Simple webhook receiver for GitHub/GitLab push events
# This creates a lightweight HTTP server that triggers updates on push
PORT="${WEBHOOK_PORT:-9000}"
SECRET="${WEBHOOK_SECRET:-change-me-in-production}"
PROJECT_DIR="${PROJECT_DIR:-$(cd "$(dirname "$0")/.." && pwd)}"
LOG_FILE="${PROJECT_DIR}/logs/webhook.log"
mkdir -p "${PROJECT_DIR}/logs"
echo "========================================="
echo "MEV Bot Webhook Receiver"
echo "========================================="
echo "Listening on port: $PORT"
echo "Project: $PROJECT_DIR"
echo "Log file: $LOG_FILE"
echo ""
echo "Configure your GitHub/GitLab webhook to:"
echo " URL: http://your-server:$PORT/webhook"
echo " Secret: $SECRET"
echo " Events: Push events on master branch"
echo ""
echo "Press Ctrl+C to stop"
echo "========================================="
echo ""
# Simple HTTP server using netcat and bash
while true; do
# Listen for incoming requests
RESPONSE=$(echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nWebhook received" | nc -l -p "$PORT" -q 1)
# Log the request
echo "[$(date)] Webhook received" | tee -a "$LOG_FILE"
# Check if it's a valid webhook request (basic check)
if echo "$RESPONSE" | grep -q "POST /webhook"; then
echo "[$(date)] Valid webhook request detected" | tee -a "$LOG_FILE"
# Trigger auto-update in background
(
cd "$PROJECT_DIR"
./scripts/auto-update.sh >> "$LOG_FILE" 2>&1
) &
echo "[$(date)] Auto-update triggered" | tee -a "$LOG_FILE"
else
echo "[$(date)] Invalid request ignored" | tee -a "$LOG_FILE"
fi
# Small delay before listening again
sleep 1
done