c54c569f30
Completed clean root directory structure: - Root now contains only: .git, .env, docs/, orig/ - Moved all remaining files and directories to orig/: - Config files (.claude, .dockerignore, .drone.yml, etc.) - All .env variants (except active .env) - Git config (.gitconfig, .github, .gitignore, etc.) - Tool configs (.golangci.yml, .revive.toml, etc.) - Documentation (*.md files, @prompts) - Build files (Dockerfiles, Makefile, go.mod, go.sum) - Docker compose files - All source directories (scripts, tests, tools, etc.) - Runtime directories (logs, monitoring, reports) - Dependency files (node_modules, lib, cache) - Special files (--delete) - Removed empty runtime directories (bin/, data/) V2 structure is now clean: - docs/planning/ - V2 planning documents - orig/ - Complete V1 codebase preserved - .env - Active environment config (not in git) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
353 lines
12 KiB
Bash
Executable File
353 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
#############################################################################
|
|
# 24-Hour Production Validation Test
|
|
#
|
|
# This script runs the MEV bot for 24 hours with comprehensive monitoring
|
|
# and validation to ensure production readiness.
|
|
#
|
|
# Usage: ./scripts/24h-validation-test.sh
|
|
#############################################################################
|
|
|
|
set -euo pipefail
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Configuration
|
|
TEST_DURATION_HOURS=24
|
|
LOG_DIR="logs/24h_validation_$(date +%Y%m%d_%H%M%S)"
|
|
PID_FILE="/tmp/mev-bot-24h-test.pid"
|
|
REPORT_FILE="${LOG_DIR}/validation_report.md"
|
|
METRICS_FILE="${LOG_DIR}/metrics.json"
|
|
|
|
# Create log directory
|
|
mkdir -p "${LOG_DIR}"
|
|
|
|
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
|
|
echo -e "${BLUE} 24-Hour MEV Bot Production Validation Test${NC}"
|
|
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
|
|
echo ""
|
|
echo -e "${GREEN}Test Start Time:${NC} $(date)"
|
|
echo -e "${GREEN}Test Duration:${NC} ${TEST_DURATION_HOURS} hours"
|
|
echo -e "${GREEN}Log Directory:${NC} ${LOG_DIR}"
|
|
echo ""
|
|
|
|
#############################################################################
|
|
# Pre-Flight Checks
|
|
#############################################################################
|
|
|
|
echo -e "${YELLOW}[1/7] Running Pre-Flight Checks...${NC}"
|
|
|
|
# Check if bot binary exists
|
|
if [ ! -f "./bin/mev-bot" ]; then
|
|
echo -e "${RED}✗ Error: MEV bot binary not found${NC}"
|
|
echo -e "${YELLOW}Building binary...${NC}"
|
|
make build
|
|
fi
|
|
|
|
# Check environment variables
|
|
if [ -z "${ARBITRUM_RPC_ENDPOINT:-}" ]; then
|
|
echo -e "${RED}✗ Error: ARBITRUM_RPC_ENDPOINT not set${NC}"
|
|
exit 1
|
|
fi
|
|
|
|
# Check if provider config exists
|
|
if [ ! -f "./config/providers_runtime.yaml" ]; then
|
|
echo -e "${RED}✗ Error: Provider config not found${NC}"
|
|
exit 1
|
|
fi
|
|
|
|
# Test RPC connection
|
|
echo -e "${YELLOW}Testing RPC connection...${NC}"
|
|
if ! timeout 10 ./bin/mev-bot --version &>/dev/null; then
|
|
echo -e "${YELLOW}Warning: Could not verify bot version${NC}"
|
|
fi
|
|
|
|
echo -e "${GREEN}✓ Pre-flight checks passed${NC}"
|
|
echo ""
|
|
|
|
#############################################################################
|
|
# Initialize Monitoring
|
|
#############################################################################
|
|
|
|
echo -e "${YELLOW}[2/7] Initializing Monitoring...${NC}"
|
|
|
|
# Create monitoring script
|
|
cat > "${LOG_DIR}/monitor.sh" << 'MONITOR_EOF'
|
|
#!/bin/bash
|
|
LOG_FILE="$1"
|
|
METRICS_FILE="$2"
|
|
|
|
while true; do
|
|
# Extract metrics from logs
|
|
OPPORTUNITIES=$(grep -c "ARBITRAGE OPPORTUNITY DETECTED" "$LOG_FILE" 2>/dev/null || echo "0")
|
|
PROFITABLE=$(grep "Net Profit:" "$LOG_FILE" | grep -v "negative" | wc -l || echo "0")
|
|
EVENTS_PROCESSED=$(grep -c "Worker.*processing.*event" "$LOG_FILE" 2>/dev/null || echo "0")
|
|
ERRORS=$(grep -c "\[ERROR\]" "$LOG_FILE" 2>/dev/null || echo "0")
|
|
WARNINGS=$(grep -c "\[WARN\]" "$LOG_FILE" 2>/dev/null || echo "0")
|
|
|
|
# Cache metrics
|
|
CACHE_HITS=$(grep "Reserve cache metrics" "$LOG_FILE" | tail -1 | grep -oP 'hits=\K[0-9]+' || echo "0")
|
|
CACHE_MISSES=$(grep "Reserve cache metrics" "$LOG_FILE" | tail -1 | grep -oP 'misses=\K[0-9]+' || echo "0")
|
|
|
|
# Calculate hit rate
|
|
if [ "$CACHE_HITS" -gt 0 ] || [ "$CACHE_MISSES" -gt 0 ]; then
|
|
TOTAL=$((CACHE_HITS + CACHE_MISSES))
|
|
HIT_RATE=$(awk "BEGIN {print ($CACHE_HITS / $TOTAL) * 100}")
|
|
else
|
|
HIT_RATE="0"
|
|
fi
|
|
|
|
# System metrics
|
|
CPU=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1)
|
|
MEM=$(free | grep Mem | awk '{print ($3/$2) * 100.0}')
|
|
DISK=$(df -h . | tail -1 | awk '{print $5}' | sed 's/%//')
|
|
|
|
# Write metrics JSON
|
|
cat > "$METRICS_FILE" << METRICS
|
|
{
|
|
"timestamp": "$(date -Iseconds)",
|
|
"uptime_seconds": $SECONDS,
|
|
"opportunities_detected": $OPPORTUNITIES,
|
|
"profitable_opportunities": $PROFITABLE,
|
|
"events_processed": $EVENTS_PROCESSED,
|
|
"errors": $ERRORS,
|
|
"warnings": $WARNINGS,
|
|
"cache": {
|
|
"hits": $CACHE_HITS,
|
|
"misses": $CACHE_MISSES,
|
|
"hit_rate_percent": $HIT_RATE
|
|
},
|
|
"system": {
|
|
"cpu_percent": $CPU,
|
|
"memory_percent": $MEM,
|
|
"disk_percent": $DISK
|
|
}
|
|
}
|
|
METRICS
|
|
|
|
sleep 60
|
|
done
|
|
MONITOR_EOF
|
|
|
|
chmod +x "${LOG_DIR}/monitor.sh"
|
|
|
|
echo -e "${GREEN}✓ Monitoring initialized${NC}"
|
|
echo ""
|
|
|
|
#############################################################################
|
|
# Start MEV Bot
|
|
#############################################################################
|
|
|
|
echo -e "${YELLOW}[3/7] Starting MEV Bot...${NC}"
|
|
|
|
# Set environment variables for the test
|
|
export LOG_LEVEL="info"
|
|
export PROVIDER_CONFIG_PATH="$PWD/config/providers_runtime.yaml"
|
|
|
|
# Start the bot with timeout
|
|
nohup timeout ${TEST_DURATION_HOURS}h ./bin/mev-bot start \
|
|
> "${LOG_DIR}/mev_bot.log" 2>&1 &
|
|
|
|
BOT_PID=$!
|
|
echo "$BOT_PID" > "$PID_FILE"
|
|
|
|
echo -e "${GREEN}✓ MEV Bot started (PID: $BOT_PID)${NC}"
|
|
echo ""
|
|
|
|
# Wait for bot to initialize
|
|
echo -e "${YELLOW}Waiting for bot initialization (30 seconds)...${NC}"
|
|
sleep 30
|
|
|
|
# Check if bot is still running
|
|
if ! kill -0 "$BOT_PID" 2>/dev/null; then
|
|
echo -e "${RED}✗ Error: Bot failed to start${NC}"
|
|
echo -e "${YELLOW}Last 50 lines of log:${NC}"
|
|
tail -50 "${LOG_DIR}/mev_bot.log"
|
|
exit 1
|
|
fi
|
|
|
|
echo -e "${GREEN}✓ Bot initialized successfully${NC}"
|
|
echo ""
|
|
|
|
#############################################################################
|
|
# Start Monitoring
|
|
#############################################################################
|
|
|
|
echo -e "${YELLOW}[4/7] Starting Background Monitoring...${NC}"
|
|
|
|
"${LOG_DIR}/monitor.sh" "${LOG_DIR}/mev_bot.log" "$METRICS_FILE" &
|
|
MONITOR_PID=$!
|
|
|
|
echo -e "${GREEN}✓ Monitoring started (PID: $MONITOR_PID)${NC}"
|
|
echo ""
|
|
|
|
#############################################################################
|
|
# Real-Time Status Display
|
|
#############################################################################
|
|
|
|
echo -e "${YELLOW}[5/7] Monitoring Progress...${NC}"
|
|
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
|
|
echo ""
|
|
echo -e "${GREEN}Test is now running for ${TEST_DURATION_HOURS} hours${NC}"
|
|
echo -e "${YELLOW}Press Ctrl+C to stop early and generate report${NC}"
|
|
echo ""
|
|
echo -e "Log file: ${LOG_DIR}/mev_bot.log"
|
|
echo -e "Metrics file: ${METRICS_FILE}"
|
|
echo ""
|
|
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
|
|
echo ""
|
|
|
|
# Trap Ctrl+C to generate report
|
|
trap 'echo -e "\n${YELLOW}Stopping test early...${NC}"; kill $BOT_PID $MONITOR_PID 2>/dev/null; generate_report; exit 0' INT
|
|
|
|
# Function to generate report
|
|
generate_report() {
|
|
echo -e "\n${YELLOW}[6/7] Generating Validation Report...${NC}"
|
|
|
|
# Read final metrics
|
|
if [ -f "$METRICS_FILE" ]; then
|
|
METRICS=$(cat "$METRICS_FILE")
|
|
else
|
|
METRICS="{}"
|
|
fi
|
|
|
|
# Generate markdown report
|
|
cat > "$REPORT_FILE" << REPORT_EOF
|
|
# 24-Hour Production Validation Test Report
|
|
|
|
**Test Date:** $(date)
|
|
**Duration:** ${TEST_DURATION_HOURS} hours
|
|
**Status:** COMPLETED
|
|
|
|
---
|
|
|
|
## Summary
|
|
|
|
\`\`\`json
|
|
$METRICS
|
|
\`\`\`
|
|
|
|
---
|
|
|
|
## Key Metrics
|
|
|
|
### Opportunities
|
|
- **Total Detected:** $(echo "$METRICS" | grep -oP '"opportunities_detected":\s*\K[0-9]+' || echo "N/A")
|
|
- **Profitable:** $(echo "$METRICS" | grep -oP '"profitable_opportunities":\s*\K[0-9]+' || echo "N/A")
|
|
- **Events Processed:** $(echo "$METRICS" | grep -oP '"events_processed":\s*\K[0-9]+' || echo "N/A")
|
|
|
|
### Cache Performance
|
|
- **Hit Rate:** $(echo "$METRICS" | grep -oP '"hit_rate_percent":\s*\K[0-9.]+' || echo "N/A")%
|
|
- **Target:** 75-85%
|
|
- **Status:** $(if [ "$(echo "$METRICS" | grep -oP '"hit_rate_percent":\s*\K[0-9.]+' || echo "0")" -ge 75 ]; then echo "✓ PASS"; else echo "⚠ BELOW TARGET"; fi)
|
|
|
|
### System Health
|
|
- **CPU Usage:** $(echo "$METRICS" | grep -oP '"cpu_percent":\s*\K[0-9.]+' || echo "N/A")%
|
|
- **Memory Usage:** $(echo "$METRICS" | grep -oP '"memory_percent":\s*\K[0-9.]+' || echo "N/A")%
|
|
- **Errors:** $(echo "$METRICS" | grep -oP '"errors":\s*\K[0-9]+' || echo "N/A")
|
|
- **Warnings:** $(echo "$METRICS" | grep -oP '"warnings":\s*\K[0-9]+' || echo "N/A")
|
|
|
|
---
|
|
|
|
## Log Analysis
|
|
|
|
### Top Errors
|
|
\`\`\`
|
|
$(grep "\[ERROR\]" "${LOG_DIR}/mev_bot.log" | sort | uniq -c | sort -rn | head -10)
|
|
\`\`\`
|
|
|
|
### Top Warnings
|
|
\`\`\`
|
|
$(grep "\[WARN\]" "${LOG_DIR}/mev_bot.log" | sort | uniq -c | sort -rn | head -10)
|
|
\`\`\`
|
|
|
|
### Sample Opportunities
|
|
\`\`\`
|
|
$(grep "ARBITRAGE OPPORTUNITY DETECTED" "${LOG_DIR}/mev_bot.log" | head -5)
|
|
\`\`\`
|
|
|
|
---
|
|
|
|
## Validation Criteria
|
|
|
|
| Criterion | Target | Actual | Status |
|
|
|-----------|--------|--------|--------|
|
|
| Uptime | 100% | $(if kill -0 $BOT_PID 2>/dev/null; then echo "100%"; else echo "< 100%"; fi) | $(if kill -0 $BOT_PID 2>/dev/null; then echo "✓ PASS"; else echo "✗ FAIL"; fi) |
|
|
| Cache Hit Rate | 75-85% | $(echo "$METRICS" | grep -oP '"hit_rate_percent":\s*\K[0-9.]+' || echo "N/A")% | $(if [ "$(echo "$METRICS" | grep -oP '"hit_rate_percent":\s*\K[0-9.]+' || echo "0")" -ge 75 ]; then echo "✓ PASS"; else echo "⚠ CHECK"; fi) |
|
|
| No Crashes | 0 | TBD | TBD |
|
|
| Error Rate | < 5% | TBD | TBD |
|
|
|
|
---
|
|
|
|
## Recommendations
|
|
|
|
1. **Cache Performance:** $(if [ "$(echo "$METRICS" | grep -oP '"hit_rate_percent":\s*\K[0-9.]+' || echo "0")" -ge 75 ]; then echo "Cache is performing within target range"; else echo "Consider tuning cache TTL and invalidation logic"; fi)
|
|
2. **Opportunities:** Review profitable opportunities and analyze why others were rejected
|
|
3. **Errors:** Address top errors before production deployment
|
|
4. **System Resources:** Monitor CPU/memory usage trends for capacity planning
|
|
|
|
---
|
|
|
|
## Next Steps
|
|
|
|
- [ ] Review this report with the team
|
|
- [ ] Address any identified issues
|
|
- [ ] Run additional 24h test if needed
|
|
- [ ] Proceed to limited production deployment
|
|
|
|
---
|
|
|
|
**Generated:** $(date)
|
|
REPORT_EOF
|
|
|
|
echo -e "${GREEN}✓ Report generated: $REPORT_FILE${NC}"
|
|
}
|
|
|
|
# Display real-time stats every 5 minutes
|
|
while kill -0 $BOT_PID 2>/dev/null; do
|
|
sleep 300 # 5 minutes
|
|
|
|
if [ -f "$METRICS_FILE" ]; then
|
|
echo -e "${BLUE}[$(date '+%H:%M:%S')] Status Update:${NC}"
|
|
echo -e " Opportunities: $(grep -oP '"opportunities_detected":\s*\K[0-9]+' "$METRICS_FILE" || echo "0")"
|
|
echo -e " Profitable: $(grep -oP '"profitable_opportunities":\s*\K[0-9]+' "$METRICS_FILE" || echo "0")"
|
|
echo -e " Events: $(grep -oP '"events_processed":\s*\K[0-9]+' "$METRICS_FILE" || echo "0")"
|
|
echo -e " Cache Hit Rate: $(grep -oP '"hit_rate_percent":\s*\K[0-9.]+' "$METRICS_FILE" || echo "0")%"
|
|
echo -e " CPU: $(grep -oP '"cpu_percent":\s*\K[0-9.]+' "$METRICS_FILE" || echo "0")%"
|
|
echo -e " Memory: $(grep -oP '"memory_percent":\s*\K[0-9.]+' "$METRICS_FILE" || echo "0")%"
|
|
echo ""
|
|
fi
|
|
done
|
|
|
|
#############################################################################
|
|
# Test Complete
|
|
#############################################################################
|
|
|
|
# Stop monitoring
|
|
kill $MONITOR_PID 2>/dev/null || true
|
|
|
|
# Generate final report
|
|
generate_report
|
|
|
|
echo ""
|
|
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
|
|
echo -e "${GREEN}✓ 24-Hour Validation Test Complete${NC}"
|
|
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
|
|
echo ""
|
|
echo -e "${GREEN}Test End Time:${NC} $(date)"
|
|
echo -e "${GREEN}Report Location:${NC} $REPORT_FILE"
|
|
echo -e "${GREEN}Logs Location:${NC} ${LOG_DIR}"
|
|
echo ""
|
|
echo -e "${YELLOW}[7/7] Next Steps:${NC}"
|
|
echo -e " 1. Review the validation report: cat $REPORT_FILE"
|
|
echo -e " 2. Analyze logs for errors: grep ERROR ${LOG_DIR}/mev_bot.log"
|
|
echo -e " 3. Check for profitable opportunities: grep 'Net Profit' ${LOG_DIR}/mev_bot.log"
|
|
echo -e " 4. Verify cache performance meets target (75-85% hit rate)"
|
|
echo ""
|
|
echo -e "${GREEN}Test completed successfully!${NC}"
|