saving in place

2025-10-04 09:31:02 -05:00
parent 76c1b5cee1
commit f358f49aa9
295 changed files with 72071 additions and 17209 deletions
--- a/cmd/mev-bot/main.go
+++ b/cmd/mev-bot/main.go
@@ -11,12 +11,13 @@ import (
 	"syscall"
 	"time"

-	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/fraktal/mev-beta/internal/config"
 	"github.com/fraktal/mev-beta/internal/logger"
 	"github.com/fraktal/mev-beta/pkg/arbitrage"
 	"github.com/fraktal/mev-beta/pkg/metrics"
 	"github.com/fraktal/mev-beta/pkg/security"
+	"github.com/fraktal/mev-beta/pkg/transport"
+	"github.com/joho/godotenv"
 	"github.com/urfave/cli/v2"
 )

@@ -49,6 +50,24 @@ func main() {
 }

 func startBot() error {
+	// Load environment variables from .env files (in order of precedence)
+	envFiles := []string{
+		".env.production", // Highest priority
+		".env.staging",
+		".env",
+		".env.local", // Lowest priority
+	}
+
+	for _, envFile := range envFiles {
+		if _, err := os.Stat(envFile); err == nil {
+			if err := godotenv.Load(envFile); err != nil {
+				fmt.Printf("Warning: Failed to load %s: %v\n", envFile, err)
+			} else {
+				fmt.Printf("Loaded environment variables from %s\n", envFile)
+			}
+		}
+	}
+
 	// Load configuration
 	configFile := "config/config.yaml"
 	if _, err := os.Stat("config/local.yaml"); err == nil {
@@ -65,11 +84,35 @@ func startBot() error {

 	// Initialize logger
 	log := logger.New(cfg.Log.Level, cfg.Log.Format, cfg.Log.File)
-	log.Info(fmt.Sprintf("Starting MEV bot with Arbitrage Service - Config: %s", configFile))
+	log.Info(fmt.Sprintf("Starting MEV bot with Enhanced Security - Config: %s", configFile))
 	log.Debug(fmt.Sprintf("RPC Endpoint: %s", cfg.Arbitrum.RPCEndpoint))
 	log.Debug(fmt.Sprintf("WS Endpoint: %s", cfg.Arbitrum.WSEndpoint))
 	log.Debug(fmt.Sprintf("Chain ID: %d", cfg.Arbitrum.ChainID))

+	// Initialize comprehensive security framework
+	securityConfig := &security.SecurityConfig{
+		KeyStoreDir:       "keystore",
+		EncryptionEnabled: true,
+		TransactionRPS:    100,
+		RPCRPS:            200,
+		MaxBurstSize:      50,
+		FailureThreshold:  5,
+		RecoveryTimeout:   5 * time.Minute,
+		TLSMinVersion:     771, // TLS 1.2
+		EmergencyStopFile: "emergency.stop",
+		MaxGasPrice:       "50000000000", // 50 gwei
+		AlertWebhookURL:   os.Getenv("SECURITY_WEBHOOK_URL"),
+		LogLevel:          cfg.Log.Level,
+	}
+
+	securityManager, err := security.NewSecurityManager(securityConfig)
+	if err != nil {
+		return fmt.Errorf("failed to initialize security manager: %w", err)
+	}
+	defer securityManager.Shutdown(context.Background())
+
+	log.Info("Security framework initialized successfully")
+
 	// Initialize metrics collector
 	metricsCollector := metrics.NewMetricsCollector(log)

@@ -89,16 +132,32 @@ func startBot() error {
 		log.Info(fmt.Sprintf("Metrics server started on port %s", metricsPort))
 	}

-	// Validate and create Ethereum client
-	if err := validateRPCEndpoint(cfg.Arbitrum.RPCEndpoint); err != nil {
-		return fmt.Errorf("invalid RPC endpoint: %w", err)
+	// Initialize unified provider manager
+	log.Info("Initializing provider manager with separate read-only, execution, and testing pools...")
+
+	// Create temporary provider config file from main config and environment variables
+	tempProviderConfigPath := "config/providers_runtime.yaml"
+	if err := cfg.CreateProviderConfigFile(tempProviderConfigPath); err != nil {
+		return fmt.Errorf("failed to create provider config: %w", err)
 	}

-	client, err := ethclient.Dial(cfg.Arbitrum.RPCEndpoint)
+	defer os.Remove(tempProviderConfigPath) // Clean up temp file
+
+	providerManager, err := transport.NewUnifiedProviderManager(tempProviderConfigPath)
 	if err != nil {
-		return fmt.Errorf("failed to connect to Ethereum client: %w", err)
+		return fmt.Errorf("failed to initialize provider manager: %w", err)
 	}
-	defer client.Close()
+	defer providerManager.Close()
+
+	// Get execution client for transaction operations
+	executionClient, err := providerManager.GetExecutionHTTPClient()
+	if err != nil {
+		return fmt.Errorf("failed to get execution client: %w", err)
+	}
+
+	// Log provider statistics
+	providerStats := providerManager.GetAllStats()
+	log.Info(fmt.Sprintf("Provider manager initialized with %d pool(s)", len(providerStats)-1)) // -1 for summary

 	// Create key manager for secure transaction signing
 	encryptionKey := os.Getenv("MEV_BOT_ENCRYPTION_KEY")
@@ -133,10 +192,15 @@ func startBot() error {
 		return fmt.Errorf("arbitrage service disabled - enable in config to run")
 	}

-	// Create arbitrage service
+	// Setup graceful shutdown BEFORE creating services
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel() // Ensure context is canceled on function exit
+
+	// Create arbitrage service with context
 	log.Info("Creating arbitrage service...")
 	arbitrageService, err := arbitrage.NewArbitrageService(
-		client,
+		ctx,
+		executionClient,
 		log,
 		&cfg.Arbitrage,
 		keyManager,
@@ -147,27 +211,44 @@ func startBot() error {
 	}
 	log.Info("Arbitrage service created successfully")

-	// Start the arbitrage service
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+	// Handle signals in a goroutine to cancel context immediately
+	go func() {
+		<-sigChan
+		log.Info("Shutdown signal received, canceling context...")
+		cancel() // This will cancel the context and stop all operations
+	}()
+
+	// Start the arbitrage service with context
 	log.Info("Starting arbitrage service...")
-	if err := arbitrageService.Start(); err != nil {
-		return fmt.Errorf("failed to start arbitrage service: %w", err)
-	}
+	errChan := make(chan error, 1)
+	go func() {
+		if err := arbitrageService.Start(); err != nil {
+			errChan <- fmt.Errorf("arbitrage service error: %w", err)
+		}
+	}()
 	defer arbitrageService.Stop()
 	log.Info("Arbitrage service started successfully")

 	log.Info("MEV bot started successfully - monitoring for arbitrage opportunities...")
+	log.Info("Press Ctrl+C to stop the bot gracefully...")

-	// Setup graceful shutdown
-	sigChan := make(chan os.Signal, 1)
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
-
-	// Wait for shutdown signal
-	<-sigChan
-	log.Info("Shutdown signal received, stopping MEV bot...")
+	// Wait for context cancellation or error
+	select {
+	case <-ctx.Done():
+		log.Info("Context canceled, stopping MEV bot...")
+	case err := <-errChan:
+		log.Error("Service error occurred: ", err)
+		return err
+	}

 	// Stop metrics server if running
 	if metricsServer != nil {
-		metricsServer.Stop()
+		if err := metricsServer.Stop(); err != nil {
+			log.Error("Failed to stop metrics server gracefully", "error", err)
+		}
 	}

 	// Get final stats
@@ -256,16 +337,18 @@ func scanOpportunities() error {
 	log := logger.New(cfg.Log.Level, cfg.Log.Format, cfg.Log.File)
 	log.Info("Starting one-time arbitrage opportunity scan...")

-	// Validate and create Ethereum client
-	if err := validateRPCEndpoint(cfg.Arbitrum.RPCEndpoint); err != nil {
-		return fmt.Errorf("invalid RPC endpoint: %w", err)
-	}
-
-	client, err := ethclient.Dial(cfg.Arbitrum.RPCEndpoint)
+	// Initialize provider manager for scanning
+	providerManager, err := transport.NewUnifiedProviderManager("config/providers.yaml")
 	if err != nil {
-		return fmt.Errorf("failed to connect to Ethereum client: %w", err)
+		return fmt.Errorf("failed to initialize provider manager: %w", err)
+	}
+	defer providerManager.Close()
+
+	// Get read-only client for scanning (more efficient)
+	client, err := providerManager.GetReadOnlyHTTPClient()
+	if err != nil {
+		return fmt.Errorf("failed to get read-only client: %w", err)
 	}
-	defer client.Close()

 	// Create key manager (not used for scanning but needed for service)
 	encryptionKey := os.Getenv("MEV_BOT_ENCRYPTION_KEY")
@@ -299,6 +382,7 @@ func scanOpportunities() error {
 	scanConfig.MaxConcurrentExecutions = 0 // Disable execution for scan mode

 	arbitrageService, err := arbitrage.NewArbitrageService(
+		context.Background(),
 		client,
 		log,
 		&scanConfig,