Add enhanced concurrency patterns, rate limiting, market management, and pipeline processing

internal/config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"fmt"
 	"os"
+	"strconv"
 
 	"gopkg.in/yaml.v3"
 )
@@ -18,36 +19,94 @@ type Config struct {
 
 // ArbitrumConfig represents the Arbitrum node configuration
 type ArbitrumConfig struct {
+	// Primary RPC endpoint
 	RPCEndpoint string `yaml:"rpc_endpoint"`
-	WSEndpoint  string `yaml:"ws_endpoint"`
-	ChainID     int64  `yaml:"chain_id"`
+	// WebSocket endpoint for Arbitrum node (optional)
+	WSEndpoint string `yaml:"ws_endpoint"`
+	// Chain ID for Arbitrum (42161 for mainnet)
+	ChainID int64 `yaml:"chain_id"`
+	// Rate limiting configuration for RPC endpoint
+	RateLimit RateLimitConfig `yaml:"rate_limit"`
+	// Fallback RPC endpoints
+	FallbackEndpoints []EndpointConfig `yaml:"fallback_endpoints"`
+}
+
+// EndpointConfig represents a fallback RPC endpoint configuration
+type EndpointConfig struct {
+	// RPC endpoint URL
+	URL string `yaml:"url"`
+	// Rate limiting configuration for this endpoint
+	RateLimit RateLimitConfig `yaml:"rate_limit"`
+}
+
+// RateLimitConfig represents rate limiting configuration
+type RateLimitConfig struct {
+	// Maximum requests per second
+	RequestsPerSecond int `yaml:"requests_per_second"`
+	// Maximum concurrent requests
+	MaxConcurrent int `yaml:"max_concurrent"`
+	// Burst size for rate limiting
+	Burst int `yaml:"burst"`
 }
 
 // BotConfig represents the bot configuration
 type BotConfig struct {
-	Enabled             bool    `yaml:"enabled"`
-	PollingInterval     int     `yaml:"polling_interval"`
-	MinProfitThreshold  float64 `yaml:"min_profit_threshold"`
-	GasPriceMultiplier  float64 `yaml:"gas_price_multiplier"`
+	// Enable or disable the bot
+	Enabled bool `yaml:"enabled"`
+	// Polling interval in seconds
+	PollingInterval int `yaml:"polling_interval"`
+	// Minimum profit threshold in USD
+	MinProfitThreshold float64 `yaml:"min_profit_threshold"`
+	// Gas price multiplier (for faster transactions)
+	GasPriceMultiplier float64 `yaml:"gas_price_multiplier"`
+	// Maximum number of concurrent workers for processing
+	MaxWorkers int `yaml:"max_workers"`
+	// Buffer size for channels
+	ChannelBufferSize int `yaml:"channel_buffer_size"`
+	// Timeout for RPC calls in seconds
+	RPCTimeout int `yaml:"rpc_timeout"`
 }
 
 // UniswapConfig represents the Uniswap configuration
 type UniswapConfig struct {
-	FactoryAddress         string  `yaml:"factory_address"`
-	PositionManagerAddress string  `yaml:"position_manager_address"`
-	FeeTiers               []int64 `yaml:"fee_tiers"`
+	// Factory contract address
+	FactoryAddress string `yaml:"factory_address"`
+	// Position manager contract address
+	PositionManagerAddress string `yaml:"position_manager_address"`
+	// Supported fee tiers
+	FeeTiers []int64 `yaml:"fee_tiers"`
+	// Cache configuration for pool data
+	Cache CacheConfig `yaml:"cache"`
+}
+
+// CacheConfig represents caching configuration
+type CacheConfig struct {
+	// Enable or disable caching
+	Enabled bool `yaml:"enabled"`
+	// Cache expiration time in seconds
+	Expiration int `yaml:"expiration"`
+	// Maximum cache size
+	MaxSize int `yaml:"max_size"`
 }
 
 // LogConfig represents the logging configuration
 type LogConfig struct {
-	Level  string `yaml:"level"`
+	// Log level (debug, info, warn, error)
+	Level string `yaml:"level"`
+	// Log format (json, text)
 	Format string `yaml:"format"`
-	File   string `yaml:"file"`
+	// Log file path (empty for stdout)
+	File string `yaml:"file"`
 }
 
 // DatabaseConfig represents the database configuration
 type DatabaseConfig struct {
+	// Database file path
 	File string `yaml:"file"`
+	// Maximum number of open connections
+	MaxOpenConnections int `yaml:"max_open_connections"`
+	// Maximum number of idle connections
+	MaxIdleConnections int `yaml:"max_idle_connections"`
 }
 
 // Load loads the configuration from a file
@@ -64,5 +123,47 @@ func Load(filename string) (*Config, error) {
 		return nil, fmt.Errorf("failed to parse config file: %w", err)
 	}
 
+	// Override with environment variables if they exist
+	config.OverrideWithEnv()
+
 	return &config, nil
+}
+
+// OverrideWithEnv overrides configuration with environment variables
+func (c *Config) OverrideWithEnv() {
+	// Override RPC endpoint
+	if rpcEndpoint := os.Getenv("ARBITRUM_RPC_ENDPOINT"); rpcEndpoint != "" {
+		c.Arbitrum.RPCEndpoint = rpcEndpoint
+	}
+
+	// Override WebSocket endpoint
+	if wsEndpoint := os.Getenv("ARBITRUM_WS_ENDPOINT"); wsEndpoint != "" {
+		c.Arbitrum.WSEndpoint = wsEndpoint
+	}
+
+	// Override rate limit settings
+	if rps := os.Getenv("RPC_REQUESTS_PER_SECOND"); rps != "" {
+		if val, err := strconv.Atoi(rps); err == nil {
+			c.Arbitrum.RateLimit.RequestsPerSecond = val
+		}
+	}
+
+	if maxConcurrent := os.Getenv("RPC_MAX_CONCURRENT"); maxConcurrent != "" {
+		if val, err := strconv.Atoi(maxConcurrent); err == nil {
+			c.Arbitrum.RateLimit.MaxConcurrent = val
+		}
+	}
+
+	// Override bot settings
+	if maxWorkers := os.Getenv("BOT_MAX_WORKERS"); maxWorkers != "" {
+		if val, err := strconv.Atoi(maxWorkers); err == nil {
+			c.Bot.MaxWorkers = val
+		}
+	}
+
+	if channelBufferSize := os.Getenv("BOT_CHANNEL_BUFFER_SIZE"); channelBufferSize != "" {
+		if val, err := strconv.Atoi(channelBufferSize); err == nil {
+			c.Bot.ChannelBufferSize = val
+		}
+	}
 }

internal/ratelimit/manager.go

@@ -0,0 +1,128 @@
+package ratelimit
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/your-username/mev-beta/internal/config"
+	"golang.org/x/time/rate"
+)
+
+// LimiterManager manages rate limiters for multiple endpoints
+type LimiterManager struct {
+	limiters map[string]*EndpointLimiter
+	mu       sync.RWMutex
+}
+
+// EndpointLimiter represents a rate limiter for a specific endpoint
+type EndpointLimiter struct {
+	URL     string
+	Limiter *rate.Limiter
+	Config  config.RateLimitConfig
+}
+
+// NewLimiterManager creates a new LimiterManager
+func NewLimiterManager(cfg *config.ArbitrumConfig) *LimiterManager {
+	lm := &LimiterManager{
+		limiters: make(map[string]*EndpointLimiter),
+	}
+
+	// Create limiter for primary endpoint
+	limiter := createLimiter(cfg.RateLimit)
+	lm.limiters[cfg.RPCEndpoint] = &EndpointLimiter{
+		URL:     cfg.RPCEndpoint,
+		Limiter: limiter,
+		Config:  cfg.RateLimit,
+	}
+
+	// Create limiters for fallback endpoints
+	for _, endpoint := range cfg.FallbackEndpoints {
+		limiter := createLimiter(endpoint.RateLimit)
+		lm.limiters[endpoint.URL] = &EndpointLimiter{
+			URL:     endpoint.URL,
+			Limiter: limiter,
+			Config:  endpoint.RateLimit,
+		}
+	}
+
+	return lm
+}
+
+// createLimiter creates a rate limiter based on the configuration
+func createLimiter(cfg config.RateLimitConfig) *rate.Limiter {
+	// Create a rate limiter with the specified rate and burst
+	r := rate.Limit(cfg.RequestsPerSecond)
+	return rate.NewLimiter(r, cfg.Burst)
+}
+
+// WaitForLimit waits for the rate limiter to allow a request
+func (lm *LimiterManager) WaitForLimit(ctx context.Context, endpointURL string) error {
+	lm.mu.RLock()
+	limiter, exists := lm.limiters[endpointURL]
+	lm.mu.RUnlock()
+
+	if !exists {
+		return fmt.Errorf("no rate limiter found for endpoint: %s", endpointURL)
+	}
+
+	// Wait for permission to make a request
+	return limiter.Limiter.Wait(ctx)
+}
+
+// TryWaitForLimit tries to wait for the rate limiter to allow a request without blocking
+func (lm *LimiterManager) TryWaitForLimit(ctx context.Context, endpointURL string) error {
+	lm.mu.RLock()
+	limiter, exists := lm.limiters[endpointURL]
+	lm.mu.RUnlock()
+
+	if !exists {
+		return fmt.Errorf("no rate limiter found for endpoint: %s", endpointURL)
+	}
+
+	// Try to wait for permission to make a request without blocking
+	if !limiter.Limiter.Allow() {
+		return fmt.Errorf("rate limit exceeded for endpoint: %s", endpointURL)
+	}
+
+	return nil
+}
+
+// GetLimiter returns the rate limiter for a specific endpoint
+func (lm *LimiterManager) GetLimiter(endpointURL string) (*rate.Limiter, error) {
+	lm.mu.RLock()
+	limiter, exists := lm.limiters[endpointURL]
+	lm.mu.RUnlock()
+
+	if !exists {
+		return nil, fmt.Errorf("no rate limiter found for endpoint: %s", endpointURL)
+	}
+
+	return limiter.Limiter, nil
+}
+
+// UpdateLimiter updates the rate limiter for an endpoint
+func (lm *LimiterManager) UpdateLimiter(endpointURL string, cfg config.RateLimitConfig) {
+	lm.mu.Lock()
+	defer lm.mu.Unlock()
+
+	limiter := createLimiter(cfg)
+	lm.limiters[endpointURL] = &EndpointLimiter{
+		URL:     endpointURL,
+		Limiter: limiter,
+		Config:  cfg,
+	}
+}
+
+// GetEndpoints returns all endpoint URLs
+func (lm *LimiterManager) GetEndpoints() []string {
+	lm.mu.RLock()
+	defer lm.mu.RUnlock()
+
+	endpoints := make([]string, 0, len(lm.limiters))
+	for url := range lm.limiters {
+		endpoints = append(endpoints, url)
+	}
+
+	return endpoints
+}