Add enhanced concurrency patterns, rate limiting, market management, and pipeline processing

internal/config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"fmt"
 	"os"
+	"strconv"
 
 	"gopkg.in/yaml.v3"
 )
@@ -18,36 +19,94 @@ type Config struct {
 
 // ArbitrumConfig represents the Arbitrum node configuration
 type ArbitrumConfig struct {
+	// Primary RPC endpoint
 	RPCEndpoint string `yaml:"rpc_endpoint"`
-	WSEndpoint  string `yaml:"ws_endpoint"`
-	ChainID     int64  `yaml:"chain_id"`
+	// WebSocket endpoint for Arbitrum node (optional)
+	WSEndpoint string `yaml:"ws_endpoint"`
+	// Chain ID for Arbitrum (42161 for mainnet)
+	ChainID int64 `yaml:"chain_id"`
+	// Rate limiting configuration for RPC endpoint
+	RateLimit RateLimitConfig `yaml:"rate_limit"`
+	// Fallback RPC endpoints
+	FallbackEndpoints []EndpointConfig `yaml:"fallback_endpoints"`
+}
+
+// EndpointConfig represents a fallback RPC endpoint configuration
+type EndpointConfig struct {
+	// RPC endpoint URL
+	URL string `yaml:"url"`
+	// Rate limiting configuration for this endpoint
+	RateLimit RateLimitConfig `yaml:"rate_limit"`
+}
+
+// RateLimitConfig represents rate limiting configuration
+type RateLimitConfig struct {
+	// Maximum requests per second
+	RequestsPerSecond int `yaml:"requests_per_second"`
+	// Maximum concurrent requests
+	MaxConcurrent int `yaml:"max_concurrent"`
+	// Burst size for rate limiting
+	Burst int `yaml:"burst"`
 }
 
 // BotConfig represents the bot configuration
 type BotConfig struct {
-	Enabled             bool    `yaml:"enabled"`
-	PollingInterval     int     `yaml:"polling_interval"`
-	MinProfitThreshold  float64 `yaml:"min_profit_threshold"`
-	GasPriceMultiplier  float64 `yaml:"gas_price_multiplier"`
+	// Enable or disable the bot
+	Enabled bool `yaml:"enabled"`
+	// Polling interval in seconds
+	PollingInterval int `yaml:"polling_interval"`
+	// Minimum profit threshold in USD
+	MinProfitThreshold float64 `yaml:"min_profit_threshold"`
+	// Gas price multiplier (for faster transactions)
+	GasPriceMultiplier float64 `yaml:"gas_price_multiplier"`
+	// Maximum number of concurrent workers for processing
+	MaxWorkers int `yaml:"max_workers"`
+	// Buffer size for channels
+	ChannelBufferSize int `yaml:"channel_buffer_size"`
+	// Timeout for RPC calls in seconds
+	RPCTimeout int `yaml:"rpc_timeout"`
 }
 
 // UniswapConfig represents the Uniswap configuration
 type UniswapConfig struct {
-	FactoryAddress         string  `yaml:"factory_address"`
-	PositionManagerAddress string  `yaml:"position_manager_address"`
-	FeeTiers               []int64 `yaml:"fee_tiers"`
+	// Factory contract address
+	FactoryAddress string `yaml:"factory_address"`
+	// Position manager contract address
+	PositionManagerAddress string `yaml:"position_manager_address"`
+	// Supported fee tiers
+	FeeTiers []int64 `yaml:"fee_tiers"`
+	// Cache configuration for pool data
+	Cache CacheConfig `yaml:"cache"`
+}
+
+// CacheConfig represents caching configuration
+type CacheConfig struct {
+	// Enable or disable caching
+	Enabled bool `yaml:"enabled"`
+	// Cache expiration time in seconds
+	Expiration int `yaml:"expiration"`
+	// Maximum cache size
+	MaxSize int `yaml:"max_size"`
 }
 
 // LogConfig represents the logging configuration
 type LogConfig struct {
-	Level  string `yaml:"level"`
+	// Log level (debug, info, warn, error)
+	Level string `yaml:"level"`
+	// Log format (json, text)
 	Format string `yaml:"format"`
-	File   string `yaml:"file"`
+	// Log file path (empty for stdout)
+	File string `yaml:"file"`
 }
 
 // DatabaseConfig represents the database configuration
 type DatabaseConfig struct {
+	// Database file path
 	File string `yaml:"file"`
+	// Maximum number of open connections
+	MaxOpenConnections int `yaml:"max_open_connections"`
+	// Maximum number of idle connections
+	MaxIdleConnections int `yaml:"max_idle_connections"`
 }
 
 // Load loads the configuration from a file
@@ -64,5 +123,47 @@ func Load(filename string) (*Config, error) {
 		return nil, fmt.Errorf("failed to parse config file: %w", err)
 	}
 
+	// Override with environment variables if they exist
+	config.OverrideWithEnv()
+
 	return &config, nil
+}
+
+// OverrideWithEnv overrides configuration with environment variables
+func (c *Config) OverrideWithEnv() {
+	// Override RPC endpoint
+	if rpcEndpoint := os.Getenv("ARBITRUM_RPC_ENDPOINT"); rpcEndpoint != "" {
+		c.Arbitrum.RPCEndpoint = rpcEndpoint
+	}
+
+	// Override WebSocket endpoint
+	if wsEndpoint := os.Getenv("ARBITRUM_WS_ENDPOINT"); wsEndpoint != "" {
+		c.Arbitrum.WSEndpoint = wsEndpoint
+	}
+
+	// Override rate limit settings
+	if rps := os.Getenv("RPC_REQUESTS_PER_SECOND"); rps != "" {
+		if val, err := strconv.Atoi(rps); err == nil {
+			c.Arbitrum.RateLimit.RequestsPerSecond = val
+		}
+	}
+
+	if maxConcurrent := os.Getenv("RPC_MAX_CONCURRENT"); maxConcurrent != "" {
+		if val, err := strconv.Atoi(maxConcurrent); err == nil {
+			c.Arbitrum.RateLimit.MaxConcurrent = val
+		}
+	}
+
+	// Override bot settings
+	if maxWorkers := os.Getenv("BOT_MAX_WORKERS"); maxWorkers != "" {
+		if val, err := strconv.Atoi(maxWorkers); err == nil {
+			c.Bot.MaxWorkers = val
+		}
+	}
+
+	if channelBufferSize := os.Getenv("BOT_CHANNEL_BUFFER_SIZE"); channelBufferSize != "" {
+		if val, err := strconv.Atoi(channelBufferSize); err == nil {
+			c.Bot.ChannelBufferSize = val
+		}
+	}
 }