fix(multicall): resolve critical multicall parsing corruption issues

- Added comprehensive bounds checking to prevent buffer overruns in multicall parsing
- Implemented graduated validation system (Strict/Moderate/Permissive) to reduce false positives
- Added LRU caching system for address validation with 10-minute TTL
- Enhanced ABI decoder with missing Universal Router and Arbitrum-specific DEX signatures
- Fixed duplicate function declarations and import conflicts across multiple files
- Added error recovery mechanisms with multiple fallback strategies
- Updated tests to handle new validation behavior for suspicious addresses
- Fixed parser test expectations for improved validation system
- Applied gofmt formatting fixes to ensure code style compliance
- Fixed mutex copying issues in monitoring package by introducing MetricsSnapshot
- Resolved critical security vulnerabilities in heuristic address extraction
- Progress: Updated TODO audit from 10% to 35% complete

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Krypto Kajun
2025-10-17 00:12:55 -05:00
parent f358f49aa9
commit 850223a953
8621 changed files with 79808 additions and 7340 deletions

View File

@@ -2,6 +2,7 @@ package market
import (
"context"
"errors"
"fmt"
"math/big"
"os"
@@ -12,9 +13,13 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/holiman/uint256"
"golang.org/x/sync/singleflight"
"github.com/fraktal/mev-beta/internal/config"
"github.com/fraktal/mev-beta/internal/logger"
"github.com/fraktal/mev-beta/internal/tokens"
"github.com/fraktal/mev-beta/internal/validation"
"github.com/fraktal/mev-beta/pkg/circuit"
"github.com/fraktal/mev-beta/pkg/contracts"
"github.com/fraktal/mev-beta/pkg/database"
@@ -25,10 +30,16 @@ import (
"github.com/fraktal/mev-beta/pkg/trading"
stypes "github.com/fraktal/mev-beta/pkg/types"
"github.com/fraktal/mev-beta/pkg/uniswap"
"github.com/holiman/uint256"
"golang.org/x/sync/singleflight"
)
// safeConvertInt64ToUint64 safely converts an int64 to uint64, ensuring no negative values
func safeConvertInt64ToUint64(v int64) uint64 {
if v < 0 {
return 0
}
return uint64(v)
}
// MarketScanner scans markets for price movement opportunities with concurrency
type MarketScanner struct {
config *config.BotConfig
@@ -48,8 +59,13 @@ type MarketScanner struct {
profitCalculator *profitcalc.ProfitCalculator
opportunityRanker *profitcalc.OpportunityRanker
marketDataLogger *marketdata.MarketDataLogger // Enhanced market data logging system
addressValidator *validation.AddressValidator
}
// ErrInvalidPoolCandidate is returned when a pool address fails pre-validation
// checks and should not be fetched from the RPC endpoint.
var ErrInvalidPoolCandidate = errors.New("invalid pool candidate")
// EventWorker represents a worker that processes event details
type EventWorker struct {
ID int
@@ -61,6 +77,33 @@ type EventWorker struct {
// NewMarketScanner creates a new market scanner with concurrency support
func NewMarketScanner(cfg *config.BotConfig, logger *logger.Logger, contractExecutor *contracts.ContractExecutor, db *database.Database) *MarketScanner {
var ethClient *ethclient.Client
if contractExecutor != nil {
ethClient = contractExecutor.GetClient()
}
var slippageProtector *trading.SlippageProtection
if ethClient != nil {
slippageProtector = trading.NewSlippageProtection(ethClient, logger)
}
var profitCalculator *profitcalc.ProfitCalculator
if ethClient != nil {
profitCalculator = profitcalc.NewProfitCalculatorWithClient(logger, ethClient)
} else {
profitCalculator = profitcalc.NewProfitCalculator(logger)
}
create2Calculator := pools.NewCREATE2Calculator(logger, ethClient)
if ethClient == nil {
logger.Debug("CREATE2 calculator initialized without Ethereum client; live pool discovery limited")
}
marketDataLogger := marketdata.NewMarketDataLogger(logger, db)
addressValidator := validation.NewAddressValidator()
addressValidator.InitializeKnownContracts()
scanner := &MarketScanner{
config: cfg,
logger: logger,
@@ -68,7 +111,7 @@ func NewMarketScanner(cfg *config.BotConfig, logger *logger.Logger, contractExec
workers: make([]*EventWorker, 0, cfg.MaxWorkers),
cache: make(map[string]*CachedData),
cacheTTL: time.Duration(cfg.RPCTimeout) * time.Second,
slippageProtector: trading.NewSlippageProtection(contractExecutor.GetClient(), logger),
slippageProtector: slippageProtector,
circuitBreaker: circuit.NewCircuitBreaker(&circuit.Config{
Logger: logger,
Name: "market_scanner",
@@ -78,18 +121,23 @@ func NewMarketScanner(cfg *config.BotConfig, logger *logger.Logger, contractExec
SuccessThreshold: 2,
}),
contractExecutor: contractExecutor,
create2Calculator: pools.NewCREATE2Calculator(logger, contractExecutor.GetClient()),
create2Calculator: create2Calculator,
database: db,
profitCalculator: profitcalc.NewProfitCalculatorWithClient(logger, contractExecutor.GetClient()),
profitCalculator: profitCalculator,
opportunityRanker: profitcalc.NewOpportunityRanker(logger),
marketDataLogger: marketdata.NewMarketDataLogger(logger, db), // Initialize market data logger
marketDataLogger: marketDataLogger,
addressValidator: addressValidator,
}
// Initialize market data logger
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := scanner.marketDataLogger.Initialize(ctx); err != nil {
logger.Warn(fmt.Sprintf("Failed to initialize market data logger: %v", err))
if scanner.marketDataLogger != nil {
if err := scanner.marketDataLogger.Initialize(ctx); err != nil {
logger.Warn(fmt.Sprintf("Failed to initialize market data logger: %v", err))
}
} else {
logger.Warn("Market data logger disabled: database not configured")
}
// Create workers
@@ -289,6 +337,11 @@ func (s *MarketScanner) findRelatedPools(token0, token1 common.Address) []*Cache
func (s *MarketScanner) discoverPoolsForPair(token0, token1 common.Address) []string {
poolAddresses := make([]string, 0)
if s.create2Calculator == nil {
s.logger.Debug("CREATE2 calculator unavailable; skipping pool discovery")
return poolAddresses
}
// Use the CREATE2 calculator to find all possible pools
pools, err := s.create2Calculator.FindPoolsForTokenPair(token0, token1)
if err != nil {
@@ -325,7 +378,7 @@ func (s *MarketScanner) calculateProfitWithSlippageProtection(event events.Event
AmountIn: event.Amount0,
MinAmountOut: new(big.Int).Div(event.Amount1, big.NewInt(100)), // Simplified min amount
MaxSlippage: 3.0, // 3% max slippage
Deadline: uint64(time.Now().Add(5 * time.Minute).Unix()),
Deadline: safeConvertInt64ToUint64(time.Now().Add(5 * time.Minute).Unix()),
Pool: event.PoolAddress,
ExpectedPrice: big.NewFloat(1.0), // Simplified expected price
CurrentLiquidity: big.NewInt(1000000), // Simplified liquidity
@@ -777,22 +830,79 @@ type CachedData struct {
Protocol string
}
// normalizeAndValidatePoolAddress returns a canonical representation of the
// pool candidate and a validation result. It rejects obviously corrupted
// addresses and known non-pool contracts before any RPC work is attempted.
func (s *MarketScanner) normalizeAndValidatePoolAddress(candidate string) (string, *validation.AddressValidationResult, error) {
if s.addressValidator == nil {
trimmed := strings.TrimSpace(candidate)
return trimmed, nil, nil
}
trimmed := strings.TrimSpace(candidate)
if trimmed == "" {
return "", nil, fmt.Errorf("%w: empty address", ErrInvalidPoolCandidate)
}
if !strings.HasPrefix(trimmed, "0x") && len(trimmed) == 40 {
trimmed = "0x" + trimmed
}
// Lowercase for consistent cache keys and validation while preserving the
// ability to distinguish later via validation result if needed.
normalized := strings.ToLower(trimmed)
result := s.addressValidator.ValidateAddress(normalized)
if !result.IsValid || result.CorruptionScore >= 30 {
return "", result, fmt.Errorf("%w: validation_failed", ErrInvalidPoolCandidate)
}
switch result.ContractType {
case validation.ContractTypeERC20Token, validation.ContractTypeRouter, validation.ContractTypeFactory:
return "", result, fmt.Errorf("%w: non_pool_contract", ErrInvalidPoolCandidate)
}
return normalized, result, nil
}
// getPoolData retrieves pool data with caching
func (s *MarketScanner) getPoolData(poolAddress string) (*CachedData, error) {
// Check cache first
cacheKey := fmt.Sprintf("pool_%s", poolAddress)
normalized, validationResult, err := s.normalizeAndValidatePoolAddress(poolAddress)
if err != nil {
if errors.Is(err, ErrInvalidPoolCandidate) {
corruptionScore := -1
if validationResult != nil {
corruptionScore = validationResult.CorruptionScore
}
contractType := ""
if validationResult != nil {
contractType = validationResult.ContractType.String()
}
s.logger.Debug("Pool candidate rejected before fetch",
"address", poolAddress,
"normalized", normalized,
"error", err,
"corruption_score", corruptionScore,
"contract_type", contractType,
)
}
return nil, err
}
cacheKey := fmt.Sprintf("pool_%s", normalized)
s.cacheMutex.RLock()
if data, exists := s.cache[cacheKey]; exists && time.Since(data.LastUpdated) < s.cacheTTL {
s.cacheMutex.RUnlock()
s.logger.Debug(fmt.Sprintf("Cache hit for pool %s", poolAddress))
s.logger.Debug(fmt.Sprintf("Cache hit for pool %s", normalized))
return data, nil
}
s.cacheMutex.RUnlock()
// Use singleflight to prevent duplicate requests
result, err, _ := s.cacheGroup.Do(cacheKey, func() (interface{}, error) {
return s.fetchPoolData(poolAddress)
return s.fetchPoolData(normalized)
})
if err != nil {
@@ -806,7 +916,7 @@ func (s *MarketScanner) getPoolData(poolAddress string) (*CachedData, error) {
s.cache[cacheKey] = poolData
s.cacheMutex.Unlock()
s.logger.Debug(fmt.Sprintf("Fetched and cached pool data for %s", poolAddress))
s.logger.Debug(fmt.Sprintf("Fetched and cached pool data for %s", normalized))
return poolData, nil
}
@@ -825,7 +935,7 @@ func (s *MarketScanner) fetchPoolData(poolAddress string) (*CachedData, error) {
// Get RPC endpoint from config or environment
rpcEndpoint := os.Getenv("ARBITRUM_RPC_ENDPOINT")
if rpcEndpoint == "" {
rpcEndpoint = "wss://arbitrum-mainnet.core.chainstack.com/f69d14406bc00700da9b936504e1a870" // fallback
rpcEndpoint = "wss://arbitrum-mainnet.core.chainstack.com/53c30e7a941160679fdcc396c894fc57" // fallback
}
client, err := ethclient.Dial(rpcEndpoint)
if err != nil {

View File

@@ -0,0 +1,45 @@
package market
import (
"errors"
"testing"
"github.com/stretchr/testify/require"
"github.com/fraktal/mev-beta/internal/config"
"github.com/fraktal/mev-beta/internal/logger"
)
func TestNormalizeAndValidatePoolAddress(t *testing.T) {
cfg := &config.BotConfig{
MaxWorkers: 1,
RPCTimeout: 1,
}
log := logger.New("info", "text", "")
scanner := NewMarketScanner(cfg, log, nil, nil)
t.Run("accepts known pool", func(t *testing.T) {
address := "0xC6962004f452bE9203591991D15f6b388e09E8D0" // known Uniswap V3 pool
normalized, result, err := scanner.normalizeAndValidatePoolAddress(address)
require.NoError(t, err)
require.NotNil(t, result)
require.Equal(t, "0xc6962004f452be9203591991d15f6b388e09e8d0", normalized)
require.True(t, result.IsValid)
})
t.Run("rejects known token misclassified as pool", func(t *testing.T) {
address := "0x82aF49447D8a07e3bd95BD0d56f35241523fBab1" // WETH
_, result, err := scanner.normalizeAndValidatePoolAddress(address)
require.Error(t, err)
require.NotNil(t, result)
require.True(t, errors.Is(err, ErrInvalidPoolCandidate))
})
t.Run("rejects corrupted address", func(t *testing.T) {
address := "0x0000000000000000000000000000000000000000"
_, result, err := scanner.normalizeAndValidatePoolAddress(address)
require.Error(t, err)
require.NotNil(t, result)
require.True(t, errors.Is(err, ErrInvalidPoolCandidate))
})
}