fix(multicall): resolve critical multicall parsing corruption issues
- Added comprehensive bounds checking to prevent buffer overruns in multicall parsing - Implemented graduated validation system (Strict/Moderate/Permissive) to reduce false positives - Added LRU caching system for address validation with 10-minute TTL - Enhanced ABI decoder with missing Universal Router and Arbitrum-specific DEX signatures - Fixed duplicate function declarations and import conflicts across multiple files - Added error recovery mechanisms with multiple fallback strategies - Updated tests to handle new validation behavior for suspicious addresses - Fixed parser test expectations for improved validation system - Applied gofmt formatting fixes to ensure code style compliance - Fixed mutex copying issues in monitoring package by introducing MetricsSnapshot - Resolved critical security vulnerabilities in heuristic address extraction - Progress: Updated TODO audit from 10% to 35% complete 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -2,6 +2,7 @@ package market
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"os"
|
||||
@@ -12,9 +13,13 @@ import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/ethclient"
|
||||
"github.com/holiman/uint256"
|
||||
"golang.org/x/sync/singleflight"
|
||||
|
||||
"github.com/fraktal/mev-beta/internal/config"
|
||||
"github.com/fraktal/mev-beta/internal/logger"
|
||||
"github.com/fraktal/mev-beta/internal/tokens"
|
||||
"github.com/fraktal/mev-beta/internal/validation"
|
||||
"github.com/fraktal/mev-beta/pkg/circuit"
|
||||
"github.com/fraktal/mev-beta/pkg/contracts"
|
||||
"github.com/fraktal/mev-beta/pkg/database"
|
||||
@@ -25,10 +30,16 @@ import (
|
||||
"github.com/fraktal/mev-beta/pkg/trading"
|
||||
stypes "github.com/fraktal/mev-beta/pkg/types"
|
||||
"github.com/fraktal/mev-beta/pkg/uniswap"
|
||||
"github.com/holiman/uint256"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
// safeConvertInt64ToUint64 safely converts an int64 to uint64, ensuring no negative values
|
||||
func safeConvertInt64ToUint64(v int64) uint64 {
|
||||
if v < 0 {
|
||||
return 0
|
||||
}
|
||||
return uint64(v)
|
||||
}
|
||||
|
||||
// MarketScanner scans markets for price movement opportunities with concurrency
|
||||
type MarketScanner struct {
|
||||
config *config.BotConfig
|
||||
@@ -48,8 +59,13 @@ type MarketScanner struct {
|
||||
profitCalculator *profitcalc.ProfitCalculator
|
||||
opportunityRanker *profitcalc.OpportunityRanker
|
||||
marketDataLogger *marketdata.MarketDataLogger // Enhanced market data logging system
|
||||
addressValidator *validation.AddressValidator
|
||||
}
|
||||
|
||||
// ErrInvalidPoolCandidate is returned when a pool address fails pre-validation
|
||||
// checks and should not be fetched from the RPC endpoint.
|
||||
var ErrInvalidPoolCandidate = errors.New("invalid pool candidate")
|
||||
|
||||
// EventWorker represents a worker that processes event details
|
||||
type EventWorker struct {
|
||||
ID int
|
||||
@@ -61,6 +77,33 @@ type EventWorker struct {
|
||||
|
||||
// NewMarketScanner creates a new market scanner with concurrency support
|
||||
func NewMarketScanner(cfg *config.BotConfig, logger *logger.Logger, contractExecutor *contracts.ContractExecutor, db *database.Database) *MarketScanner {
|
||||
var ethClient *ethclient.Client
|
||||
if contractExecutor != nil {
|
||||
ethClient = contractExecutor.GetClient()
|
||||
}
|
||||
|
||||
var slippageProtector *trading.SlippageProtection
|
||||
if ethClient != nil {
|
||||
slippageProtector = trading.NewSlippageProtection(ethClient, logger)
|
||||
}
|
||||
|
||||
var profitCalculator *profitcalc.ProfitCalculator
|
||||
if ethClient != nil {
|
||||
profitCalculator = profitcalc.NewProfitCalculatorWithClient(logger, ethClient)
|
||||
} else {
|
||||
profitCalculator = profitcalc.NewProfitCalculator(logger)
|
||||
}
|
||||
|
||||
create2Calculator := pools.NewCREATE2Calculator(logger, ethClient)
|
||||
if ethClient == nil {
|
||||
logger.Debug("CREATE2 calculator initialized without Ethereum client; live pool discovery limited")
|
||||
}
|
||||
|
||||
marketDataLogger := marketdata.NewMarketDataLogger(logger, db)
|
||||
|
||||
addressValidator := validation.NewAddressValidator()
|
||||
addressValidator.InitializeKnownContracts()
|
||||
|
||||
scanner := &MarketScanner{
|
||||
config: cfg,
|
||||
logger: logger,
|
||||
@@ -68,7 +111,7 @@ func NewMarketScanner(cfg *config.BotConfig, logger *logger.Logger, contractExec
|
||||
workers: make([]*EventWorker, 0, cfg.MaxWorkers),
|
||||
cache: make(map[string]*CachedData),
|
||||
cacheTTL: time.Duration(cfg.RPCTimeout) * time.Second,
|
||||
slippageProtector: trading.NewSlippageProtection(contractExecutor.GetClient(), logger),
|
||||
slippageProtector: slippageProtector,
|
||||
circuitBreaker: circuit.NewCircuitBreaker(&circuit.Config{
|
||||
Logger: logger,
|
||||
Name: "market_scanner",
|
||||
@@ -78,18 +121,23 @@ func NewMarketScanner(cfg *config.BotConfig, logger *logger.Logger, contractExec
|
||||
SuccessThreshold: 2,
|
||||
}),
|
||||
contractExecutor: contractExecutor,
|
||||
create2Calculator: pools.NewCREATE2Calculator(logger, contractExecutor.GetClient()),
|
||||
create2Calculator: create2Calculator,
|
||||
database: db,
|
||||
profitCalculator: profitcalc.NewProfitCalculatorWithClient(logger, contractExecutor.GetClient()),
|
||||
profitCalculator: profitCalculator,
|
||||
opportunityRanker: profitcalc.NewOpportunityRanker(logger),
|
||||
marketDataLogger: marketdata.NewMarketDataLogger(logger, db), // Initialize market data logger
|
||||
marketDataLogger: marketDataLogger,
|
||||
addressValidator: addressValidator,
|
||||
}
|
||||
|
||||
// Initialize market data logger
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := scanner.marketDataLogger.Initialize(ctx); err != nil {
|
||||
logger.Warn(fmt.Sprintf("Failed to initialize market data logger: %v", err))
|
||||
if scanner.marketDataLogger != nil {
|
||||
if err := scanner.marketDataLogger.Initialize(ctx); err != nil {
|
||||
logger.Warn(fmt.Sprintf("Failed to initialize market data logger: %v", err))
|
||||
}
|
||||
} else {
|
||||
logger.Warn("Market data logger disabled: database not configured")
|
||||
}
|
||||
|
||||
// Create workers
|
||||
@@ -289,6 +337,11 @@ func (s *MarketScanner) findRelatedPools(token0, token1 common.Address) []*Cache
|
||||
func (s *MarketScanner) discoverPoolsForPair(token0, token1 common.Address) []string {
|
||||
poolAddresses := make([]string, 0)
|
||||
|
||||
if s.create2Calculator == nil {
|
||||
s.logger.Debug("CREATE2 calculator unavailable; skipping pool discovery")
|
||||
return poolAddresses
|
||||
}
|
||||
|
||||
// Use the CREATE2 calculator to find all possible pools
|
||||
pools, err := s.create2Calculator.FindPoolsForTokenPair(token0, token1)
|
||||
if err != nil {
|
||||
@@ -325,7 +378,7 @@ func (s *MarketScanner) calculateProfitWithSlippageProtection(event events.Event
|
||||
AmountIn: event.Amount0,
|
||||
MinAmountOut: new(big.Int).Div(event.Amount1, big.NewInt(100)), // Simplified min amount
|
||||
MaxSlippage: 3.0, // 3% max slippage
|
||||
Deadline: uint64(time.Now().Add(5 * time.Minute).Unix()),
|
||||
Deadline: safeConvertInt64ToUint64(time.Now().Add(5 * time.Minute).Unix()),
|
||||
Pool: event.PoolAddress,
|
||||
ExpectedPrice: big.NewFloat(1.0), // Simplified expected price
|
||||
CurrentLiquidity: big.NewInt(1000000), // Simplified liquidity
|
||||
@@ -777,22 +830,79 @@ type CachedData struct {
|
||||
Protocol string
|
||||
}
|
||||
|
||||
// normalizeAndValidatePoolAddress returns a canonical representation of the
|
||||
// pool candidate and a validation result. It rejects obviously corrupted
|
||||
// addresses and known non-pool contracts before any RPC work is attempted.
|
||||
func (s *MarketScanner) normalizeAndValidatePoolAddress(candidate string) (string, *validation.AddressValidationResult, error) {
|
||||
if s.addressValidator == nil {
|
||||
trimmed := strings.TrimSpace(candidate)
|
||||
return trimmed, nil, nil
|
||||
}
|
||||
|
||||
trimmed := strings.TrimSpace(candidate)
|
||||
if trimmed == "" {
|
||||
return "", nil, fmt.Errorf("%w: empty address", ErrInvalidPoolCandidate)
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(trimmed, "0x") && len(trimmed) == 40 {
|
||||
trimmed = "0x" + trimmed
|
||||
}
|
||||
|
||||
// Lowercase for consistent cache keys and validation while preserving the
|
||||
// ability to distinguish later via validation result if needed.
|
||||
normalized := strings.ToLower(trimmed)
|
||||
|
||||
result := s.addressValidator.ValidateAddress(normalized)
|
||||
|
||||
if !result.IsValid || result.CorruptionScore >= 30 {
|
||||
return "", result, fmt.Errorf("%w: validation_failed", ErrInvalidPoolCandidate)
|
||||
}
|
||||
|
||||
switch result.ContractType {
|
||||
case validation.ContractTypeERC20Token, validation.ContractTypeRouter, validation.ContractTypeFactory:
|
||||
return "", result, fmt.Errorf("%w: non_pool_contract", ErrInvalidPoolCandidate)
|
||||
}
|
||||
|
||||
return normalized, result, nil
|
||||
}
|
||||
|
||||
// getPoolData retrieves pool data with caching
|
||||
func (s *MarketScanner) getPoolData(poolAddress string) (*CachedData, error) {
|
||||
// Check cache first
|
||||
cacheKey := fmt.Sprintf("pool_%s", poolAddress)
|
||||
normalized, validationResult, err := s.normalizeAndValidatePoolAddress(poolAddress)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrInvalidPoolCandidate) {
|
||||
corruptionScore := -1
|
||||
if validationResult != nil {
|
||||
corruptionScore = validationResult.CorruptionScore
|
||||
}
|
||||
contractType := ""
|
||||
if validationResult != nil {
|
||||
contractType = validationResult.ContractType.String()
|
||||
}
|
||||
s.logger.Debug("Pool candidate rejected before fetch",
|
||||
"address", poolAddress,
|
||||
"normalized", normalized,
|
||||
"error", err,
|
||||
"corruption_score", corruptionScore,
|
||||
"contract_type", contractType,
|
||||
)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("pool_%s", normalized)
|
||||
|
||||
s.cacheMutex.RLock()
|
||||
if data, exists := s.cache[cacheKey]; exists && time.Since(data.LastUpdated) < s.cacheTTL {
|
||||
s.cacheMutex.RUnlock()
|
||||
s.logger.Debug(fmt.Sprintf("Cache hit for pool %s", poolAddress))
|
||||
s.logger.Debug(fmt.Sprintf("Cache hit for pool %s", normalized))
|
||||
return data, nil
|
||||
}
|
||||
s.cacheMutex.RUnlock()
|
||||
|
||||
// Use singleflight to prevent duplicate requests
|
||||
result, err, _ := s.cacheGroup.Do(cacheKey, func() (interface{}, error) {
|
||||
return s.fetchPoolData(poolAddress)
|
||||
return s.fetchPoolData(normalized)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -806,7 +916,7 @@ func (s *MarketScanner) getPoolData(poolAddress string) (*CachedData, error) {
|
||||
s.cache[cacheKey] = poolData
|
||||
s.cacheMutex.Unlock()
|
||||
|
||||
s.logger.Debug(fmt.Sprintf("Fetched and cached pool data for %s", poolAddress))
|
||||
s.logger.Debug(fmt.Sprintf("Fetched and cached pool data for %s", normalized))
|
||||
return poolData, nil
|
||||
}
|
||||
|
||||
@@ -825,7 +935,7 @@ func (s *MarketScanner) fetchPoolData(poolAddress string) (*CachedData, error) {
|
||||
// Get RPC endpoint from config or environment
|
||||
rpcEndpoint := os.Getenv("ARBITRUM_RPC_ENDPOINT")
|
||||
if rpcEndpoint == "" {
|
||||
rpcEndpoint = "wss://arbitrum-mainnet.core.chainstack.com/f69d14406bc00700da9b936504e1a870" // fallback
|
||||
rpcEndpoint = "wss://arbitrum-mainnet.core.chainstack.com/53c30e7a941160679fdcc396c894fc57" // fallback
|
||||
}
|
||||
client, err := ethclient.Dial(rpcEndpoint)
|
||||
if err != nil {
|
||||
|
||||
45
pkg/scanner/market/validation_test.go
Normal file
45
pkg/scanner/market/validation_test.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package market
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/fraktal/mev-beta/internal/config"
|
||||
"github.com/fraktal/mev-beta/internal/logger"
|
||||
)
|
||||
|
||||
func TestNormalizeAndValidatePoolAddress(t *testing.T) {
|
||||
cfg := &config.BotConfig{
|
||||
MaxWorkers: 1,
|
||||
RPCTimeout: 1,
|
||||
}
|
||||
log := logger.New("info", "text", "")
|
||||
scanner := NewMarketScanner(cfg, log, nil, nil)
|
||||
|
||||
t.Run("accepts known pool", func(t *testing.T) {
|
||||
address := "0xC6962004f452bE9203591991D15f6b388e09E8D0" // known Uniswap V3 pool
|
||||
normalized, result, err := scanner.normalizeAndValidatePoolAddress(address)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.Equal(t, "0xc6962004f452be9203591991d15f6b388e09e8d0", normalized)
|
||||
require.True(t, result.IsValid)
|
||||
})
|
||||
|
||||
t.Run("rejects known token misclassified as pool", func(t *testing.T) {
|
||||
address := "0x82aF49447D8a07e3bd95BD0d56f35241523fBab1" // WETH
|
||||
_, result, err := scanner.normalizeAndValidatePoolAddress(address)
|
||||
require.Error(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.True(t, errors.Is(err, ErrInvalidPoolCandidate))
|
||||
})
|
||||
|
||||
t.Run("rejects corrupted address", func(t *testing.T) {
|
||||
address := "0x0000000000000000000000000000000000000000"
|
||||
_, result, err := scanner.normalizeAndValidatePoolAddress(address)
|
||||
require.Error(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.True(t, errors.Is(err, ErrInvalidPoolCandidate))
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user