feat(core): implement core MEV bot functionality with market scanning and Uniswap V3 pricing

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
Krypto Kajun
2025-09-14 10:16:29 -05:00
parent 5db7587923
commit c16182d80c
1364 changed files with 473970 additions and 1202 deletions

View File

@@ -6,19 +6,19 @@ import (
"sync"
"time"
"github.com/ethereum/go-ethereum/core/types"
"github.com/fraktal/mev-beta/internal/config"
"github.com/fraktal/mev-beta/internal/logger"
"github.com/fraktal/mev-beta/internal/ratelimit"
"github.com/ethereum/go-ethereum/core/types"
)
// FanManager manages fan-in/fan-out patterns for multiple data sources
type FanManager struct {
config *config.Config
logger *logger.Logger
rateLimiter *ratelimit.LimiterManager
bufferSize int
maxWorkers int
config *config.Config
logger *logger.Logger
rateLimiter *ratelimit.LimiterManager
bufferSize int
maxWorkers int
}
// NewFanManager creates a new fan manager
@@ -36,10 +36,10 @@ func NewFanManager(cfg *config.Config, logger *logger.Logger, rateLimiter *ratel
func (fm *FanManager) FanOut(ctx context.Context, jobs <-chan *types.Transaction, numWorkers int) <-chan *types.Transaction {
// Create the output channel
out := make(chan *types.Transaction, fm.bufferSize)
// Create a wait group to wait for all workers
var wg sync.WaitGroup
// Start the workers
for i := 0; i < numWorkers; i++ {
wg.Add(1)
@@ -48,13 +48,13 @@ func (fm *FanManager) FanOut(ctx context.Context, jobs <-chan *types.Transaction
fm.worker(ctx, jobs, out, workerID)
}(i)
}
// Close the output channel when all workers are done
go func() {
wg.Wait()
close(out)
}()
return out
}
@@ -66,21 +66,21 @@ func (fm *FanManager) worker(ctx context.Context, jobs <-chan *types.Transaction
if !ok {
return // Channel closed
}
// Process the job (in this case, just pass it through)
// In practice, you would do some processing here
fm.logger.Debug(fmt.Sprintf("Worker %d processing transaction %s", workerID, job.Hash().Hex()))
// Simulate some work
time.Sleep(10 * time.Millisecond)
// Send the result to the output channel
select {
case out <- job:
case <-ctx.Done():
return
}
case <-ctx.Done():
return
}
@@ -91,10 +91,10 @@ func (fm *FanManager) worker(ctx context.Context, jobs <-chan *types.Transaction
func (fm *FanManager) FanIn(ctx context.Context, inputs ...<-chan *types.Transaction) <-chan *types.Transaction {
// Create the output channel
out := make(chan *types.Transaction, fm.bufferSize)
// Create a wait group to wait for all input channels
var wg sync.WaitGroup
// Start a goroutine for each input channel
for i, input := range inputs {
wg.Add(1)
@@ -103,13 +103,13 @@ func (fm *FanManager) FanIn(ctx context.Context, inputs ...<-chan *types.Transac
fm.fanInWorker(ctx, inputChan, out, inputID)
}(i, input)
}
// Close the output channel when all input channels are done
go func() {
wg.Wait()
close(out)
}()
return out
}
@@ -121,14 +121,14 @@ func (fm *FanManager) fanInWorker(ctx context.Context, input <-chan *types.Trans
if !ok {
return // Channel closed
}
// Send the job to the output channel
select {
case out <- job:
case <-ctx.Done():
return
}
case <-ctx.Done():
return
}
@@ -139,43 +139,43 @@ func (fm *FanManager) fanInWorker(ctx context.Context, input <-chan *types.Trans
func (fm *FanManager) Multiplex(ctx context.Context, transactions <-chan *types.Transaction) []<-chan *types.Transaction {
endpoints := fm.rateLimiter.GetEndpoints()
outputs := make([]<-chan *types.Transaction, len(endpoints))
// Create a channel for each endpoint
for i, endpoint := range endpoints {
// Create a buffered channel for this endpoint
endpointChan := make(chan *types.Transaction, fm.bufferSize)
outputs[i] = endpointChan
// Start a worker for this endpoint
go func(endpointURL string, outChan chan<- *types.Transaction) {
defer close(outChan)
for {
select {
case tx, ok := <-transactions:
if !ok {
return // Input channel closed
}
// Wait for rate limiter
if err := fm.rateLimiter.WaitForLimit(ctx, endpointURL); err != nil {
fm.logger.Error(fmt.Sprintf("Rate limiter error for %s: %v", endpointURL, err))
continue
}
// Send to endpoint-specific channel
select {
case outChan <- tx:
case <-ctx.Done():
return
}
case <-ctx.Done():
return
}
}
}(endpoint, endpointChan)
}
return outputs
}
}

View File

@@ -6,9 +6,9 @@ import (
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/fraktal/mev-beta/internal/config"
"github.com/fraktal/mev-beta/internal/logger"
"github.com/ethereum/go-ethereum/common"
"github.com/holiman/uint256"
"golang.org/x/sync/singleflight"
)
@@ -26,15 +26,15 @@ type MarketManager struct {
// PoolData represents data for a Uniswap V3 pool
type PoolData struct {
Address common.Address
Token0 common.Address
Token1 common.Address
Fee int64
Liquidity *uint256.Int
SqrtPriceX96 *uint256.Int
Tick int
TickSpacing int
LastUpdated time.Time
Address common.Address
Token0 common.Address
Token1 common.Address
Fee int64
Liquidity *uint256.Int
SqrtPriceX96 *uint256.Int
Tick int
TickSpacing int
LastUpdated time.Time
}
// NewMarketManager creates a new market manager
@@ -52,7 +52,7 @@ func NewMarketManager(cfg *config.UniswapConfig, logger *logger.Logger) *MarketM
func (mm *MarketManager) GetPool(ctx context.Context, poolAddress common.Address) (*PoolData, error) {
// Check if we have it in cache and it's still valid
poolKey := poolAddress.Hex()
mm.mu.RLock()
if pool, exists := mm.pools[poolKey]; exists {
// Check if cache is still valid
@@ -67,13 +67,13 @@ func (mm *MarketManager) GetPool(ctx context.Context, poolAddress common.Address
result, err, _ := mm.cacheGroup.Do(poolKey, func() (interface{}, error) {
return mm.fetchPoolData(ctx, poolAddress)
})
if err != nil {
return nil, err
}
pool := result.(*PoolData)
// Update cache
mm.mu.Lock()
// Check if we need to evict old entries
@@ -82,7 +82,7 @@ func (mm *MarketManager) GetPool(ctx context.Context, poolAddress common.Address
}
mm.pools[poolKey] = pool
mm.mu.Unlock()
return pool, nil
}
@@ -90,22 +90,22 @@ func (mm *MarketManager) GetPool(ctx context.Context, poolAddress common.Address
func (mm *MarketManager) fetchPoolData(ctx context.Context, poolAddress common.Address) (*PoolData, error) {
// This is a simplified implementation
// In practice, you would interact with the Ethereum blockchain to get real data
// For now, we'll return mock data
pool := &PoolData{
Address: poolAddress,
Token0: common.HexToAddress("0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"), // USDC
Token1: common.HexToAddress("0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"), // WETH
Fee: 3000, // 0.3%
Liquidity: uint256.NewInt(1000000000000000000), // 1 ETH equivalent
SqrtPriceX96: uint256.NewInt(2505414483750470000), // Mock sqrt price
Tick: 200000, // Mock tick
TickSpacing: 60, // Tick spacing for 0.3% fee
Fee: 3000, // 0.3%
Liquidity: uint256.NewInt(1000000000000000000), // 1 ETH equivalent
SqrtPriceX96: uint256.NewInt(2505414483750470000), // Mock sqrt price
Tick: 200000, // Mock tick
TickSpacing: 60, // Tick spacing for 0.3% fee
LastUpdated: time.Now(),
}
mm.logger.Debug(fmt.Sprintf("Fetched pool data for %s", poolAddress.Hex()))
return pool, nil
}
@@ -113,14 +113,14 @@ func (mm *MarketManager) fetchPoolData(ctx context.Context, poolAddress common.A
func (mm *MarketManager) evictOldest() {
oldestKey := ""
var oldestTime time.Time
for key, pool := range mm.pools {
if oldestKey == "" || pool.LastUpdated.Before(oldestTime) {
oldestKey = key
oldestTime = pool.LastUpdated
}
}
if oldestKey != "" {
delete(mm.pools, oldestKey)
mm.logger.Debug(fmt.Sprintf("Evicted pool %s from cache", oldestKey))
@@ -130,10 +130,10 @@ func (mm *MarketManager) evictOldest() {
// UpdatePool updates pool data
func (mm *MarketManager) UpdatePool(poolAddress common.Address, liquidity *uint256.Int, sqrtPriceX96 *uint256.Int, tick int) {
poolKey := poolAddress.Hex()
mm.mu.Lock()
defer mm.mu.Unlock()
if pool, exists := mm.pools[poolKey]; exists {
pool.Liquidity = liquidity
pool.SqrtPriceX96 = sqrtPriceX96
@@ -156,17 +156,17 @@ func (mm *MarketManager) UpdatePool(poolAddress common.Address, liquidity *uint2
func (mm *MarketManager) GetPoolsByTokens(token0, token1 common.Address) []*PoolData {
mm.mu.RLock()
defer mm.mu.RUnlock()
pools := make([]*PoolData, 0)
for _, pool := range mm.pools {
// Check if this pool contains the token pair
if (pool.Token0 == token0 && pool.Token1 == token1) ||
(pool.Token0 == token1 && pool.Token1 == token0) {
if (pool.Token0 == token0 && pool.Token1 == token1) ||
(pool.Token0 == token1 && pool.Token1 == token0) {
pools = append(pools, pool)
}
}
return pools
}
@@ -174,12 +174,12 @@ func (mm *MarketManager) GetPoolsByTokens(token0, token1 common.Address) []*Pool
func (mm *MarketManager) GetAllPools() []*PoolData {
mm.mu.RLock()
defer mm.mu.RUnlock()
pools := make([]*PoolData, 0, len(mm.pools))
for _, pool := range mm.pools {
pools = append(pools, pool)
}
return pools
}
@@ -187,7 +187,7 @@ func (mm *MarketManager) GetAllPools() []*PoolData {
func (mm *MarketManager) ClearCache() {
mm.mu.Lock()
defer mm.mu.Unlock()
mm.pools = make(map[string]*PoolData)
mm.logger.Info("Cleared pool cache")
}
@@ -196,6 +196,6 @@ func (mm *MarketManager) ClearCache() {
func (mm *MarketManager) GetCacheStats() (int, int) {
mm.mu.RLock()
defer mm.mu.RUnlock()
return len(mm.pools), mm.maxCacheSize
}
}

View File

@@ -5,9 +5,9 @@ import (
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/fraktal/mev-beta/internal/config"
"github.com/fraktal/mev-beta/internal/logger"
"github.com/ethereum/go-ethereum/common"
"github.com/holiman/uint256"
"github.com/stretchr/testify/assert"
)
@@ -289,4 +289,4 @@ func TestGetCacheStats(t *testing.T) {
// Verify results
assert.Equal(t, 2, currentSize)
assert.Equal(t, 10000, maxSize)
}
}

View File

@@ -6,29 +6,29 @@ import (
"math/big"
"sync"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/fraktal/mev-beta/internal/config"
"github.com/fraktal/mev-beta/internal/logger"
"github.com/fraktal/mev-beta/pkg/events"
"github.com/fraktal/mev-beta/pkg/scanner"
"github.com/fraktal/mev-beta/pkg/uniswap"
"github.com/fraktal/mev-beta/pkg/validation"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/holiman/uint256"
)
// Pipeline processes transactions through multiple stages
type Pipeline struct {
config *config.BotConfig
logger *logger.Logger
marketMgr *MarketManager
scanner *scanner.MarketScanner
stages []PipelineStage
bufferSize int
concurrency int
eventParser *events.EventParser
validator *validation.InputValidator
ethClient *ethclient.Client // Add Ethereum client for fetching receipts
config *config.BotConfig
logger *logger.Logger
marketMgr *MarketManager
scanner *scanner.MarketScanner
stages []PipelineStage
bufferSize int
concurrency int
eventParser *events.EventParser
validator *validation.InputValidator
ethClient *ethclient.Client // Add Ethereum client for fetching receipts
}
// PipelineStage represents a stage in the processing pipeline
@@ -53,10 +53,10 @@ func NewPipeline(
validator: validation.NewInputValidator(),
ethClient: ethClient, // Store the Ethereum client
}
// Add default stages
pipeline.AddStage(TransactionDecoderStage(cfg, logger, marketMgr, pipeline.validator, pipeline.ethClient))
return pipeline
}
@@ -80,7 +80,7 @@ func (p *Pipeline) ProcessTransactions(ctx context.Context, transactions []*type
// Parse events from transaction receipts
eventChan := make(chan *events.Event, p.bufferSize)
// Parse transactions in a goroutine
go func() {
defer close(eventChan)
@@ -90,28 +90,28 @@ func (p *Pipeline) ProcessTransactions(ctx context.Context, transactions []*type
p.logger.Warn(fmt.Sprintf("Invalid transaction %s: %v", tx.Hash().Hex(), err))
continue
}
// Fetch transaction receipt
receipt, err := p.ethClient.TransactionReceipt(ctx, tx.Hash())
if err != nil {
p.logger.Error(fmt.Sprintf("Error fetching receipt for transaction %s: %v", tx.Hash().Hex(), err))
continue
}
// Parse events from receipt logs
events, err := p.eventParser.ParseTransactionReceipt(receipt, blockNumber, timestamp)
if err != nil {
p.logger.Error(fmt.Sprintf("Error parsing receipt for transaction %s: %v", tx.Hash().Hex(), err))
continue
}
for _, event := range events {
// Validate the parsed event
if err := p.validator.ValidateEvent(event); err != nil {
p.logger.Warn(fmt.Sprintf("Invalid event from transaction %s: %v", tx.Hash().Hex(), err))
continue
}
select {
case eventChan <- event:
case <-ctx.Done():
@@ -123,21 +123,21 @@ func (p *Pipeline) ProcessTransactions(ctx context.Context, transactions []*type
// Process through each stage
var currentChan <-chan *events.Event = eventChan
for i, stage := range p.stages {
// Create output channel for this stage
outputChan := make(chan *events.Event, p.bufferSize)
go func(stage PipelineStage, input <-chan *events.Event, output chan<- *events.Event, stageIndex int) {
err := stage(ctx, input, output)
if err != nil {
p.logger.Error(fmt.Sprintf("Pipeline stage %d error: %v", stageIndex, err))
}
}(stage, currentChan, outputChan, i)
currentChan = outputChan
}
// Process the final output
if currentChan != nil {
go func() {
@@ -149,7 +149,7 @@ func (p *Pipeline) ProcessTransactions(ctx context.Context, transactions []*type
p.processSwapDetails(ctx, currentChan)
}()
}
return nil
}
@@ -161,10 +161,10 @@ func (p *Pipeline) processSwapDetails(ctx context.Context, eventDetails <-chan *
if !ok {
return // Channel closed
}
// Submit to the market scanner for processing
p.scanner.SubmitEvent(*event)
case <-ctx.Done():
return
}
@@ -181,7 +181,7 @@ func TransactionDecoderStage(
) PipelineStage {
return func(ctx context.Context, input <-chan *events.Event, output chan<- *events.Event) error {
var wg sync.WaitGroup
// Process events concurrently
for i := 0; i < cfg.MaxWorkers; i++ {
wg.Add(1)
@@ -193,7 +193,7 @@ func TransactionDecoderStage(
if !ok {
return // Channel closed
}
// Process the event (in this case, it's already decoded)
// In a real implementation, you might do additional processing here
if event != nil {
@@ -202,21 +202,21 @@ func TransactionDecoderStage(
logger.Warn(fmt.Sprintf("Event validation failed in decoder stage: %v", err))
continue
}
select {
case output <- event:
case <-ctx.Done():
return
}
}
case <-ctx.Done():
return
}
}
}()
}
// Wait for all workers to finish, then close the output channel
go func() {
wg.Wait()
@@ -233,7 +233,7 @@ func TransactionDecoderStage(
close(output)
}
}()
return nil
}
}
@@ -247,7 +247,7 @@ func MarketAnalysisStage(
) PipelineStage {
return func(ctx context.Context, input <-chan *events.Event, output chan<- *events.Event) error {
var wg sync.WaitGroup
// Process events concurrently
for i := 0; i < cfg.MaxWorkers; i++ {
wg.Add(1)
@@ -259,13 +259,13 @@ func MarketAnalysisStage(
if !ok {
return // Channel closed
}
// Validate event before processing
if err := validator.ValidateEvent(event); err != nil {
logger.Warn(fmt.Sprintf("Event validation failed in analysis stage: %v", err))
continue
}
// Only process swap events
if event.Type != events.Swap {
// Forward non-swap events without processing
@@ -276,7 +276,7 @@ func MarketAnalysisStage(
}
continue
}
// Get pool data from market manager
poolData, err := marketMgr.GetPool(ctx, event.PoolAddress)
if err != nil {
@@ -289,7 +289,7 @@ func MarketAnalysisStage(
}
continue
}
// Calculate price impact using Uniswap V3 math
priceImpact, err := calculatePriceImpact(event, poolData)
if err != nil {
@@ -302,26 +302,26 @@ func MarketAnalysisStage(
}
continue
}
// Add price impact to the event
// Note: In a real implementation, you might want to create a new struct
// that extends EventDetails with additional fields
logger.Debug(fmt.Sprintf("Price impact for pool %s: %f", event.PoolAddress, priceImpact))
// Forward the processed event
select {
case output <- event:
case <-ctx.Done():
return
}
case <-ctx.Done():
return
}
}
}()
}
// Wait for all workers to finish, then close the output channel
go func() {
wg.Wait()
@@ -338,7 +338,7 @@ func MarketAnalysisStage(
close(output)
}
}()
return nil
}
}
@@ -348,10 +348,10 @@ func calculatePriceImpact(event *events.Event, poolData *PoolData) (float64, err
// Convert event amounts to uint256 for calculations
amount0In := uint256.NewInt(0)
amount0In.SetFromBig(event.Amount0)
amount1In := uint256.NewInt(0)
amount1In.SetFromBig(event.Amount1)
// Determine which token is being swapped in
var amountIn *uint256.Int
if amount0In.Cmp(uint256.NewInt(0)) > 0 {
@@ -359,28 +359,28 @@ func calculatePriceImpact(event *events.Event, poolData *PoolData) (float64, err
} else {
amountIn = amount1In
}
// If no amount is being swapped in, return 0 impact
if amountIn.Cmp(uint256.NewInt(0)) == 0 {
return 0.0, nil
}
// Calculate price impact as a percentage of liquidity
// priceImpact = amountIn / liquidity
liquidity := poolData.Liquidity
// If liquidity is 0, we can't calculate impact
if liquidity.Cmp(uint256.NewInt(0)) == 0 {
return 0.0, nil
}
// Calculate impact
impact := new(uint256.Int).Div(amountIn, liquidity)
// Convert to float64 for percentage
impactFloat := new(big.Float).SetInt(impact.ToBig())
percentage, _ := impactFloat.Float64()
// Convert to percentage (multiply by 100)
return percentage * 100.0, nil
}
@@ -394,7 +394,7 @@ func ArbitrageDetectionStage(
) PipelineStage {
return func(ctx context.Context, input <-chan *events.Event, output chan<- *events.Event) error {
var wg sync.WaitGroup
// Process events concurrently
for i := 0; i < cfg.MaxWorkers; i++ {
wg.Add(1)
@@ -406,13 +406,13 @@ func ArbitrageDetectionStage(
if !ok {
return // Channel closed
}
// Validate event before processing
if err := validator.ValidateEvent(event); err != nil {
logger.Warn(fmt.Sprintf("Event validation failed in arbitrage detection stage: %v", err))
continue
}
// Only process swap events
if event.Type != events.Swap {
// Forward non-swap events without processing
@@ -423,7 +423,7 @@ func ArbitrageDetectionStage(
}
continue
}
// Look for arbitrage opportunities
opportunities, err := findArbitrageOpportunities(ctx, event, marketMgr, logger)
if err != nil {
@@ -436,7 +436,7 @@ func ArbitrageDetectionStage(
}
continue
}
// Log any found opportunities
if len(opportunities) > 0 {
logger.Info(fmt.Sprintf("Found %d arbitrage opportunities for pool %s", len(opportunities), event.PoolAddress))
@@ -444,21 +444,21 @@ func ArbitrageDetectionStage(
logger.Info(fmt.Sprintf("Arbitrage opportunity: %+v", opp))
}
}
// Forward the processed event
select {
case output <- event:
case <-ctx.Done():
return
}
case <-ctx.Done():
return
}
}
}()
}
// Wait for all workers to finish, then close the output channel
go func() {
wg.Wait()
@@ -475,7 +475,7 @@ func ArbitrageDetectionStage(
close(output)
}
}()
return nil
}
}
@@ -483,17 +483,17 @@ func ArbitrageDetectionStage(
// findArbitrageOpportunities looks for arbitrage opportunities based on a swap event
func findArbitrageOpportunities(ctx context.Context, event *events.Event, marketMgr *MarketManager, logger *logger.Logger) ([]scanner.ArbitrageOpportunity, error) {
opportunities := make([]scanner.ArbitrageOpportunity, 0)
// Get all pools for the same token pair
pools := marketMgr.GetPoolsByTokens(event.Token0, event.Token1)
// If we don't have multiple pools, we can't do arbitrage
if len(pools) < 2 {
return opportunities, nil
}
// Get the pool that triggered the event
// Find the pool that triggered the event
var eventPool *PoolData
for _, pool := range pools {
@@ -502,29 +502,29 @@ func findArbitrageOpportunities(ctx context.Context, event *events.Event, market
break
}
}
// If we can't find the event pool, return
if eventPool == nil {
return opportunities, nil
}
// Convert sqrtPriceX96 to price for the event pool
eventPoolPrice := uniswap.SqrtPriceX96ToPrice(eventPool.SqrtPriceX96.ToBig())
// Compare with other pools
for _, pool := range pools {
// Skip the event pool
if pool.Address == event.PoolAddress {
continue
}
// Convert sqrtPriceX96 to price for comparison pool
compPoolPrice := uniswap.SqrtPriceX96ToPrice(pool.SqrtPriceX96.ToBig())
// Calculate potential profit (simplified)
// In practice, this would involve more complex calculations
profit := new(big.Float).Sub(compPoolPrice, eventPoolPrice)
// If there's a price difference, we might have an opportunity
if profit.Cmp(big.NewFloat(0)) > 0 {
opp := scanner.ArbitrageOpportunity{
@@ -532,12 +532,12 @@ func findArbitrageOpportunities(ctx context.Context, event *events.Event, market
Pools: []string{event.PoolAddress.Hex(), pool.Address.Hex()},
Profit: big.NewInt(1000000000000000000), // 1 ETH (mock value)
GasEstimate: big.NewInt(200000000000000000), // 0.2 ETH (mock value)
ROI: 5.0, // 500% (mock value)
ROI: 5.0, // 500% (mock value)
Protocol: event.Protocol,
}
opportunities = append(opportunities, opp)
}
}
return opportunities, nil
}
}

View File

@@ -5,11 +5,11 @@ import (
"math/big"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/fraktal/mev-beta/internal/config"
"github.com/fraktal/mev-beta/internal/logger"
"github.com/fraktal/mev-beta/pkg/events"
scannerpkg "github.com/fraktal/mev-beta/pkg/scanner"
"github.com/ethereum/go-ethereum/common"
"github.com/holiman/uint256"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
@@ -202,4 +202,4 @@ func TestCalculatePriceImpactNoLiquidity(t *testing.T) {
// Verify results
assert.NoError(t, err)
assert.Equal(t, 0.0, impact)
}
}