copper-tone-tech/mev-beta
1
0
Fork 0
Code Issues Pull Requests 7 Actions Packages Projects Releases Wiki Activity

Sequencer is working (minimal parsing)

Browse Source
This commit is contained in:
Krypto Kajun
2025-09-14 06:21:10 -05:00
parent 7dd5b5b692
commit 518758790a
59 changed files with 10539 additions and 471 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
pkg/market/pipeline.go
View File

@@ -5,15 +5,15 @@ import (
"fmt"
"math/big"
"sync"
"time"
"github.com/fraktal/mev-beta/internal/config"
"github.com/fraktal/mev-beta/internal/logger"
"github.com/fraktal/mev-beta/pkg/events"
"github.com/fraktal/mev-beta/pkg/scanner"
"github.com/fraktal/mev-beta/pkg/uniswap"
"github.com/ethereum/go-ethereum/common"
"github.com/fraktal/mev-beta/pkg/validation"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/holiman/uint256"
)
@@ -27,10 +27,12 @@ type Pipeline struct {
bufferSize int
concurrency int
eventParser *events.EventParser
validator *validation.InputValidator
ethClient *ethclient.Client // Add Ethereum client for fetching receipts
}
// PipelineStage represents a stage in the processing pipeline
type PipelineStage func(context.Context, <-chan *scanner.EventDetails, chan<- *scanner.EventDetails) error
type PipelineStage func(context.Context, <-chan *events.Event, chan<- *events.Event) error
// NewPipeline creates a new transaction processing pipeline
func NewPipeline(
@@ -38,6 +40,7 @@ func NewPipeline(
logger *logger.Logger,
marketMgr *MarketManager,
scanner *scanner.MarketScanner,
ethClient *ethclient.Client, // Add Ethereum client parameter
) *Pipeline {
pipeline := &Pipeline{
config: cfg,
@@ -47,19 +50,21 @@ func NewPipeline(
bufferSize: cfg.ChannelBufferSize,
concurrency: cfg.MaxWorkers,
eventParser: events.NewEventParser(),
validator: validation.NewInputValidator(),
ethClient: ethClient, // Store the Ethereum client
}
// Add default stages
pipeline.AddStage(TransactionDecoderStage(cfg, logger, marketMgr))
pipeline.AddStage(TransactionDecoderStage(cfg, logger, marketMgr, pipeline.validator, pipeline.ethClient))
return pipeline
}
// AddDefaultStages adds the default processing stages to the pipeline
func (p *Pipeline) AddDefaultStages() {
p.AddStage(TransactionDecoderStage(p.config, p.logger, p.marketMgr))
p.AddStage(MarketAnalysisStage(p.config, p.logger, p.marketMgr))
p.AddStage(ArbitrageDetectionStage(p.config, p.logger, p.marketMgr))
p.AddStage(TransactionDecoderStage(p.config, p.logger, p.marketMgr, p.validator, p.ethClient))
p.AddStage(MarketAnalysisStage(p.config, p.logger, p.marketMgr, p.validator))
p.AddStage(ArbitrageDetectionStage(p.config, p.logger, p.marketMgr, p.validator))
}
// AddStage adds a processing stage to the pipeline
@@ -73,25 +78,40 @@ func (p *Pipeline) ProcessTransactions(ctx context.Context, transactions []*type
return fmt.Errorf("no pipeline stages configured")
}
// Parse events from transactions
// Parse events from transaction receipts
eventChan := make(chan *events.Event, p.bufferSize)
// Parse transactions in a goroutine
go func() {
defer close(eventChan)
for _, tx := range transactions {
// Skip transactions that don't interact with DEX contracts
if !p.eventParser.IsDEXInteraction(tx) {
// Validate transaction input
if err := p.validator.ValidateTransaction(tx); err != nil {
p.logger.Warn(fmt.Sprintf("Invalid transaction %s: %v", tx.Hash().Hex(), err))
continue
}
events, err := p.eventParser.ParseTransaction(tx, blockNumber, timestamp)
// Fetch transaction receipt
receipt, err := p.ethClient.TransactionReceipt(ctx, tx.Hash())
if err != nil {
p.logger.Error(fmt.Sprintf("Error parsing transaction %s: %v", tx.Hash().Hex(), err))
p.logger.Error(fmt.Sprintf("Error fetching receipt for transaction %s: %v", tx.Hash().Hex(), err))
continue
}
// Parse events from receipt logs
events, err := p.eventParser.ParseTransactionReceipt(receipt, blockNumber, timestamp)
if err != nil {
p.logger.Error(fmt.Sprintf("Error parsing receipt for transaction %s: %v", tx.Hash().Hex(), err))
continue
}
for _, event := range events {
// Validate the parsed event
if err := p.validator.ValidateEvent(event); err != nil {
p.logger.Warn(fmt.Sprintf("Invalid event from transaction %s: %v", tx.Hash().Hex(), err))
continue
}
select {
case eventChan <- event:
case <-ctx.Done():
@@ -102,76 +122,39 @@ func (p *Pipeline) ProcessTransactions(ctx context.Context, transactions []*type
}()
// Process through each stage
var currentChan <-chan *scanner.EventDetails = nil
var currentChan <-chan *events.Event = eventChan
for i, stage := range p.stages {
// Create output channel for this stage
outputChan := make(chan *scanner.EventDetails, p.bufferSize)
outputChan := make(chan *events.Event, p.bufferSize)
// For the first stage, we process events
if i == 0 {
// Special handling for first stage
go func(stage PipelineStage, input <-chan *events.Event, output chan<- *scanner.EventDetails) {
defer close(output)
// Convert events.Event to scanner.EventDetails
convertedInput := make(chan *scanner.EventDetails, p.bufferSize)
go func() {
defer close(convertedInput)
for event := range input {
eventDetails := &scanner.EventDetails{
Type: event.Type,
Protocol: event.Protocol,
PoolAddress: event.PoolAddress.Hex(),
Token0: event.Token0.Hex(),
Token1: event.Token1.Hex(),
Amount0In: event.Amount0,
Amount0Out: big.NewInt(0),
Amount1In: big.NewInt(0),
Amount1Out: event.Amount1,
SqrtPriceX96: event.SqrtPriceX96,
Liquidity: event.Liquidity,
Tick: event.Tick,
Timestamp: time.Unix(int64(event.Timestamp), 0),
TransactionHash: event.TransactionHash,
}
select {
case convertedInput <- eventDetails:
case <-ctx.Done():
return
}
}
}()
err := stage(ctx, convertedInput, output)
if err != nil {
p.logger.Error(fmt.Sprintf("Pipeline stage %d error: %v", i, err))
}
}(stage, eventChan, outputChan)
} else {
// For subsequent stages
go func(stage PipelineStage, input <-chan *scanner.EventDetails, output chan<- *scanner.EventDetails) {
defer close(output)
err := stage(ctx, input, output)
if err != nil {
p.logger.Error(fmt.Sprintf("Pipeline stage %d error: %v", i, err))
}
}(stage, currentChan, outputChan)
}
go func(stage PipelineStage, input <-chan *events.Event, output chan<- *events.Event, stageIndex int) {
err := stage(ctx, input, output)
if err != nil {
p.logger.Error(fmt.Sprintf("Pipeline stage %d error: %v", stageIndex, err))
}
}(stage, currentChan, outputChan, i)
currentChan = outputChan
}
// Process the final output
if currentChan != nil {
go p.processSwapDetails(ctx, currentChan)
go func() {
defer func() {
if r := recover(); r != nil {
p.logger.Error(fmt.Sprintf("Final output processor panic recovered: %v", r))
}
}()
p.processSwapDetails(ctx, currentChan)
}()
}
return nil
}
// processSwapDetails processes the final output of the pipeline
func (p *Pipeline) processSwapDetails(ctx context.Context, eventDetails <-chan *scanner.EventDetails) {
func (p *Pipeline) processSwapDetails(ctx context.Context, eventDetails <-chan *events.Event) {
for {
select {
case event, ok := <-eventDetails:
@@ -193,8 +176,10 @@ func TransactionDecoderStage(
cfg *config.BotConfig,
logger *logger.Logger,
marketMgr *MarketManager,
validator *validation.InputValidator,
ethClient *ethclient.Client, // Add Ethereum client parameter
) PipelineStage {
return func(ctx context.Context, input <-chan *scanner.EventDetails, output chan<- *scanner.EventDetails) error {
return func(ctx context.Context, input <-chan *events.Event, output chan<- *events.Event) error {
var wg sync.WaitGroup
// Process events concurrently
@@ -212,6 +197,12 @@ func TransactionDecoderStage(
// Process the event (in this case, it's already decoded)
// In a real implementation, you might do additional processing here
if event != nil {
// Additional validation at the stage level
if err := validator.ValidateEvent(event); err != nil {
logger.Warn(fmt.Sprintf("Event validation failed in decoder stage: %v", err))
continue
}
select {
case output <- event:
case <-ctx.Done():
@@ -229,13 +220,18 @@ func TransactionDecoderStage(
// Wait for all workers to finish, then close the output channel
go func() {
wg.Wait()
// Use recover to handle potential panic from closing already closed channel
// Safely close the output channel
defer func() {
if r := recover(); r != nil {
// Channel already closed, that's fine
logger.Debug("Channel already closed in TransactionDecoderStage")
}
}()
close(output)
select {
case <-ctx.Done():
// Context cancelled, don't close channel as it might be used elsewhere
default:
close(output)
}
}()
return nil
@@ -247,8 +243,9 @@ func MarketAnalysisStage(
cfg *config.BotConfig,
logger *logger.Logger,
marketMgr *MarketManager,
validator *validation.InputValidator,
) PipelineStage {
return func(ctx context.Context, input <-chan *scanner.EventDetails, output chan<- *scanner.EventDetails) error {
return func(ctx context.Context, input <-chan *events.Event, output chan<- *events.Event) error {
var wg sync.WaitGroup
// Process events concurrently
@@ -263,6 +260,12 @@ func MarketAnalysisStage(
return // Channel closed
}
// Validate event before processing
if err := validator.ValidateEvent(event); err != nil {
logger.Warn(fmt.Sprintf("Event validation failed in analysis stage: %v", err))
continue
}
// Only process swap events
if event.Type != events.Swap {
// Forward non-swap events without processing
@@ -275,8 +278,7 @@ func MarketAnalysisStage(
}
// Get pool data from market manager
poolAddress := common.HexToAddress(event.PoolAddress)
poolData, err := marketMgr.GetPool(ctx, poolAddress)
poolData, err := marketMgr.GetPool(ctx, event.PoolAddress)
if err != nil {
logger.Error(fmt.Sprintf("Error getting pool data for %s: %v", event.PoolAddress, err))
// Forward the event even if we can't get pool data
@@ -323,13 +325,18 @@ func MarketAnalysisStage(
// Wait for all workers to finish, then close the output channel
go func() {
wg.Wait()
// Use recover to handle potential panic from closing already closed channel
// Safely close the output channel
defer func() {
if r := recover(); r != nil {
// Channel already closed, that's fine
logger.Debug("Channel already closed in MarketAnalysisStage")
}
}()
close(output)
select {
case <-ctx.Done():
// Context cancelled, don't close channel as it might be used elsewhere
default:
close(output)
}
}()
return nil
@@ -337,13 +344,13 @@ func MarketAnalysisStage(
}
// calculatePriceImpact calculates the price impact of a swap using Uniswap V3 math
func calculatePriceImpact(event *scanner.EventDetails, poolData *PoolData) (float64, error) {
func calculatePriceImpact(event *events.Event, poolData *PoolData) (float64, error) {
// Convert event amounts to uint256 for calculations
amount0In := uint256.NewInt(0)
amount0In.SetFromBig(event.Amount0In)
amount0In.SetFromBig(event.Amount0)
amount1In := uint256.NewInt(0)
amount1In.SetFromBig(event.Amount1In)
amount1In.SetFromBig(event.Amount1)
// Determine which token is being swapped in
var amountIn *uint256.Int
@@ -383,8 +390,9 @@ func ArbitrageDetectionStage(
cfg *config.BotConfig,
logger *logger.Logger,
marketMgr *MarketManager,
validator *validation.InputValidator,
) PipelineStage {
return func(ctx context.Context, input <-chan *scanner.EventDetails, output chan<- *scanner.EventDetails) error {
return func(ctx context.Context, input <-chan *events.Event, output chan<- *events.Event) error {
var wg sync.WaitGroup
// Process events concurrently
@@ -399,6 +407,12 @@ func ArbitrageDetectionStage(
return // Channel closed
}
// Validate event before processing
if err := validator.ValidateEvent(event); err != nil {
logger.Warn(fmt.Sprintf("Event validation failed in arbitrage detection stage: %v", err))
continue
}
// Only process swap events
if event.Type != events.Swap {
// Forward non-swap events without processing
@@ -448,13 +462,18 @@ func ArbitrageDetectionStage(
// Wait for all workers to finish, then close the output channel
go func() {
wg.Wait()
// Use recover to handle potential panic from closing already closed channel
// Safely close the output channel
defer func() {
if r := recover(); r != nil {
// Channel already closed, that's fine
logger.Debug("Channel already closed in ArbitrageDetectionStage")
}
}()
close(output)
select {
case <-ctx.Done():
// Context cancelled, don't close channel as it might be used elsewhere
default:
close(output)
}
}()
return nil
@@ -462,13 +481,11 @@ func ArbitrageDetectionStage(
}
// findArbitrageOpportunities looks for arbitrage opportunities based on a swap event
func findArbitrageOpportunities(ctx context.Context, event *scanner.EventDetails, marketMgr *MarketManager, logger *logger.Logger) ([]scanner.ArbitrageOpportunity, error) {
func findArbitrageOpportunities(ctx context.Context, event *events.Event, marketMgr *MarketManager, logger *logger.Logger) ([]scanner.ArbitrageOpportunity, error) {
opportunities := make([]scanner.ArbitrageOpportunity, 0)
// Get all pools for the same token pair
token0 := common.HexToAddress(event.Token0)
token1 := common.HexToAddress(event.Token1)
pools := marketMgr.GetPoolsByTokens(token0, token1)
pools := marketMgr.GetPoolsByTokens(event.Token0, event.Token1)
// If we don't have multiple pools, we can't do arbitrage
if len(pools) < 2 {
@@ -476,12 +493,11 @@ func findArbitrageOpportunities(ctx context.Context, event *scanner.EventDetails
}
// Get the pool that triggered the event
eventPoolAddress := common.HexToAddress(event.PoolAddress)
// Find the pool that triggered the event
var eventPool *PoolData
for _, pool := range pools {
if pool.Address == eventPoolAddress {
if pool.Address == event.PoolAddress {
eventPool = pool
break
}
@@ -498,7 +514,7 @@ func findArbitrageOpportunities(ctx context.Context, event *scanner.EventDetails
// Compare with other pools
for _, pool := range pools {
// Skip the event pool
if pool.Address == eventPoolAddress {
if pool.Address == event.PoolAddress {
continue
}
@@ -512,8 +528,8 @@ func findArbitrageOpportunities(ctx context.Context, event *scanner.EventDetails
// If there's a price difference, we might have an opportunity
if profit.Cmp(big.NewFloat(0)) > 0 {
opp := scanner.ArbitrageOpportunity{
Path: []string{event.Token0, event.Token1},
Pools: []string{event.PoolAddress, pool.Address.Hex()},
Path: []string{event.Token0.Hex(), event.Token1.Hex()},
Pools: []string{event.PoolAddress.Hex(), pool.Address.Hex()},
Profit: big.NewInt(1000000000000000000), // 1 ETH (mock value)
GasEstimate: big.NewInt(200000000000000000), // 0.2 ETH (mock value)
ROI: 5.0, // 500% (mock value)

21
pkg/market/pipeline_test.go
View File

@@ -7,6 +7,7 @@ import (
"github.com/fraktal/mev-beta/internal/config"
"github.com/fraktal/mev-beta/internal/logger"
"github.com/fraktal/mev-beta/pkg/events"
scannerpkg "github.com/fraktal/mev-beta/pkg/scanner"
"github.com/ethereum/go-ethereum/common"
"github.com/holiman/uint256"
@@ -93,7 +94,7 @@ func TestAddStage(t *testing.T) {
pipeline := NewPipeline(cfg, logger, marketMgr, scannerObj)
// Add a new stage
newStage := func(ctx context.Context, input <-chan *scannerpkg.EventDetails, output chan<- *scannerpkg.EventDetails) error {
newStage := func(ctx context.Context, input <-chan *events.Event, output chan<- *events.Event) error {
return nil
}
pipeline.AddStage(newStage)
@@ -142,9 +143,9 @@ func TestTransactionDecoderStage(t *testing.T) {
func TestCalculatePriceImpact(t *testing.T) {
// Create test event
event := &scannerpkg.EventDetails{
Amount0In: big.NewInt(1000000000), // 1000 tokens
Amount1In: big.NewInt(0),
event := &events.Event{
Amount0: big.NewInt(1000000000), // 1000 tokens
Amount1: big.NewInt(0),
}
// Create test pool data
@@ -163,9 +164,9 @@ func TestCalculatePriceImpact(t *testing.T) {
func TestCalculatePriceImpactNoAmount(t *testing.T) {
// Create test event with no amount
event := &scannerpkg.EventDetails{
Amount0In: big.NewInt(0),
Amount1In: big.NewInt(0),
event := &events.Event{
Amount0: big.NewInt(0),
Amount1: big.NewInt(0),
}
// Create test pool data
@@ -184,9 +185,9 @@ func TestCalculatePriceImpactNoAmount(t *testing.T) {
func TestCalculatePriceImpactNoLiquidity(t *testing.T) {
// Create test event
event := &scannerpkg.EventDetails{
Amount0In: big.NewInt(1000000000),
Amount1In: big.NewInt(0),
event := &events.Event{
Amount0: big.NewInt(1000000000),
Amount1: big.NewInt(0),
}
// Create test pool data with zero liquidity