mev-beta/test/sequencer_simulation.go

package test

import (
	"context"
	"encoding/hex"
	"fmt"
	"math/big"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/ethclient"
	"github.com/ethereum/go-ethereum/rpc"
	"github.com/fraktal/mev-beta/internal/logger"
	"github.com/fraktal/mev-beta/pkg/arbitrum"
	"github.com/fraktal/mev-beta/pkg/oracle"
)

// ArbitrumSequencerSimulator simulates the Arbitrum sequencer for testing
type ArbitrumSequencerSimulator struct {
	config            *SequencerConfig
	logger            *logger.Logger
	realDataCollector *RealDataCollector
	mockService       *MockSequencerService
	validator         *ParserValidator
	storage           *TransactionStorage
	benchmark         *PerformanceBenchmark
	mu                sync.RWMutex
	isRunning         bool
}

// SequencerConfig contains configuration for the sequencer simulator
type SequencerConfig struct {
	// Data collection
	RPCEndpoint string `json:"rpc_endpoint"`
	WSEndpoint  string `json:"ws_endpoint"`
	DataDir     string `json:"data_dir"`

	// Block range for data collection
	StartBlock        uint64 `json:"start_block"`
	EndBlock          uint64 `json:"end_block"`
	MaxBlocksPerBatch int    `json:"max_blocks_per_batch"`

	// Filtering criteria
	MinSwapValueUSD float64  `json:"min_swap_value_usd"`
	TargetProtocols []string `json:"target_protocols"`

	// Simulation parameters
	SequencerTiming  time.Duration `json:"sequencer_timing"`
	BatchSize        int           `json:"batch_size"`
	CompressionLevel int           `json:"compression_level"`

	// Performance testing
	MaxConcurrentOps int           `json:"max_concurrent_ops"`
	TestDuration     time.Duration `json:"test_duration"`

	// Validation
	ValidateResults  bool    `json:"validate_results"`
	StrictMode       bool    `json:"strict_mode"`
	ExpectedAccuracy float64 `json:"expected_accuracy"`
}

// RealTransactionData represents real Arbitrum transaction data
type RealTransactionData struct {
	Hash               common.Hash     `json:"hash"`
	BlockNumber        uint64          `json:"block_number"`
	BlockHash          common.Hash     `json:"block_hash"`
	TransactionIndex   uint            `json:"transaction_index"`
	From               common.Address  `json:"from"`
	To                 *common.Address `json:"to"`
	Value              *big.Int        `json:"value"`
	GasLimit           uint64          `json:"gas_limit"`
	GasUsed            uint64          `json:"gas_used"`
	GasPrice           *big.Int        `json:"gas_price"`
	GasTipCap          *big.Int        `json:"gas_tip_cap,omitempty"`
	GasFeeCap          *big.Int        `json:"gas_fee_cap,omitempty"`
	Data               []byte          `json:"data"`
	Logs               []*types.Log    `json:"logs"`
	Status             uint64          `json:"status"`
	SequencerTimestamp time.Time       `json:"sequencer_timestamp"`
	L1BlockNumber      uint64          `json:"l1_block_number,omitempty"`

	// L2-specific fields
	ArbTxType      int      `json:"arb_tx_type,omitempty"`
	RequestId      *big.Int `json:"request_id,omitempty"`
	SequenceNumber uint64   `json:"sequence_number,omitempty"`

	// Parsed information
	ParsedDEX         *arbitrum.DEXTransaction `json:"parsed_dex,omitempty"`
	SwapDetails       *SequencerSwapDetails    `json:"swap_details,omitempty"`
	MEVClassification string                   `json:"mev_classification,omitempty"`
	EstimatedValueUSD float64                  `json:"estimated_value_usd,omitempty"`
}

// SequencerSwapDetails contains detailed swap information from sequencer perspective
type SequencerSwapDetails struct {
	Protocol    string   `json:"protocol"`
	TokenIn     string   `json:"token_in"`
	TokenOut    string   `json:"token_out"`
	AmountIn    *big.Int `json:"amount_in"`
	AmountOut   *big.Int `json:"amount_out"`
	AmountMin   *big.Int `json:"amount_min"`
	Fee         uint32   `json:"fee,omitempty"`
	Slippage    float64  `json:"slippage"`
	PriceImpact float64  `json:"price_impact"`
	PoolAddress string   `json:"pool_address,omitempty"`
	Recipient   string   `json:"recipient"`
	Deadline    uint64   `json:"deadline"`

	// Sequencer-specific metrics
	SequencerLatency time.Duration `json:"sequencer_latency"`
	BatchPosition    int           `json:"batch_position"`
	CompressionRatio float64       `json:"compression_ratio"`
}

// RealDataCollector fetches and processes real Arbitrum transaction data
type RealDataCollector struct {
	config    *SequencerConfig
	logger    *logger.Logger
	ethClient *ethclient.Client
	rpcClient *rpc.Client
	l2Parser  *arbitrum.ArbitrumL2Parser
	oracle    *oracle.PriceOracle
	storage   *TransactionStorage
}

// TransactionStorage manages storage and indexing of transaction data
type TransactionStorage struct {
	config    *SequencerConfig
	logger    *logger.Logger
	dataDir   string
	indexFile string
	mu        sync.RWMutex
	index     map[string]*TransactionIndex
}

// TransactionIndex provides fast lookup for stored transactions
type TransactionIndex struct {
	Hash            string                 `json:"hash"`
	BlockNumber     uint64                 `json:"block_number"`
	Protocol        string                 `json:"protocol"`
	ValueUSD        float64                `json:"value_usd"`
	MEVType         string                 `json:"mev_type"`
	FilePath        string                 `json:"file_path"`
	StoredAt        time.Time              `json:"stored_at"`
	DataSize        int64                  `json:"data_size"`
	CompressionMeta map[string]interface{} `json:"compression_meta,omitempty"`
}

// NewArbitrumSequencerSimulator creates a new sequencer simulator
func NewArbitrumSequencerSimulator(config *SequencerConfig, logger *logger.Logger) (*ArbitrumSequencerSimulator, error) {
	if config == nil {
		config = DefaultSequencerConfig()
	}

	// Create data directory
	if err := os.MkdirAll(config.DataDir, 0755); err != nil {
		return nil, fmt.Errorf("failed to create data directory: %w", err)
	}

	// Initialize storage
	storage, err := NewTransactionStorage(config, logger)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize storage: %w", err)
	}

	// Initialize real data collector
	collector, err := NewRealDataCollector(config, logger, storage)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize data collector: %w", err)
	}

	// Initialize mock sequencer service
	mockService := NewMockSequencerService(config, logger, storage)

	// Initialize parser validator
	validator := NewParserValidator(config, logger, storage)

	// Initialize performance benchmark
	benchmark := NewPerformanceBenchmark(config, logger)

	return &ArbitrumSequencerSimulator{
		config:            config,
		logger:            logger,
		realDataCollector: collector,
		mockService:       mockService,
		validator:         validator,
		storage:           storage,
		benchmark:         benchmark,
	}, nil
}

// DefaultSequencerConfig returns default configuration
func DefaultSequencerConfig() *SequencerConfig {
	return &SequencerConfig{
		RPCEndpoint:       "https://arb1.arbitrum.io/rpc",
		WSEndpoint:        "wss://arb1.arbitrum.io/ws",
		DataDir:           "./test_data/sequencer_simulation",
		StartBlock:        0, // Will be set to recent block
		EndBlock:          0, // Will be set to latest
		MaxBlocksPerBatch: 10,
		MinSwapValueUSD:   1000.0, // Only collect swaps > $1k
		TargetProtocols:   []string{"UniswapV2", "UniswapV3", "SushiSwap", "Camelot", "TraderJoe", "1Inch"},
		SequencerTiming:   250 * time.Millisecond, // ~4 blocks per second
		BatchSize:         100,
		CompressionLevel:  6,
		MaxConcurrentOps:  10,
		TestDuration:      5 * time.Minute,
		ValidateResults:   true,
		StrictMode:        false,
		ExpectedAccuracy:  0.95, // 95% accuracy required
	}
}

// NewRealDataCollector creates a new real data collector
func NewRealDataCollector(config *SequencerConfig, logger *logger.Logger, storage *TransactionStorage) (*RealDataCollector, error) {
	// Connect to Arbitrum
	ethClient, err := ethclient.Dial(config.RPCEndpoint)
	if err != nil {
		return nil, fmt.Errorf("failed to connect to Arbitrum: %w", err)
	}

	rpcClient, err := rpc.Dial(config.RPCEndpoint)
	if err != nil {
		ethClient.Close()
		return nil, fmt.Errorf("failed to connect to Arbitrum RPC: %w", err)
	}

	// Create price oracle
	oracle := oracle.NewPriceOracle(ethClient, logger)

	// Create L2 parser
	l2Parser, err := arbitrum.NewArbitrumL2Parser(config.RPCEndpoint, logger, oracle)
	if err != nil {
		ethClient.Close()
		rpcClient.Close()
		return nil, fmt.Errorf("failed to create L2 parser: %w", err)
	}

	return &RealDataCollector{
		config:    config,
		logger:    logger,
		ethClient: ethClient,
		rpcClient: rpcClient,
		l2Parser:  l2Parser,
		oracle:    oracle,
		storage:   storage,
	}, nil
}

// CollectRealData fetches real transaction data from Arbitrum
func (rdc *RealDataCollector) CollectRealData(ctx context.Context) error {
	rdc.logger.Info("Starting real data collection from Arbitrum...")

	// Get latest block if end block is not set
	if rdc.config.EndBlock == 0 {
		header, err := rdc.ethClient.HeaderByNumber(ctx, nil)
		if err != nil {
			return fmt.Errorf("failed to get latest block: %w", err)
		}
		rdc.config.EndBlock = header.Number.Uint64()
	}

	// Set start block if not set (collect recent data)
	if rdc.config.StartBlock == 0 {
		rdc.config.StartBlock = rdc.config.EndBlock - 1000 // Last 1000 blocks
	}

	rdc.logger.Info(fmt.Sprintf("Collecting data from blocks %d to %d", rdc.config.StartBlock, rdc.config.EndBlock))

	// Process blocks in batches
	for blockNum := rdc.config.StartBlock; blockNum <= rdc.config.EndBlock; blockNum += uint64(rdc.config.MaxBlocksPerBatch) {
		endBlock := blockNum + uint64(rdc.config.MaxBlocksPerBatch) - 1
		if endBlock > rdc.config.EndBlock {
			endBlock = rdc.config.EndBlock
		}

		if err := rdc.processBlockBatch(ctx, blockNum, endBlock); err != nil {
			rdc.logger.Error(fmt.Sprintf("Failed to process block batch %d-%d: %v", blockNum, endBlock, err))
			continue
		}

		// Check context cancellation
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}
	}

	return nil
}

// processBlockBatch processes a batch of blocks
func (rdc *RealDataCollector) processBlockBatch(ctx context.Context, startBlock, endBlock uint64) error {
	rdc.logger.Debug(fmt.Sprintf("Processing block batch %d-%d", startBlock, endBlock))

	var collectedTxs []*RealTransactionData

	for blockNum := startBlock; blockNum <= endBlock; blockNum++ {
		block, err := rdc.ethClient.BlockByNumber(ctx, big.NewInt(int64(blockNum)))
		if err != nil {
			rdc.logger.Warn(fmt.Sprintf("Failed to get block %d: %v", blockNum, err))
			continue
		}

		// Get block receipts for logs
		receipts, err := rdc.getBlockReceipts(ctx, block.Hash())
		if err != nil {
			rdc.logger.Warn(fmt.Sprintf("Failed to get receipts for block %d: %v", blockNum, err))
			continue
		}

		// Process transactions
		blockTxs := rdc.processBlockTransactions(ctx, block, receipts)
		collectedTxs = append(collectedTxs, blockTxs...)

		rdc.logger.Debug(fmt.Sprintf("Block %d: collected %d DEX transactions", blockNum, len(blockTxs)))
	}

	// Store collected transactions
	if len(collectedTxs) > 0 {
		if err := rdc.storage.StoreBatch(collectedTxs); err != nil {
			return fmt.Errorf("failed to store transaction batch: %w", err)
		}
		rdc.logger.Info(fmt.Sprintf("Stored %d transactions from blocks %d-%d", len(collectedTxs), startBlock, endBlock))
	}

	return nil
}

// processBlockTransactions processes transactions in a block
func (rdc *RealDataCollector) processBlockTransactions(ctx context.Context, block *types.Block, receipts []*types.Receipt) []*RealTransactionData {
	var dexTxs []*RealTransactionData

	for i, tx := range block.Transactions() {
		// Skip non-DEX transactions quickly
		if !rdc.isLikelyDEXTransaction(tx) {
			continue
		}

		// Get receipt
		var receipt *types.Receipt
		if i < len(receipts) {
			receipt = receipts[i]
		} else {
			var err error
			receipt, err = rdc.ethClient.TransactionReceipt(ctx, tx.Hash())
			if err != nil {
				continue
			}
		}

		// Skip failed transactions
		if receipt.Status != 1 {
			continue
		}

		// Parse DEX transaction
		dexTx := rdc.parseRealTransaction(ctx, block, tx, receipt)
		if dexTx != nil && rdc.meetsCriteria(dexTx) {
			dexTxs = append(dexTxs, dexTx)
		}
	}

	return dexTxs
}

// isLikelyDEXTransaction performs quick filtering for DEX transactions
func (rdc *RealDataCollector) isLikelyDEXTransaction(tx *types.Transaction) bool {
	// Must have recipient
	if tx.To() == nil {
		return false
	}

	// Must have data
	if len(tx.Data()) < 4 {
		return false
	}

	// Check function signature for known DEX functions
	funcSig := hex.EncodeToString(tx.Data()[:4])
	dexFunctions := map[string]bool{
		"38ed1739": true, // swapExactTokensForTokens
		"8803dbee": true, // swapTokensForExactTokens
		"7ff36ab5": true, // swapExactETHForTokens
		"414bf389": true, // exactInputSingle
		"c04b8d59": true, // exactInput
		"db3e2198": true, // exactOutputSingle
		"ac9650d8": true, // multicall
		"5ae401dc": true, // multicall with deadline
		"7c025200": true, // 1inch swap
		"3593564c": true, // universal router execute
	}

	return dexFunctions[funcSig]
}

// parseRealTransaction parses a real transaction into structured data
func (rdc *RealDataCollector) parseRealTransaction(ctx context.Context, block *types.Block, tx *types.Transaction, receipt *types.Receipt) *RealTransactionData {
	// Create base transaction data
	realTx := &RealTransactionData{
		Hash:               tx.Hash(),
		BlockNumber:        block.NumberU64(),
		BlockHash:          block.Hash(),
		TransactionIndex:   receipt.TransactionIndex,
		From:               receipt.From,
		To:                 tx.To(),
		Value:              tx.Value(),
		GasLimit:           tx.Gas(),
		GasUsed:            receipt.GasUsed,
		GasPrice:           tx.GasPrice(),
		Data:               tx.Data(),
		Logs:               receipt.Logs,
		Status:             receipt.Status,
		SequencerTimestamp: time.Unix(int64(block.Time()), 0),
	}

	// Add L2-specific fields for dynamic fee transactions
	if tx.Type() == types.DynamicFeeTxType {
		realTx.GasTipCap = tx.GasTipCap()
		realTx.GasFeeCap = tx.GasFeeCap()
	}

	// Parse using L2 parser to get DEX details
	rawTx := convertToRawL2Transaction(tx, receipt)
	if parsedDEX := rdc.l2Parser.parseDEXTransaction(rawTx); parsedDEX != nil {
		realTx.ParsedDEX = parsedDEX

		// Extract detailed swap information
		if parsedDEX.SwapDetails != nil && parsedDEX.SwapDetails.IsValid {
			realTx.SwapDetails = &SequencerSwapDetails{
				Protocol:      parsedDEX.Protocol,
				TokenIn:       parsedDEX.SwapDetails.TokenIn,
				TokenOut:      parsedDEX.SwapDetails.TokenOut,
				AmountIn:      parsedDEX.SwapDetails.AmountIn,
				AmountOut:     parsedDEX.SwapDetails.AmountOut,
				AmountMin:     parsedDEX.SwapDetails.AmountMin,
				Fee:           parsedDEX.SwapDetails.Fee,
				Recipient:     parsedDEX.SwapDetails.Recipient,
				Deadline:      parsedDEX.SwapDetails.Deadline,
				BatchPosition: int(receipt.TransactionIndex),
			}

			// Calculate price impact and slippage if possible
			if realTx.SwapDetails.AmountIn != nil && realTx.SwapDetails.AmountOut != nil &&
				realTx.SwapDetails.AmountIn.Sign() > 0 && realTx.SwapDetails.AmountOut.Sign() > 0 {
				// Simplified slippage calculation
				if realTx.SwapDetails.AmountMin != nil && realTx.SwapDetails.AmountMin.Sign() > 0 {
					minFloat := new(big.Float).SetInt(realTx.SwapDetails.AmountMin)
					outFloat := new(big.Float).SetInt(realTx.SwapDetails.AmountOut)
					if minFloat.Sign() > 0 {
						slippage := new(big.Float).Quo(
							new(big.Float).Sub(outFloat, minFloat),
							outFloat,
						)
						slippageFloat, _ := slippage.Float64()
						realTx.SwapDetails.Slippage = slippageFloat * 100 // Convert to percentage
					}
				}
			}
		}

		// Classify MEV type based on transaction characteristics
		realTx.MEVClassification = rdc.classifyMEVTransaction(realTx)

		// Estimate USD value (simplified)
		realTx.EstimatedValueUSD = rdc.estimateTransactionValueUSD(realTx)
	}

	return realTx
}

// convertToRawL2Transaction converts standard transaction to raw L2 format
func convertToRawL2Transaction(tx *types.Transaction, receipt *types.Receipt) arbitrum.RawL2Transaction {
	var to string
	if tx.To() != nil {
		to = tx.To().Hex()
	}

	return arbitrum.RawL2Transaction{
		Hash:             tx.Hash().Hex(),
		From:             receipt.From.Hex(),
		To:               to,
		Value:            fmt.Sprintf("0x%x", tx.Value()),
		Gas:              fmt.Sprintf("0x%x", tx.Gas()),
		GasPrice:         fmt.Sprintf("0x%x", tx.GasPrice()),
		Input:            hex.EncodeToString(tx.Data()),
		Nonce:            fmt.Sprintf("0x%x", tx.Nonce()),
		TransactionIndex: fmt.Sprintf("0x%x", receipt.TransactionIndex),
		Type:             fmt.Sprintf("0x%x", tx.Type()),
	}
}

// getBlockReceipts fetches all receipts for a block
func (rdc *RealDataCollector) getBlockReceipts(ctx context.Context, blockHash common.Hash) ([]*types.Receipt, error) {
	var receipts []*types.Receipt
	err := rdc.rpcClient.CallContext(ctx, &receipts, "eth_getBlockReceipts", blockHash.Hex())
	return receipts, err
}

// meetsCriteria checks if transaction meets collection criteria
func (rdc *RealDataCollector) meetsCriteria(tx *RealTransactionData) bool {
	// Must have valid DEX parsing
	if tx.ParsedDEX == nil {
		return false
	}

	// Check protocol filter
	if len(rdc.config.TargetProtocols) > 0 {
		found := false
		for _, protocol := range rdc.config.TargetProtocols {
			if strings.EqualFold(tx.ParsedDEX.Protocol, protocol) {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}

	// Check minimum value
	if rdc.config.MinSwapValueUSD > 0 && tx.EstimatedValueUSD < rdc.config.MinSwapValueUSD {
		return false
	}

	return true
}

// classifyMEVTransaction classifies the MEV type of a transaction
func (rdc *RealDataCollector) classifyMEVTransaction(tx *RealTransactionData) string {
	if tx.ParsedDEX == nil {
		return "unknown"
	}

	// Analyze transaction characteristics
	funcName := strings.ToLower(tx.ParsedDEX.FunctionName)

	// Arbitrage indicators
	if strings.Contains(funcName, "multicall") {
		return "potential_arbitrage"
	}

	// Large swap indicators
	if tx.EstimatedValueUSD > 100000 { // > $100k
		return "large_swap"
	}

	// Sandwich attack indicators (would need more context analysis)
	if tx.SwapDetails != nil && tx.SwapDetails.Slippage > 5.0 { // > 5% slippage
		return "high_slippage"
	}

	// Regular swap
	return "regular_swap"
}

// estimateTransactionValueUSD estimates the USD value of a transaction
func (rdc *RealDataCollector) estimateTransactionValueUSD(tx *RealTransactionData) float64 {
	if tx.SwapDetails == nil || tx.SwapDetails.AmountIn == nil {
		return 0.0
	}

	// Simplified estimation - in practice, use price oracle
	// Convert to ETH equivalent (assuming 18 decimals)
	amountFloat := new(big.Float).Quo(
		new(big.Float).SetInt(tx.SwapDetails.AmountIn),
		big.NewFloat(1e18),
	)

	amountEth, _ := amountFloat.Float64()

	// Rough ETH price estimation (would use real oracle in production)
	ethPriceUSD := 2000.0

	return amountEth * ethPriceUSD
}

// Close closes the data collector connections
func (rdc *RealDataCollector) Close() {
	if rdc.ethClient != nil {
		rdc.ethClient.Close()
	}
	if rdc.rpcClient != nil {
		rdc.rpcClient.Close()
	}
	if rdc.l2Parser != nil {
		rdc.l2Parser.Close()
	}
}