feat: create v2-prep branch with comprehensive planning

Restructured project for V2 refactor:

**Structure Changes:**
- Moved all V1 code to orig/ folder (preserved with git mv)
- Created docs/planning/ directory
- Added orig/README_V1.md explaining V1 preservation

**Planning Documents:**
- 00_V2_MASTER_PLAN.md: Complete architecture overview
  - Executive summary of critical V1 issues
  - High-level component architecture diagrams
  - 5-phase implementation roadmap
  - Success metrics and risk mitigation

- 07_TASK_BREAKDOWN.md: Atomic task breakdown
  - 99+ hours of detailed tasks
  - Every task < 2 hours (atomic)
  - Clear dependencies and success criteria
  - Organized by implementation phase

**V2 Key Improvements:**
- Per-exchange parsers (factory pattern)
- Multi-layer strict validation
- Multi-index pool cache
- Background validation pipeline
- Comprehensive observability

**Critical Issues Addressed:**
- Zero address tokens (strict validation + cache enrichment)
- Parsing accuracy (protocol-specific parsers)
- No audit trail (background validation channel)
- Inefficient lookups (multi-index cache)
- Stats disconnection (event-driven metrics)

Next Steps:
1. Review planning documents
2. Begin Phase 1: Foundation (P1-001 through P1-010)
3. Implement parsers in Phase 2
4. Build cache system in Phase 3
5. Add validation pipeline in Phase 4
6. Migrate and test in Phase 5

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Administrator
2025-11-10 10:14:26 +01:00
parent 1773daffe7
commit 803de231ba
411 changed files with 20390 additions and 8680 deletions

148
orig/pkg/tokens/decimals.go Normal file
View File

@@ -0,0 +1,148 @@
package tokens
import (
"math"
"math/big"
"github.com/ethereum/go-ethereum/common"
)
// Common token addresses on Arbitrum
var (
WETH = common.HexToAddress("0x82aF49447D8a07e3bd95BD0d56f35241523fBab1")
USDC = common.HexToAddress("0xFF970A61A04b1cA14834A43f5dE4533eBDDB5CC8")
USDT = common.HexToAddress("0xFd086bC7CD5C481DCC9C85ebE478A1C0b69FCbb9")
WBTC = common.HexToAddress("0x2f2a2543B76A4166549F7aaB2e75Bef0aefC5B0f")
ARB = common.HexToAddress("0x912CE59144191C1204E64559FE8253a0e49E6548")
DAI = common.HexToAddress("0xDA10009cBd5D07dd0CeCc66161FC93D7c9000da1")
LINK = common.HexToAddress("0xf97f4df75117a78c1A5a0DBb814Af92458539FB4")
UNI = common.HexToAddress("0xFa7F8980b0f1E64A2062791cc3b0871572f1F7f0")
GMX = common.HexToAddress("0xfc5A1A6EB076a2C7aD06eD22C90d7E710E35ad0a")
MAGIC = common.HexToAddress("0x539bdE0d7Dbd336b79148AA742883198BBF60342")
)
// TokenDecimals maps token addresses to their decimal places
var TokenDecimals = map[common.Address]uint8{
WETH: 18,
USDC: 6,
USDT: 6,
WBTC: 8,
ARB: 18,
DAI: 18,
LINK: 18,
UNI: 18,
GMX: 18,
MAGIC: 18,
}
// GetTokenDecimals returns the decimal places for a token, defaulting to 18 if unknown
func GetTokenDecimals(tokenAddress common.Address) uint8 {
if decimals, ok := TokenDecimals[tokenAddress]; ok {
return decimals
}
// Default to 18 decimals for unknown tokens (most ERC20 tokens use 18)
return 18
}
// ConvertToFloat converts a token amount to a float64 considering its decimals
func ConvertToFloat(amount *big.Int, tokenAddress common.Address) float64 {
if amount == nil {
return 0
}
decimals := GetTokenDecimals(tokenAddress)
divisor := math.Pow(10, float64(decimals))
amountFloat := new(big.Float).SetInt(amount)
divisorFloat := big.NewFloat(divisor)
result := new(big.Float).Quo(amountFloat, divisorFloat)
val, _ := result.Float64()
return val
}
// ConvertToBigFloat converts a token amount to a *big.Float considering its decimals
func ConvertToBigFloat(amount *big.Int, tokenAddress common.Address) *big.Float {
if amount == nil {
return big.NewFloat(0)
}
decimals := GetTokenDecimals(tokenAddress)
divisor := math.Pow(10, float64(decimals))
amountFloat := new(big.Float).SetInt(amount)
divisorFloat := big.NewFloat(divisor)
return new(big.Float).Quo(amountFloat, divisorFloat)
}
// ConvertFromFloat converts a float64 to the token's smallest unit (*big.Int)
func ConvertFromFloat(amount float64, tokenAddress common.Address) *big.Int {
decimals := GetTokenDecimals(tokenAddress)
multiplier := math.Pow(10, float64(decimals))
// Convert to smallest unit
rawAmount := amount * multiplier
// Convert to big.Int
amountBig := new(big.Float).SetFloat64(rawAmount)
result, _ := amountBig.Int(nil)
return result
}
// NormalizeAmount converts an amount from one token's decimals to another's
func NormalizeAmount(amount *big.Int, fromToken, toToken common.Address) *big.Int {
if amount == nil {
return big.NewInt(0)
}
fromDecimals := GetTokenDecimals(fromToken)
toDecimals := GetTokenDecimals(toToken)
if fromDecimals == toDecimals {
return new(big.Int).Set(amount)
}
// Convert to float with proper decimals
amountFloat := ConvertToBigFloat(amount, fromToken)
// Convert back with target token decimals
toMultiplier := math.Pow(10, float64(toDecimals))
multiplierBig := big.NewFloat(toMultiplier)
result := new(big.Float).Mul(amountFloat, multiplierBig)
resultInt, _ := result.Int(nil)
return resultInt
}
// GetMinimumTradeAmount returns the minimum trade amount for a token
// This helps avoid dust trades that aren't economically viable
func GetMinimumTradeAmount(tokenAddress common.Address) *big.Int {
// Note: decimals are handled in ConvertFromFloat
// Minimum amounts in token units (not smallest units)
var minAmount float64
switch tokenAddress {
case USDC, USDT:
minAmount = 10.0 // $10 minimum for stablecoins
case WBTC:
minAmount = 0.0001 // 0.0001 BTC minimum
case WETH, ARB, UNI, LINK, GMX:
minAmount = 0.01 // 0.01 token minimum
default:
minAmount = 0.01 // Default minimum
}
return ConvertFromFloat(minAmount, tokenAddress)
}
// FormatTokenAmount formats a token amount for display
func FormatTokenAmount(amount *big.Int, tokenAddress common.Address, precision int) string {
if amount == nil {
return "0"
}
amountFloat := ConvertToBigFloat(amount, tokenAddress)
return amountFloat.Text('f', precision)
}

View File

@@ -0,0 +1,612 @@
package tokens
import (
"encoding/json"
"fmt"
"os"
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/fraktal/mev-beta/internal/logger"
)
// TokenMetadata represents cached token information
type TokenMetadata struct {
Address common.Address `json:"address"`
Symbol string `json:"symbol"`
Name string `json:"name"`
Decimals uint8 `json:"decimals"`
TotalSupply string `json:"totalSupply,omitempty"`
Verified bool `json:"verified"`
FirstSeen time.Time `json:"firstSeen"`
LastSeen time.Time `json:"lastSeen"`
SeenCount uint64 `json:"seenCount"`
}
// MetadataCache manages token metadata with persistent storage
type MetadataCache struct {
cache map[common.Address]*TokenMetadata
mutex sync.RWMutex
logger *logger.Logger
cacheFile string
}
// NewMetadataCache creates a new token metadata cache
func NewMetadataCache(logger *logger.Logger) *MetadataCache {
mc := &MetadataCache{
cache: make(map[common.Address]*TokenMetadata),
logger: logger,
cacheFile: "data/tokens.json",
}
// Ensure data directory exists
os.MkdirAll("data", 0750)
// Load persisted data
mc.loadFromDisk()
return mc
}
// Get retrieves token metadata from cache
func (mc *MetadataCache) Get(address common.Address) (*TokenMetadata, bool) {
mc.mutex.RLock()
defer mc.mutex.RUnlock()
metadata, exists := mc.cache[address]
if exists {
return metadata, true
}
return nil, false
}
// Set stores token metadata in cache
func (mc *MetadataCache) Set(metadata *TokenMetadata) {
mc.mutex.Lock()
defer mc.mutex.Unlock()
// Update last seen and count
if existing, exists := mc.cache[metadata.Address]; exists {
metadata.FirstSeen = existing.FirstSeen
metadata.SeenCount = existing.SeenCount + 1
} else {
metadata.FirstSeen = time.Now()
metadata.SeenCount = 1
}
metadata.LastSeen = time.Now()
mc.cache[metadata.Address] = metadata
// Persist every 10 additions
if metadata.SeenCount%10 == 0 {
go mc.saveToDisk()
}
}
// GetOrCreate retrieves metadata or creates placeholder
func (mc *MetadataCache) GetOrCreate(address common.Address) *TokenMetadata {
if metadata, exists := mc.Get(address); exists {
return metadata
}
// Create placeholder
metadata := &TokenMetadata{
Address: address,
Symbol: "UNKNOWN",
Name: "Unknown Token",
Decimals: 18, // Default assumption
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
}
mc.Set(metadata)
return metadata
}
// Update modifies existing token metadata
func (mc *MetadataCache) Update(address common.Address, symbol, name string, decimals uint8) {
mc.mutex.Lock()
defer mc.mutex.Unlock()
metadata, exists := mc.cache[address]
if !exists {
metadata = &TokenMetadata{
Address: address,
FirstSeen: time.Now(),
}
}
metadata.Symbol = symbol
metadata.Name = name
metadata.Decimals = decimals
metadata.Verified = true
metadata.LastSeen = time.Now()
metadata.SeenCount++
mc.cache[address] = metadata
// Persist after verification
go mc.saveToDisk()
}
// Count returns the number of cached tokens
func (mc *MetadataCache) Count() int {
mc.mutex.RLock()
defer mc.mutex.RUnlock()
return len(mc.cache)
}
// GetAll returns all cached tokens
func (mc *MetadataCache) GetAll() map[common.Address]*TokenMetadata {
mc.mutex.RLock()
defer mc.mutex.RUnlock()
// Create a copy to avoid race conditions
result := make(map[common.Address]*TokenMetadata, len(mc.cache))
for addr, metadata := range mc.cache {
result[addr] = metadata
}
return result
}
// GetVerified returns only verified tokens
func (mc *MetadataCache) GetVerified() []*TokenMetadata {
mc.mutex.RLock()
defer mc.mutex.RUnlock()
verified := make([]*TokenMetadata, 0)
for _, metadata := range mc.cache {
if metadata.Verified {
verified = append(verified, metadata)
}
}
return verified
}
// saveToDisk persists cache to disk
func (mc *MetadataCache) saveToDisk() {
mc.mutex.RLock()
defer mc.mutex.RUnlock()
// Convert map to slice for JSON marshaling
tokens := make([]*TokenMetadata, 0, len(mc.cache))
for _, metadata := range mc.cache {
tokens = append(tokens, metadata)
}
data, err := json.MarshalIndent(tokens, "", " ")
if err != nil {
mc.logger.Error(fmt.Sprintf("Failed to marshal token cache: %v", err))
return
}
if err := os.WriteFile(mc.cacheFile, data, 0644); err != nil {
mc.logger.Error(fmt.Sprintf("Failed to save token cache: %v", err))
return
}
mc.logger.Debug(fmt.Sprintf("Saved %d tokens to cache", len(tokens)))
}
// loadFromDisk loads persisted cache
func (mc *MetadataCache) loadFromDisk() {
data, err := os.ReadFile(mc.cacheFile)
if err != nil {
// File doesn't exist yet, that's okay
mc.logger.Debug("No existing token cache found, starting fresh")
return
}
var tokens []*TokenMetadata
if err := json.Unmarshal(data, &tokens); err != nil {
mc.logger.Error(fmt.Sprintf("Failed to unmarshal token cache: %v", err))
return
}
mc.mutex.Lock()
defer mc.mutex.Unlock()
for _, metadata := range tokens {
mc.cache[metadata.Address] = metadata
}
mc.logger.Info(fmt.Sprintf("Loaded %d tokens from cache", len(tokens)))
}
// PopulateWithKnownTokens loads all known Arbitrum tokens into the cache
func (mc *MetadataCache) PopulateWithKnownTokens() {
mc.mutex.Lock()
defer mc.mutex.Unlock()
// Define all known Arbitrum tokens with their metadata
// CRITICAL FIX: Expanded from 20 to 100+ tokens to cover most swap pairs
knownTokens := map[string]*TokenMetadata{
// Tier 1 - Major Assets
"0x82aF49447D8a07e3bd95BD0d56f35241523fBab1": {
Address: common.HexToAddress("0x82aF49447D8a07e3bd95BD0d56f35241523fBab1"),
Symbol: "WETH",
Name: "Wrapped Ether",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xaf88d065e77c8cC2239327C5EDb3A432268e5831": {
Address: common.HexToAddress("0xaf88d065e77c8cC2239327C5EDb3A432268e5831"),
Symbol: "USDC",
Name: "USD Coin",
Decimals: 6,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xFd086bC7CD5C481DCC9C85ebE478A1C0b69FCbb9": {
Address: common.HexToAddress("0xFd086bC7CD5C481DCC9C85ebE478A1C0b69FCbb9"),
Symbol: "USDT",
Name: "Tether USD",
Decimals: 6,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x912CE59144191C1204E64559FE8253a0e49E6548": {
Address: common.HexToAddress("0x912CE59144191C1204E64559FE8253a0e49E6548"),
Symbol: "ARB",
Name: "Arbitrum",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x2f2a2543B76A4166549F7aaB2e75Bef0aefC5B0f": {
Address: common.HexToAddress("0x2f2a2543B76A4166549F7aaB2e75Bef0aefC5B0f"),
Symbol: "WBTC",
Name: "Wrapped Bitcoin",
Decimals: 8,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xDA10009cBd5D07dd0CeCc66161FC93D7c9000da1": {
Address: common.HexToAddress("0xDA10009cBd5D07dd0CeCc66161FC93D7c9000da1"),
Symbol: "DAI",
Name: "Dai Stablecoin",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xf97f4df75117a78c1A5a0DBb814Af92458539FB4": {
Address: common.HexToAddress("0xf97f4df75117a78c1A5a0DBb814Af92458539FB4"),
Symbol: "LINK",
Name: "ChainLink Token",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xFa7F8980b0f1E64A2062791cc3b0871572f1F7f0": {
Address: common.HexToAddress("0xFa7F8980b0f1E64A2062791cc3b0871572f1F7f0"),
Symbol: "UNI",
Name: "Uniswap",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xfc5A1A6EB076a2C7aD06eD22C90d7E710E35ad0a": {
Address: common.HexToAddress("0xfc5A1A6EB076a2C7aD06eD22C90d7E710E35ad0a"),
Symbol: "GMX",
Name: "GMX",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x9623063377AD1B27544C965cCd7342f7EA7e88C7": {
Address: common.HexToAddress("0x9623063377AD1B27544C965cCd7342f7EA7e88C7"),
Symbol: "GRT",
Name: "The Graph",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
// Tier 2 - DeFi Blue Chips
"0xFF970A61A04b1cA14834A43f5dE4533eBDDB5CC8": {
Address: common.HexToAddress("0xFF970A61A04b1cA14834A43f5dE4533eBDDB5CC8"),
Symbol: "USDC.e",
Name: "USD Coin (Bridged)",
Decimals: 6,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x0c880f6761F1af8d9Aa9C466984b80DAb9a8c9e8": {
Address: common.HexToAddress("0x0c880f6761F1af8d9Aa9C466984b80DAb9a8c9e8"),
Symbol: "PENDLE",
Name: "Pendle",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x3082CC23568eA640225c2467653dB90e9250AaA0": {
Address: common.HexToAddress("0x3082CC23568eA640225c2467653dB90e9250AaA0"),
Symbol: "RDNT",
Name: "Radiant Capital",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x539bdE0d7Dbd336b79148AA742883198BBF60342": {
Address: common.HexToAddress("0x539bdE0d7Dbd336b79148AA742883198BBF60342"),
Symbol: "MAGIC",
Name: "Magic",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x3d9907F9a368ad0a51Be60f7Da3b97cf940982D8": {
Address: common.HexToAddress("0x3d9907F9a368ad0a51Be60f7Da3b97cf940982D8"),
Symbol: "GRAIL",
Name: "Camelot (GRAIL)",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
// Tier 3 - Additional High Volume
"0xba5DdD1f9d7F570dc94a51479a000E3BCE967196": {
Address: common.HexToAddress("0xba5DdD1f9d7F570dc94a51479a000E3BCE967196"),
Symbol: "AAVE",
Name: "Aave",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x11cDb42B0EB46D95f990BeDD4695A6e3fA034978": {
Address: common.HexToAddress("0x11cDb42B0EB46D95f990BeDD4695A6e3fA034978"),
Symbol: "CRV",
Name: "Curve",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x040d1EdC9569d4Bab2D15287Dc5A4F10F56a56B8": {
Address: common.HexToAddress("0x040d1EdC9569d4Bab2D15287Dc5A4F10F56a56B8"),
Symbol: "BAL",
Name: "Balancer",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x354A6dA3fcde098F8389cad84b0182725c6C91dE": {
Address: common.HexToAddress("0x354A6dA3fcde098F8389cad84b0182725c6C91dE"),
Symbol: "COMP",
Name: "Compound",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x2e9a6Df78E42a30712c10a9Dc4b1C8656f8F2879": {
Address: common.HexToAddress("0x2e9a6Df78E42a30712c10a9Dc4b1C8656f8F2879"),
Symbol: "MKR",
Name: "Maker",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
// Additional tokens from recent swap logs (Aug-Nov 2025)
"0x60bf4E7c928fBa30a1Dd929a41239e0d07F2a81": {
Address: common.HexToAddress("0x60bf4E7c928fBa30a1Dd929a41239e0d07F2a81"),
Symbol: "UNKNOWN1",
Name: "Unknown Token 1",
Decimals: 18,
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x440017A1e65eFEC28c2A68e45F5D1E8d86F0CaA": {
Address: common.HexToAddress("0x440017A1e65eFEC28c2A68e45F5D1E8d86F0CaA"),
Symbol: "G@ARB",
Name: "G@ARB Token",
Decimals: 18,
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x25118290285e6e485ab10cA39fb37e1bab6dFfd": {
Address: common.HexToAddress("0x25118290285e6e485ab10cA39fb37e1bab6dFfd"),
Symbol: "UNKNOWN2",
Name: "Unknown Token 2",
Decimals: 18,
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xc87B37a5876c32c533Cff8f38e9D68FddB159Fe0": {
Address: common.HexToAddress("0xc87B37a5876c32c533Cff8f38e9D68FddB159Fe0"),
Symbol: "UNKNOWN3",
Name: "Unknown Token 3",
Decimals: 18,
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xEC70Dcb4A1EFa46b8F2D97C310dd9592301166e1": {
Address: common.HexToAddress("0xEC70Dcb4A1EFa46b8F2D97C310dd9592301166e1"),
Symbol: "UNKNOWN4",
Name: "Unknown Token 4",
Decimals: 18,
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x5979D7b5D69714eFf41eEA881e32bfeE7e76c2a0": {
Address: common.HexToAddress("0x5979D7b5D69714eFf41eEA881e32bfeE7e76c2a0"),
Symbol: "UNKNOWN5",
Name: "Unknown Token 5",
Decimals: 18,
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
// Additional commonly traded tokens
"0x13Ad51ed4F1B7d4DDc2299712B5be4vB9c3ca3d0": {
Address: common.HexToAddress("0x13Ad51ed4F1B7d4DDc2299712B5be4vB9c3ca3d0"),
Symbol: "CAMELOT",
Name: "Camelot",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x8d9ba70aATA2c8c87eaa49b6ccBFf5eFDCD1e98F": {
Address: common.HexToAddress("0x8d9ba70aATA2c8c87eaa49b6ccBFf5eFDCD1e98F"),
Symbol: "SUSHI",
Name: "Sushi",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x17FC002b466eKd288CcF891F6d0bAb1a905c9F0": {
Address: common.HexToAddress("0x17FC002b466eKd288CcF891F6d0bAb1a905c9F0"),
Symbol: "1INCH",
Name: "1inch Token",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xcAFcD85b12dEa2039E4aC0F4eeB552014e41F716": {
Address: common.HexToAddress("0xcAFcD85b12dEa2039E4aC0F4eeB552014e41F716"),
Symbol: "WSTETH",
Name: "Wrapped stETH",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x0000000000000000000000000000000000000000": {
Address: common.HexToAddress("0x0000000000000000000000000000000000000000"),
Symbol: "ETH",
Name: "Ether",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
}
// Load all known tokens into cache
for _, metadata := range knownTokens {
// Only add if not already in cache
if _, exists := mc.cache[metadata.Address]; !exists {
mc.cache[metadata.Address] = metadata
}
}
mc.logger.Info(fmt.Sprintf("✅ Populated token metadata cache with %d known tokens", len(knownTokens)))
}
// SaveAndClose persists cache and cleans up
func (mc *MetadataCache) SaveAndClose() {
mc.saveToDisk()
mc.logger.Info("Token metadata cache saved and closed")
}
// PruneOld removes tokens not seen in the last 30 days
func (mc *MetadataCache) PruneOld(daysOld int) int {
mc.mutex.Lock()
defer mc.mutex.Unlock()
cutoff := time.Now().AddDate(0, 0, -daysOld)
pruned := 0
for addr, metadata := range mc.cache {
if metadata.LastSeen.Before(cutoff) {
delete(mc.cache, addr)
pruned++
}
}
if pruned > 0 {
mc.logger.Info(fmt.Sprintf("Pruned %d old tokens from cache", pruned))
go mc.saveToDisk()
}
return pruned
}
// GetStatistics returns cache statistics
func (mc *MetadataCache) GetStatistics() map[string]interface{} {
mc.mutex.RLock()
defer mc.mutex.RUnlock()
verified := 0
unverified := 0
totalSeen := uint64(0)
for _, metadata := range mc.cache {
if metadata.Verified {
verified++
} else {
unverified++
}
totalSeen += metadata.SeenCount
}
return map[string]interface{}{
"total_tokens": len(mc.cache),
"verified_tokens": verified,
"unverified_tokens": unverified,
"total_observations": totalSeen,
"cache_file": mc.cacheFile,
}
}

View File

@@ -0,0 +1,383 @@
package tokens
import (
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/stretchr/testify/assert"
"github.com/fraktal/mev-beta/internal/logger"
)
func TestNewMetadataCache(t *testing.T) {
log := logger.New("info", "text", "")
cache := NewMetadataCache(log)
assert.NotNil(t, cache)
assert.NotNil(t, cache.cache)
assert.Equal(t, log, cache.logger)
assert.Equal(t, "data/tokens.json", cache.cacheFile)
}
func TestTokenMetadataCreation(t *testing.T) {
metadata := &TokenMetadata{
Address: common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"),
Symbol: "USDC",
Name: "USD Coin",
Decimals: 6,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
}
assert.Equal(t, "USDC", metadata.Symbol)
assert.Equal(t, "USD Coin", metadata.Name)
assert.Equal(t, uint8(6), metadata.Decimals)
assert.True(t, metadata.Verified)
assert.Equal(t, uint64(1), metadata.SeenCount)
}
func TestCacheGetMissing(t *testing.T) {
log := logger.New("info", "text", "")
cache := NewMetadataCache(log)
address := common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48")
metadata, exists := cache.Get(address)
assert.False(t, exists)
assert.Nil(t, metadata)
}
func TestCacheSetAndGet(t *testing.T) {
log := logger.New("info", "text", "")
cache := NewMetadataCache(log)
metadata := &TokenMetadata{
Address: common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"),
Symbol: "USDC",
Name: "USD Coin",
Decimals: 6,
Verified: true,
SeenCount: 1,
}
cache.Set(metadata)
retrieved, exists := cache.Get(metadata.Address)
assert.True(t, exists)
assert.NotNil(t, retrieved)
assert.Equal(t, "USDC", retrieved.Symbol)
assert.Equal(t, uint8(6), retrieved.Decimals)
}
func TestCacheSetFirstSeen(t *testing.T) {
log := logger.New("info", "text", "")
cache := NewMetadataCache(log)
metadata := &TokenMetadata{
Address: common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"),
Symbol: "USDC",
Name: "USD Coin",
Decimals: 6,
}
before := time.Now()
cache.Set(metadata)
after := time.Now()
retrieved, exists := cache.Get(metadata.Address)
assert.True(t, exists)
assert.NotNil(t, retrieved.FirstSeen)
assert.True(t, retrieved.FirstSeen.After(before.Add(-1*time.Second)) || retrieved.FirstSeen.Equal(before))
assert.True(t, retrieved.FirstSeen.Before(after.Add(1*time.Second)) || retrieved.FirstSeen.Equal(after))
}
func TestCacheMultipleTokens(t *testing.T) {
log := logger.New("info", "text", "")
cache := NewMetadataCache(log)
tokens := []struct {
address string
symbol string
decimals uint8
}{
{"0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", "USDC", 6},
{"0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", "WETH", 18},
{"0xdAC17F958D2ee523a2206206994597C13D831ec7", "USDT", 6},
}
for _, token := range tokens {
metadata := &TokenMetadata{
Address: common.HexToAddress(token.address),
Symbol: token.symbol,
Decimals: token.decimals,
Verified: true,
SeenCount: 1,
}
cache.Set(metadata)
}
assert.Equal(t, 3, len(cache.cache))
for _, token := range tokens {
retrieved, exists := cache.Get(common.HexToAddress(token.address))
assert.True(t, exists)
assert.Equal(t, token.symbol, retrieved.Symbol)
assert.Equal(t, token.decimals, retrieved.Decimals)
}
}
func TestGetOrCreateMissing(t *testing.T) {
log := logger.New("info", "text", "")
cache := NewMetadataCache(log)
address := common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48")
metadata := cache.GetOrCreate(address)
assert.NotNil(t, metadata)
assert.Equal(t, "UNKNOWN", metadata.Symbol)
assert.Equal(t, "Unknown Token", metadata.Name)
assert.Equal(t, uint8(18), metadata.Decimals) // Default assumption
assert.False(t, metadata.Verified)
assert.Equal(t, address, metadata.Address)
}
func TestGetOrCreateExisting(t *testing.T) {
log := logger.New("info", "text", "")
cache := NewMetadataCache(log)
address := common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48")
metadata := &TokenMetadata{
Address: address,
Symbol: "USDC",
Decimals: 6,
Verified: true,
SeenCount: 1,
}
cache.Set(metadata)
retrieved := cache.GetOrCreate(address)
assert.Equal(t, "USDC", retrieved.Symbol)
assert.Equal(t, uint8(6), retrieved.Decimals)
assert.True(t, retrieved.Verified)
}
func TestSeenCountIncrement(t *testing.T) {
log := logger.New("info", "text", "")
cache := NewMetadataCache(log)
address := common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48")
metadata := &TokenMetadata{
Address: address,
Symbol: "USDC",
Decimals: 6,
SeenCount: 1,
}
// First set
cache.Set(metadata)
retrieved, _ := cache.Get(address)
assert.Equal(t, uint64(1), retrieved.SeenCount)
// Second set - should increment
metadata2 := &TokenMetadata{
Address: address,
Symbol: "USDC",
Decimals: 6,
SeenCount: 1,
}
cache.Set(metadata2)
retrieved2, _ := cache.Get(address)
assert.Equal(t, uint64(2), retrieved2.SeenCount)
}
func TestLastSeenUpdate(t *testing.T) {
log := logger.New("info", "text", "")
cache := NewMetadataCache(log)
address := common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48")
metadata := &TokenMetadata{
Address: address,
Symbol: "USDC",
Decimals: 6,
}
cache.Set(metadata)
firstLastSeen := cache.cache[address].LastSeen
time.Sleep(10 * time.Millisecond)
metadata2 := &TokenMetadata{
Address: address,
Symbol: "USDC",
Decimals: 6,
}
cache.Set(metadata2)
secondLastSeen := cache.cache[address].LastSeen
assert.True(t, secondLastSeen.After(firstLastSeen))
}
func TestFirstSeenPreserved(t *testing.T) {
log := logger.New("info", "text", "")
cache := NewMetadataCache(log)
address := common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48")
metadata := &TokenMetadata{
Address: address,
Symbol: "USDC",
Decimals: 6,
}
cache.Set(metadata)
firstFirstSeen := cache.cache[address].FirstSeen
time.Sleep(10 * time.Millisecond)
metadata2 := &TokenMetadata{
Address: address,
Symbol: "USDC",
Decimals: 6,
}
cache.Set(metadata2)
secondFirstSeen := cache.cache[address].FirstSeen
assert.Equal(t, firstFirstSeen, secondFirstSeen)
}
func TestTokenMetadataVerified(t *testing.T) {
tests := []struct {
name string
verified bool
symbol string
}{
{"Verified USDC", true, "USDC"},
{"Verified WETH", true, "WETH"},
{"Unverified token", false, "UNKNOWN"},
{"Unverified custom", false, "CUSTOM"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
metadata := &TokenMetadata{
Address: common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"),
Symbol: tt.symbol,
Verified: tt.verified,
SeenCount: 1,
}
assert.Equal(t, tt.verified, metadata.Verified)
})
}
}
func TestTotalSupplyMetadata(t *testing.T) {
metadata := &TokenMetadata{
Address: common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"),
Symbol: "USDC",
TotalSupply: "30000000000000000", // 30M USDC
Decimals: 6,
Verified: true,
SeenCount: 1,
}
assert.NotEmpty(t, metadata.TotalSupply)
assert.Equal(t, "30000000000000000", metadata.TotalSupply)
}
func TestCacheConcurrency(t *testing.T) {
log := logger.New("info", "text", "")
cache := NewMetadataCache(log)
done := make(chan bool, 10)
errors := make(chan error, 10)
// Test concurrent writes
for i := 0; i < 5; i++ {
go func(index int) {
addr := common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48")
metadata := &TokenMetadata{
Address: addr,
Symbol: "USDC",
Decimals: 6,
SeenCount: uint64(index),
}
cache.Set(metadata)
done <- true
}(i)
}
// Test concurrent reads
for i := 0; i < 5; i++ {
go func() {
addr := common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48")
_, _ = cache.Get(addr)
done <- true
}()
}
// Wait for all goroutines
for i := 0; i < 10; i++ {
select {
case <-done:
// Success
case <-errors:
t.Fatal("Concurrent operation failed")
}
}
}
func TestCacheSize(t *testing.T) {
log := logger.New("info", "text", "")
cache := NewMetadataCache(log)
addresses := []string{
"0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48",
"0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2",
"0xdAC17F958D2ee523a2206206994597C13D831ec7",
"0x6B175474E89094C44Da98b954EedeAC495271d0F",
"0x2260FAC5E5542a773Aa44fBCfeDd66150d0310be",
}
for _, addr := range addresses {
metadata := &TokenMetadata{
Address: common.HexToAddress(addr),
Symbol: "TEST",
Decimals: 18,
SeenCount: 1,
}
cache.Set(metadata)
}
assert.Equal(t, len(addresses), len(cache.cache))
}
func TestMetadataDecimalVariations(t *testing.T) {
tests := []struct {
symbol string
decimals uint8
expected uint8
}{
{"USDC", 6, 6},
{"USDT", 6, 6},
{"WETH", 18, 18},
{"DAI", 18, 18},
{"WBTC", 8, 8},
{"LINK", 18, 18},
{"AAVE", 18, 18},
}
for _, tt := range tests {
t.Run(tt.symbol, func(t *testing.T) {
metadata := &TokenMetadata{
Address: common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"),
Symbol: tt.symbol,
Decimals: tt.decimals,
SeenCount: 1,
}
assert.Equal(t, tt.expected, metadata.Decimals)
})
}
}