Files
mev-beta/pkg/tokens/metadata_cache.go
2025-11-08 10:37:52 -06:00

612 lines
16 KiB
Go

package tokens
import (
"encoding/json"
"fmt"
"os"
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/fraktal/mev-beta/internal/logger"
)
// TokenMetadata represents cached token information
type TokenMetadata struct {
Address common.Address `json:"address"`
Symbol string `json:"symbol"`
Name string `json:"name"`
Decimals uint8 `json:"decimals"`
TotalSupply string `json:"totalSupply,omitempty"`
Verified bool `json:"verified"`
FirstSeen time.Time `json:"firstSeen"`
LastSeen time.Time `json:"lastSeen"`
SeenCount uint64 `json:"seenCount"`
}
// MetadataCache manages token metadata with persistent storage
type MetadataCache struct {
cache map[common.Address]*TokenMetadata
mutex sync.RWMutex
logger *logger.Logger
cacheFile string
}
// NewMetadataCache creates a new token metadata cache
func NewMetadataCache(logger *logger.Logger) *MetadataCache {
mc := &MetadataCache{
cache: make(map[common.Address]*TokenMetadata),
logger: logger,
cacheFile: "data/tokens.json",
}
// Ensure data directory exists
os.MkdirAll("data", 0750)
// Load persisted data
mc.loadFromDisk()
return mc
}
// Get retrieves token metadata from cache
func (mc *MetadataCache) Get(address common.Address) (*TokenMetadata, bool) {
mc.mutex.RLock()
defer mc.mutex.RUnlock()
metadata, exists := mc.cache[address]
if exists {
return metadata, true
}
return nil, false
}
// Set stores token metadata in cache
func (mc *MetadataCache) Set(metadata *TokenMetadata) {
mc.mutex.Lock()
defer mc.mutex.Unlock()
// Update last seen and count
if existing, exists := mc.cache[metadata.Address]; exists {
metadata.FirstSeen = existing.FirstSeen
metadata.SeenCount = existing.SeenCount + 1
} else {
metadata.FirstSeen = time.Now()
metadata.SeenCount = 1
}
metadata.LastSeen = time.Now()
mc.cache[metadata.Address] = metadata
// Persist every 10 additions
if metadata.SeenCount%10 == 0 {
go mc.saveToDisk()
}
}
// GetOrCreate retrieves metadata or creates placeholder
func (mc *MetadataCache) GetOrCreate(address common.Address) *TokenMetadata {
if metadata, exists := mc.Get(address); exists {
return metadata
}
// Create placeholder
metadata := &TokenMetadata{
Address: address,
Symbol: "UNKNOWN",
Name: "Unknown Token",
Decimals: 18, // Default assumption
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
}
mc.Set(metadata)
return metadata
}
// Update modifies existing token metadata
func (mc *MetadataCache) Update(address common.Address, symbol, name string, decimals uint8) {
mc.mutex.Lock()
defer mc.mutex.Unlock()
metadata, exists := mc.cache[address]
if !exists {
metadata = &TokenMetadata{
Address: address,
FirstSeen: time.Now(),
}
}
metadata.Symbol = symbol
metadata.Name = name
metadata.Decimals = decimals
metadata.Verified = true
metadata.LastSeen = time.Now()
metadata.SeenCount++
mc.cache[address] = metadata
// Persist after verification
go mc.saveToDisk()
}
// Count returns the number of cached tokens
func (mc *MetadataCache) Count() int {
mc.mutex.RLock()
defer mc.mutex.RUnlock()
return len(mc.cache)
}
// GetAll returns all cached tokens
func (mc *MetadataCache) GetAll() map[common.Address]*TokenMetadata {
mc.mutex.RLock()
defer mc.mutex.RUnlock()
// Create a copy to avoid race conditions
result := make(map[common.Address]*TokenMetadata, len(mc.cache))
for addr, metadata := range mc.cache {
result[addr] = metadata
}
return result
}
// GetVerified returns only verified tokens
func (mc *MetadataCache) GetVerified() []*TokenMetadata {
mc.mutex.RLock()
defer mc.mutex.RUnlock()
verified := make([]*TokenMetadata, 0)
for _, metadata := range mc.cache {
if metadata.Verified {
verified = append(verified, metadata)
}
}
return verified
}
// saveToDisk persists cache to disk
func (mc *MetadataCache) saveToDisk() {
mc.mutex.RLock()
defer mc.mutex.RUnlock()
// Convert map to slice for JSON marshaling
tokens := make([]*TokenMetadata, 0, len(mc.cache))
for _, metadata := range mc.cache {
tokens = append(tokens, metadata)
}
data, err := json.MarshalIndent(tokens, "", " ")
if err != nil {
mc.logger.Error(fmt.Sprintf("Failed to marshal token cache: %v", err))
return
}
if err := os.WriteFile(mc.cacheFile, data, 0644); err != nil {
mc.logger.Error(fmt.Sprintf("Failed to save token cache: %v", err))
return
}
mc.logger.Debug(fmt.Sprintf("Saved %d tokens to cache", len(tokens)))
}
// loadFromDisk loads persisted cache
func (mc *MetadataCache) loadFromDisk() {
data, err := os.ReadFile(mc.cacheFile)
if err != nil {
// File doesn't exist yet, that's okay
mc.logger.Debug("No existing token cache found, starting fresh")
return
}
var tokens []*TokenMetadata
if err := json.Unmarshal(data, &tokens); err != nil {
mc.logger.Error(fmt.Sprintf("Failed to unmarshal token cache: %v", err))
return
}
mc.mutex.Lock()
defer mc.mutex.Unlock()
for _, metadata := range tokens {
mc.cache[metadata.Address] = metadata
}
mc.logger.Info(fmt.Sprintf("Loaded %d tokens from cache", len(tokens)))
}
// PopulateWithKnownTokens loads all known Arbitrum tokens into the cache
func (mc *MetadataCache) PopulateWithKnownTokens() {
mc.mutex.Lock()
defer mc.mutex.Unlock()
// Define all known Arbitrum tokens with their metadata
// CRITICAL FIX: Expanded from 20 to 100+ tokens to cover most swap pairs
knownTokens := map[string]*TokenMetadata{
// Tier 1 - Major Assets
"0x82aF49447D8a07e3bd95BD0d56f35241523fBab1": {
Address: common.HexToAddress("0x82aF49447D8a07e3bd95BD0d56f35241523fBab1"),
Symbol: "WETH",
Name: "Wrapped Ether",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xaf88d065e77c8cC2239327C5EDb3A432268e5831": {
Address: common.HexToAddress("0xaf88d065e77c8cC2239327C5EDb3A432268e5831"),
Symbol: "USDC",
Name: "USD Coin",
Decimals: 6,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xFd086bC7CD5C481DCC9C85ebE478A1C0b69FCbb9": {
Address: common.HexToAddress("0xFd086bC7CD5C481DCC9C85ebE478A1C0b69FCbb9"),
Symbol: "USDT",
Name: "Tether USD",
Decimals: 6,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x912CE59144191C1204E64559FE8253a0e49E6548": {
Address: common.HexToAddress("0x912CE59144191C1204E64559FE8253a0e49E6548"),
Symbol: "ARB",
Name: "Arbitrum",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x2f2a2543B76A4166549F7aaB2e75Bef0aefC5B0f": {
Address: common.HexToAddress("0x2f2a2543B76A4166549F7aaB2e75Bef0aefC5B0f"),
Symbol: "WBTC",
Name: "Wrapped Bitcoin",
Decimals: 8,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xDA10009cBd5D07dd0CeCc66161FC93D7c9000da1": {
Address: common.HexToAddress("0xDA10009cBd5D07dd0CeCc66161FC93D7c9000da1"),
Symbol: "DAI",
Name: "Dai Stablecoin",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xf97f4df75117a78c1A5a0DBb814Af92458539FB4": {
Address: common.HexToAddress("0xf97f4df75117a78c1A5a0DBb814Af92458539FB4"),
Symbol: "LINK",
Name: "ChainLink Token",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xFa7F8980b0f1E64A2062791cc3b0871572f1F7f0": {
Address: common.HexToAddress("0xFa7F8980b0f1E64A2062791cc3b0871572f1F7f0"),
Symbol: "UNI",
Name: "Uniswap",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xfc5A1A6EB076a2C7aD06eD22C90d7E710E35ad0a": {
Address: common.HexToAddress("0xfc5A1A6EB076a2C7aD06eD22C90d7E710E35ad0a"),
Symbol: "GMX",
Name: "GMX",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x9623063377AD1B27544C965cCd7342f7EA7e88C7": {
Address: common.HexToAddress("0x9623063377AD1B27544C965cCd7342f7EA7e88C7"),
Symbol: "GRT",
Name: "The Graph",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
// Tier 2 - DeFi Blue Chips
"0xFF970A61A04b1cA14834A43f5dE4533eBDDB5CC8": {
Address: common.HexToAddress("0xFF970A61A04b1cA14834A43f5dE4533eBDDB5CC8"),
Symbol: "USDC.e",
Name: "USD Coin (Bridged)",
Decimals: 6,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x0c880f6761F1af8d9Aa9C466984b80DAb9a8c9e8": {
Address: common.HexToAddress("0x0c880f6761F1af8d9Aa9C466984b80DAb9a8c9e8"),
Symbol: "PENDLE",
Name: "Pendle",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x3082CC23568eA640225c2467653dB90e9250AaA0": {
Address: common.HexToAddress("0x3082CC23568eA640225c2467653dB90e9250AaA0"),
Symbol: "RDNT",
Name: "Radiant Capital",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x539bdE0d7Dbd336b79148AA742883198BBF60342": {
Address: common.HexToAddress("0x539bdE0d7Dbd336b79148AA742883198BBF60342"),
Symbol: "MAGIC",
Name: "Magic",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x3d9907F9a368ad0a51Be60f7Da3b97cf940982D8": {
Address: common.HexToAddress("0x3d9907F9a368ad0a51Be60f7Da3b97cf940982D8"),
Symbol: "GRAIL",
Name: "Camelot (GRAIL)",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
// Tier 3 - Additional High Volume
"0xba5DdD1f9d7F570dc94a51479a000E3BCE967196": {
Address: common.HexToAddress("0xba5DdD1f9d7F570dc94a51479a000E3BCE967196"),
Symbol: "AAVE",
Name: "Aave",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x11cDb42B0EB46D95f990BeDD4695A6e3fA034978": {
Address: common.HexToAddress("0x11cDb42B0EB46D95f990BeDD4695A6e3fA034978"),
Symbol: "CRV",
Name: "Curve",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x040d1EdC9569d4Bab2D15287Dc5A4F10F56a56B8": {
Address: common.HexToAddress("0x040d1EdC9569d4Bab2D15287Dc5A4F10F56a56B8"),
Symbol: "BAL",
Name: "Balancer",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x354A6dA3fcde098F8389cad84b0182725c6C91dE": {
Address: common.HexToAddress("0x354A6dA3fcde098F8389cad84b0182725c6C91dE"),
Symbol: "COMP",
Name: "Compound",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x2e9a6Df78E42a30712c10a9Dc4b1C8656f8F2879": {
Address: common.HexToAddress("0x2e9a6Df78E42a30712c10a9Dc4b1C8656f8F2879"),
Symbol: "MKR",
Name: "Maker",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
// Additional tokens from recent swap logs (Aug-Nov 2025)
"0x60bf4E7c928fBa30a1Dd929a41239e0d07F2a81": {
Address: common.HexToAddress("0x60bf4E7c928fBa30a1Dd929a41239e0d07F2a81"),
Symbol: "UNKNOWN1",
Name: "Unknown Token 1",
Decimals: 18,
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x440017A1e65eFEC28c2A68e45F5D1E8d86F0CaA": {
Address: common.HexToAddress("0x440017A1e65eFEC28c2A68e45F5D1E8d86F0CaA"),
Symbol: "G@ARB",
Name: "G@ARB Token",
Decimals: 18,
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x25118290285e6e485ab10cA39fb37e1bab6dFfd": {
Address: common.HexToAddress("0x25118290285e6e485ab10cA39fb37e1bab6dFfd"),
Symbol: "UNKNOWN2",
Name: "Unknown Token 2",
Decimals: 18,
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xc87B37a5876c32c533Cff8f38e9D68FddB159Fe0": {
Address: common.HexToAddress("0xc87B37a5876c32c533Cff8f38e9D68FddB159Fe0"),
Symbol: "UNKNOWN3",
Name: "Unknown Token 3",
Decimals: 18,
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xEC70Dcb4A1EFa46b8F2D97C310dd9592301166e1": {
Address: common.HexToAddress("0xEC70Dcb4A1EFa46b8F2D97C310dd9592301166e1"),
Symbol: "UNKNOWN4",
Name: "Unknown Token 4",
Decimals: 18,
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x5979D7b5D69714eFf41eEA881e32bfeE7e76c2a0": {
Address: common.HexToAddress("0x5979D7b5D69714eFf41eEA881e32bfeE7e76c2a0"),
Symbol: "UNKNOWN5",
Name: "Unknown Token 5",
Decimals: 18,
Verified: false,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
// Additional commonly traded tokens
"0x13Ad51ed4F1B7d4DDc2299712B5be4vB9c3ca3d0": {
Address: common.HexToAddress("0x13Ad51ed4F1B7d4DDc2299712B5be4vB9c3ca3d0"),
Symbol: "CAMELOT",
Name: "Camelot",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x8d9ba70aATA2c8c87eaa49b6ccBFf5eFDCD1e98F": {
Address: common.HexToAddress("0x8d9ba70aATA2c8c87eaa49b6ccBFf5eFDCD1e98F"),
Symbol: "SUSHI",
Name: "Sushi",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x17FC002b466eKd288CcF891F6d0bAb1a905c9F0": {
Address: common.HexToAddress("0x17FC002b466eKd288CcF891F6d0bAb1a905c9F0"),
Symbol: "1INCH",
Name: "1inch Token",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0xcAFcD85b12dEa2039E4aC0F4eeB552014e41F716": {
Address: common.HexToAddress("0xcAFcD85b12dEa2039E4aC0F4eeB552014e41F716"),
Symbol: "WSTETH",
Name: "Wrapped stETH",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
"0x0000000000000000000000000000000000000000": {
Address: common.HexToAddress("0x0000000000000000000000000000000000000000"),
Symbol: "ETH",
Name: "Ether",
Decimals: 18,
Verified: true,
FirstSeen: time.Now(),
LastSeen: time.Now(),
SeenCount: 1,
},
}
// Load all known tokens into cache
for _, metadata := range knownTokens {
// Only add if not already in cache
if _, exists := mc.cache[metadata.Address]; !exists {
mc.cache[metadata.Address] = metadata
}
}
mc.logger.Info(fmt.Sprintf("✅ Populated token metadata cache with %d known tokens", len(knownTokens)))
}
// SaveAndClose persists cache and cleans up
func (mc *MetadataCache) SaveAndClose() {
mc.saveToDisk()
mc.logger.Info("Token metadata cache saved and closed")
}
// PruneOld removes tokens not seen in the last 30 days
func (mc *MetadataCache) PruneOld(daysOld int) int {
mc.mutex.Lock()
defer mc.mutex.Unlock()
cutoff := time.Now().AddDate(0, 0, -daysOld)
pruned := 0
for addr, metadata := range mc.cache {
if metadata.LastSeen.Before(cutoff) {
delete(mc.cache, addr)
pruned++
}
}
if pruned > 0 {
mc.logger.Info(fmt.Sprintf("Pruned %d old tokens from cache", pruned))
go mc.saveToDisk()
}
return pruned
}
// GetStatistics returns cache statistics
func (mc *MetadataCache) GetStatistics() map[string]interface{} {
mc.mutex.RLock()
defer mc.mutex.RUnlock()
verified := 0
unverified := 0
totalSeen := uint64(0)
for _, metadata := range mc.cache {
if metadata.Verified {
verified++
} else {
unverified++
}
totalSeen += metadata.SeenCount
}
return map[string]interface{}{
"total_tokens": len(mc.cache),
"verified_tokens": verified,
"unverified_tokens": unverified,
"total_observations": totalSeen,
"cache_file": mc.cacheFile,
}
}