mev-beta/pkg/transport/persistence.go

package transport

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"time"
)

// FilePersistenceLayer implements file-based message persistence
type FilePersistenceLayer struct {
	basePath    string
	maxFileSize int64
	maxFiles    int
	compression bool
	encryption  EncryptionConfig
	mu          sync.RWMutex
}

// EncryptionConfig configures message encryption at rest
type EncryptionConfig struct {
	Enabled   bool
	Algorithm string
	Key       []byte
}

// PersistedMessage represents a message stored on disk
type PersistedMessage struct {
	ID        string                 `json:"id"`
	Topic     string                 `json:"topic"`
	Message   *Message               `json:"message"`
	Stored    time.Time              `json:"stored"`
	Metadata  map[string]interface{} `json:"metadata"`
	Encrypted bool                   `json:"encrypted"`
}

// PersistenceMetrics tracks persistence layer statistics
type PersistenceMetrics struct {
	MessagesStored    int64
	MessagesRetrieved int64
	MessagesDeleted   int64
	StorageSize       int64
	FileCount         int
	LastCleanup       time.Time
	Errors            int64
}

// NewFilePersistenceLayer creates a new file-based persistence layer
func NewFilePersistenceLayer(basePath string) *FilePersistenceLayer {
	return &FilePersistenceLayer{
		basePath:    basePath,
		maxFileSize: 100 * 1024 * 1024, // 100MB default
		maxFiles:    1000,
		compression: false,
	}
}

// SetMaxFileSize configures the maximum file size
func (fpl *FilePersistenceLayer) SetMaxFileSize(size int64) {
	fpl.mu.Lock()
	defer fpl.mu.Unlock()
	fpl.maxFileSize = size
}

// SetMaxFiles configures the maximum number of files
func (fpl *FilePersistenceLayer) SetMaxFiles(count int) {
	fpl.mu.Lock()
	defer fpl.mu.Unlock()
	fpl.maxFiles = count
}

// EnableCompression enables/disables compression
func (fpl *FilePersistenceLayer) EnableCompression(enabled bool) {
	fpl.mu.Lock()
	defer fpl.mu.Unlock()
	fpl.compression = enabled
}

// SetEncryption configures encryption settings
func (fpl *FilePersistenceLayer) SetEncryption(config EncryptionConfig) {
	fpl.mu.Lock()
	defer fpl.mu.Unlock()
	fpl.encryption = config
}

// Store persists a message to disk
func (fpl *FilePersistenceLayer) Store(msg *Message) error {
	fpl.mu.Lock()
	defer fpl.mu.Unlock()

	// Create directory if it doesn't exist
	topicDir := filepath.Join(fpl.basePath, msg.Topic)
	if err := os.MkdirAll(topicDir, 0755); err != nil {
		return fmt.Errorf("failed to create topic directory: %w", err)
	}

	// Create persisted message
	persistedMsg := &PersistedMessage{
		ID:       msg.ID,
		Topic:    msg.Topic,
		Message:  msg,
		Stored:   time.Now(),
		Metadata: make(map[string]interface{}),
	}

	// Serialize message
	data, err := json.Marshal(persistedMsg)
	if err != nil {
		return fmt.Errorf("failed to marshal message: %w", err)
	}

	// Apply encryption if enabled
	if fpl.encryption.Enabled {
		encryptedData, err := fpl.encrypt(data)
		if err != nil {
			return fmt.Errorf("encryption failed: %w", err)
		}
		data = encryptedData
		persistedMsg.Encrypted = true
	}

	// Apply compression if enabled
	if fpl.compression {
		compressedData, err := fpl.compress(data)
		if err != nil {
			return fmt.Errorf("compression failed: %w", err)
		}
		data = compressedData
	}

	// Find appropriate file to write to
	filename, err := fpl.getWritableFile(topicDir, len(data))
	if err != nil {
		return fmt.Errorf("failed to get writable file: %w", err)
	}

	// Write to file
	file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return fmt.Errorf("failed to open file: %w", err)
	}
	defer file.Close()

	// Write length prefix and data
	lengthPrefix := fmt.Sprintf("%d\n", len(data))
	if _, err := file.WriteString(lengthPrefix); err != nil {
		return fmt.Errorf("failed to write length prefix: %w", err)
	}
	if _, err := file.Write(data); err != nil {
		return fmt.Errorf("failed to write data: %w", err)
	}

	return nil
}

// Retrieve loads a message from disk by ID
func (fpl *FilePersistenceLayer) Retrieve(id string) (*Message, error) {
	fpl.mu.RLock()
	defer fpl.mu.RUnlock()

	// Search all topic directories
	topicDirs, err := fpl.getTopicDirectories()
	if err != nil {
		return nil, fmt.Errorf("failed to get topic directories: %w", err)
	}

	for _, topicDir := range topicDirs {
		files, err := fpl.getTopicFiles(topicDir)
		if err != nil {
			continue
		}

		for _, file := range files {
			msg, err := fpl.findMessageInFile(file, id)
			if err != nil {
				continue
			}
			if msg != nil {
				return msg, nil
			}
		}
	}

	return nil, fmt.Errorf("message not found: %s", id)
}

// Delete removes a message from disk by ID
func (fpl *FilePersistenceLayer) Delete(id string) error {
	fpl.mu.Lock()
	defer fpl.mu.Unlock()

	// This is a simplified implementation
	// In a production system, you might want to mark messages as deleted
	// and compact files periodically instead of rewriting entire files
	return fmt.Errorf("delete operation not yet implemented")
}

// List returns messages for a topic with optional limit
func (fpl *FilePersistenceLayer) List(topic string, limit int) ([]*Message, error) {
	fpl.mu.RLock()
	defer fpl.mu.RUnlock()

	topicDir := filepath.Join(fpl.basePath, topic)
	if _, err := os.Stat(topicDir); os.IsNotExist(err) {
		return []*Message{}, nil
	}

	files, err := fpl.getTopicFiles(topicDir)
	if err != nil {
		return nil, fmt.Errorf("failed to get topic files: %w", err)
	}

	var messages []*Message
	count := 0

	// Read files in chronological order (newest first)
	sort.Slice(files, func(i, j int) bool {
		infoI, _ := os.Stat(files[i])
		infoJ, _ := os.Stat(files[j])
		return infoI.ModTime().After(infoJ.ModTime())
	})

	for _, file := range files {
		fileMessages, err := fpl.readMessagesFromFile(file)
		if err != nil {
			continue
		}

		for _, msg := range fileMessages {
			messages = append(messages, msg)
			count++
			if limit > 0 && count >= limit {
				break
			}
		}

		if limit > 0 && count >= limit {
			break
		}
	}

	return messages, nil
}

// Cleanup removes messages older than maxAge
func (fpl *FilePersistenceLayer) Cleanup(maxAge time.Duration) error {
	fpl.mu.Lock()
	defer fpl.mu.Unlock()

	cutoff := time.Now().Add(-maxAge)

	topicDirs, err := fpl.getTopicDirectories()
	if err != nil {
		return fmt.Errorf("failed to get topic directories: %w", err)
	}

	for _, topicDir := range topicDirs {
		files, err := fpl.getTopicFiles(topicDir)
		if err != nil {
			continue
		}

		for _, file := range files {
			// Check file modification time
			info, err := os.Stat(file)
			if err != nil {
				continue
			}

			if info.ModTime().Before(cutoff) {
				os.Remove(file)
			}
		}

		// Remove empty topic directories
		if isEmpty, _ := fpl.isDirectoryEmpty(topicDir); isEmpty {
			os.Remove(topicDir)
		}
	}

	return nil
}

// GetMetrics returns persistence layer metrics
func (fpl *FilePersistenceLayer) GetMetrics() (PersistenceMetrics, error) {
	fpl.mu.RLock()
	defer fpl.mu.RUnlock()

	metrics := PersistenceMetrics{}

	// Calculate storage size and file count
	err := filepath.Walk(fpl.basePath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() {
			metrics.FileCount++
			metrics.StorageSize += info.Size()
		}
		return nil
	})

	return metrics, err
}

// Private helper methods

func (fpl *FilePersistenceLayer) getWritableFile(topicDir string, dataSize int) (string, error) {
	files, err := fpl.getTopicFiles(topicDir)
	if err != nil {
		return "", err
	}

	// Find a file with enough space
	for _, file := range files {
		info, err := os.Stat(file)
		if err != nil {
			continue
		}

		if info.Size()+int64(dataSize) <= fpl.maxFileSize {
			return file, nil
		}
	}

	// Create new file
	timestamp := time.Now().Format("20060102_150405")
	filename := filepath.Join(topicDir, fmt.Sprintf("messages_%s.dat", timestamp))
	return filename, nil
}

func (fpl *FilePersistenceLayer) getTopicDirectories() ([]string, error) {
	entries, err := ioutil.ReadDir(fpl.basePath)
	if err != nil {
		return nil, err
	}

	var dirs []string
	for _, entry := range entries {
		if entry.IsDir() {
			dirs = append(dirs, filepath.Join(fpl.basePath, entry.Name()))
		}
	}

	return dirs, nil
}

func (fpl *FilePersistenceLayer) getTopicFiles(topicDir string) ([]string, error) {
	entries, err := ioutil.ReadDir(topicDir)
	if err != nil {
		return nil, err
	}

	var files []string
	for _, entry := range entries {
		if !entry.IsDir() && filepath.Ext(entry.Name()) == ".dat" {
			files = append(files, filepath.Join(topicDir, entry.Name()))
		}
	}

	return files, nil
}

func (fpl *FilePersistenceLayer) findMessageInFile(filename, messageID string) (*Message, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}

	// Parse messages from file data
	messages, err := fpl.parseFileData(data)
	if err != nil {
		return nil, err
	}

	for _, msg := range messages {
		if msg.ID == messageID {
			return msg, nil
		}
	}

	return nil, nil
}

func (fpl *FilePersistenceLayer) readMessagesFromFile(filename string) ([]*Message, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}

	return fpl.parseFileData(data)
}

func (fpl *FilePersistenceLayer) parseFileData(data []byte) ([]*Message, error) {
	var messages []*Message
	offset := 0

	for offset < len(data) {
		// Read length prefix
		lengthEnd := -1
		for i := offset; i < len(data); i++ {
			if data[i] == '\n' {
				lengthEnd = i
				break
			}
		}

		if lengthEnd == -1 {
			break // No more complete messages
		}

		lengthStr := string(data[offset:lengthEnd])
		var messageLength int
		if _, err := fmt.Sscanf(lengthStr, "%d", &messageLength); err != nil {
			break // Invalid length prefix
		}

		messageStart := lengthEnd + 1
		messageEnd := messageStart + messageLength

		if messageEnd > len(data) {
			break // Incomplete message
		}

		messageData := data[messageStart:messageEnd]

		// Apply decompression if needed
		if fpl.compression {
			decompressed, err := fpl.decompress(messageData)
			if err != nil {
				offset = messageEnd
				continue
			}
			messageData = decompressed
		}

		// Apply decryption if needed
		if fpl.encryption.Enabled {
			decrypted, err := fpl.decrypt(messageData)
			if err != nil {
				offset = messageEnd
				continue
			}
			messageData = decrypted
		}

		// Parse message
		var persistedMsg PersistedMessage
		if err := json.Unmarshal(messageData, &persistedMsg); err != nil {
			offset = messageEnd
			continue
		}

		messages = append(messages, persistedMsg.Message)
		offset = messageEnd
	}

	return messages, nil
}

func (fpl *FilePersistenceLayer) isDirectoryEmpty(dir string) (bool, error) {
	entries, err := ioutil.ReadDir(dir)
	if err != nil {
		return false, err
	}
	return len(entries) == 0, nil
}

func (fpl *FilePersistenceLayer) encrypt(data []byte) ([]byte, error) {
	// Placeholder for encryption implementation
	// In a real implementation, you would use proper encryption libraries
	return data, nil
}

func (fpl *FilePersistenceLayer) decrypt(data []byte) ([]byte, error) {
	// Placeholder for decryption implementation
	return data, nil
}

func (fpl *FilePersistenceLayer) compress(data []byte) ([]byte, error) {
	// Placeholder for compression implementation
	// In a real implementation, you would use libraries like gzip
	return data, nil
}

func (fpl *FilePersistenceLayer) decompress(data []byte) ([]byte, error) {
	// Placeholder for decompression implementation
	return data, nil
}

// InMemoryPersistenceLayer implements in-memory persistence for testing/development
type InMemoryPersistenceLayer struct {
	messages map[string]*Message
	topics   map[string][]string
	mu       sync.RWMutex
}

// NewInMemoryPersistenceLayer creates a new in-memory persistence layer
func NewInMemoryPersistenceLayer() *InMemoryPersistenceLayer {
	return &InMemoryPersistenceLayer{
		messages: make(map[string]*Message),
		topics:   make(map[string][]string),
	}
}

// Store stores a message in memory
func (impl *InMemoryPersistenceLayer) Store(msg *Message) error {
	impl.mu.Lock()
	defer impl.mu.Unlock()

	impl.messages[msg.ID] = msg

	if _, exists := impl.topics[msg.Topic]; !exists {
		impl.topics[msg.Topic] = make([]string, 0)
	}
	impl.topics[msg.Topic] = append(impl.topics[msg.Topic], msg.ID)

	return nil
}

// Retrieve retrieves a message from memory
func (impl *InMemoryPersistenceLayer) Retrieve(id string) (*Message, error) {
	impl.mu.RLock()
	defer impl.mu.RUnlock()

	msg, exists := impl.messages[id]
	if !exists {
		return nil, fmt.Errorf("message not found: %s", id)
	}

	return msg, nil
}

// Delete removes a message from memory
func (impl *InMemoryPersistenceLayer) Delete(id string) error {
	impl.mu.Lock()
	defer impl.mu.Unlock()

	msg, exists := impl.messages[id]
	if !exists {
		return fmt.Errorf("message not found: %s", id)
	}

	delete(impl.messages, id)

	// Remove from topic index
	if messageIDs, exists := impl.topics[msg.Topic]; exists {
		for i, msgID := range messageIDs {
			if msgID == id {
				impl.topics[msg.Topic] = append(messageIDs[:i], messageIDs[i+1:]...)
				break
			}
		}
	}

	return nil
}

// List returns messages for a topic
func (impl *InMemoryPersistenceLayer) List(topic string, limit int) ([]*Message, error) {
	impl.mu.RLock()
	defer impl.mu.RUnlock()

	messageIDs, exists := impl.topics[topic]
	if !exists {
		return []*Message{}, nil
	}

	var messages []*Message
	count := 0

	for _, msgID := range messageIDs {
		if limit > 0 && count >= limit {
			break
		}

		if msg, exists := impl.messages[msgID]; exists {
			messages = append(messages, msg)
			count++
		}
	}

	return messages, nil
}

// Cleanup removes messages older than maxAge
func (impl *InMemoryPersistenceLayer) Cleanup(maxAge time.Duration) error {
	impl.mu.Lock()
	defer impl.mu.Unlock()

	cutoff := time.Now().Add(-maxAge)
	var toDelete []string

	for id, msg := range impl.messages {
		if msg.Timestamp.Before(cutoff) {
			toDelete = append(toDelete, id)
		}
	}

	for _, id := range toDelete {
		impl.Delete(id)
	}

	return nil
}