fix(multicall): resolve critical multicall parsing corruption issues
- Added comprehensive bounds checking to prevent buffer overruns in multicall parsing - Implemented graduated validation system (Strict/Moderate/Permissive) to reduce false positives - Added LRU caching system for address validation with 10-minute TTL - Enhanced ABI decoder with missing Universal Router and Arbitrum-specific DEX signatures - Fixed duplicate function declarations and import conflicts across multiple files - Added error recovery mechanisms with multiple fallback strategies - Updated tests to handle new validation behavior for suspicious addresses - Fixed parser test expectations for improved validation system - Applied gofmt formatting fixes to ensure code style compliance - Fixed mutex copying issues in monitoring package by introducing MetricsSnapshot - Resolved critical security vulnerabilities in heuristic address extraction - Progress: Updated TODO audit from 10% to 35% complete 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Krypto Kajun
321
harness/local-ci-pipeline.sh
Executable file
321
harness/local-ci-pipeline.sh
Executable file
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
HARNESS_DIR="$ROOT_DIR/harness"
|
||||
LOG_DIR="${HARNESS_LOG_DIR:-$HARNESS_DIR/logs}"
|
||||
REPORT_DIR="${HARNESS_REPORT_DIR:-$HARNESS_DIR/reports}"
|
||||
mkdir -p "$LOG_DIR" "$REPORT_DIR"
|
||||
|
||||
export GOCACHE="${HARNESS_GOCACHE:-$ROOT_DIR/.gocache}"
|
||||
export GOMODCACHE="${HARNESS_GOMODCACHE:-$ROOT_DIR/.gomodcache}"
|
||||
mkdir -p "$GOCACHE" "$GOMODCACHE"
|
||||
|
||||
CONTAINER_RUNTIME="${HARNESS_RUNTIME:-}"
|
||||
if [[ -z "$CONTAINER_RUNTIME" ]]; then
|
||||
if command -v podman >/dev/null 2>&1; then
|
||||
CONTAINER_RUNTIME=podman
|
||||
elif command -v docker >/dev/null 2>&1; then
|
||||
CONTAINER_RUNTIME=docker
|
||||
else
|
||||
echo "WARNING: No container runtime found. Some tests will be skipped." >&2
|
||||
CONTAINER_RUNTIME=""
|
||||
fi
|
||||
fi
|
||||
|
||||
SKIP_DOCKER="${HARNESS_SKIP_DOCKER:-false}"
|
||||
SKIP_MATH_AUDIT="${HARNESS_SKIP_MATH_AUDIT:-false}"
|
||||
SKIP_SECURITY="${HARNESS_SKIP_SECURITY:-false}"
|
||||
PARALLEL_JOBS="${HARNESS_PARALLEL_JOBS:-4}"
|
||||
|
||||
GO_IMAGE="${HARNESS_GO_IMAGE:-golang:1.25-alpine}"
|
||||
GOLANGCI_IMAGE="${HARNESS_GOLANGCI_IMAGE:-golangci/golangci-lint:v1.60.1}"
|
||||
|
||||
log() {
|
||||
printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"
|
||||
}
|
||||
|
||||
run_step() {
|
||||
local name="$1"
|
||||
shift
|
||||
local logfile="$LOG_DIR/${name}.log"
|
||||
log "Starting $name"
|
||||
if "$@" |& tee "$logfile"; then
|
||||
log "✅ Completed $name"
|
||||
return 0
|
||||
else
|
||||
log "❌ Failed $name - see $logfile"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_requirements() {
|
||||
log "Checking requirements..."
|
||||
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
echo "ERROR: git is required" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v go >/dev/null 2>&1; then
|
||||
echo "ERROR: Go is required" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
GO_VERSION=$(go version | grep -o 'go[0-9.]*' | head -1)
|
||||
log "Using Go version: $GO_VERSION"
|
||||
|
||||
if [[ -n "$CONTAINER_RUNTIME" ]]; then
|
||||
log "Using container runtime: $CONTAINER_RUNTIME"
|
||||
fi
|
||||
}
|
||||
|
||||
setup_directories() {
|
||||
log "Setting up directories..."
|
||||
mkdir -p logs reports/math/latest reports/security reports/coverage
|
||||
}
|
||||
|
||||
run_go_deps() {
|
||||
run_step "go-deps" go mod download
|
||||
run_step "go-verify" go mod verify
|
||||
run_step "go-tidy-check" bash -c "
|
||||
cp go.mod go.mod.bak
|
||||
cp go.sum go.sum.bak
|
||||
go mod tidy
|
||||
if ! diff -q go.mod go.mod.bak >/dev/null || ! diff -q go.sum go.sum.bak >/dev/null; then
|
||||
echo 'ERROR: go.mod or go.sum not tidy - run go mod tidy'
|
||||
mv go.mod.bak go.mod
|
||||
mv go.sum.bak go.sum
|
||||
exit 1
|
||||
fi
|
||||
rm go.mod.bak go.sum.bak
|
||||
"
|
||||
}
|
||||
|
||||
run_formatting() {
|
||||
run_step "go-fmt-check" bash -c "
|
||||
fmt_out=\$(gofmt -l \$(find . -name '*.go' -not -path './vendor/*' -not -path './.gomodcache/*'))
|
||||
if [[ -n \"\$fmt_out\" ]]; then
|
||||
echo 'ERROR: Following files need gofmt:'
|
||||
echo \"\$fmt_out\"
|
||||
exit 1
|
||||
fi
|
||||
"
|
||||
}
|
||||
|
||||
run_static_checks() {
|
||||
run_step "go-vet" go vet ./...
|
||||
|
||||
if command -v golangci-lint >/dev/null 2>&1; then
|
||||
run_step "golangci-lint" timeout 300 golangci-lint run --timeout=10m
|
||||
elif [[ -n "$CONTAINER_RUNTIME" && "$SKIP_DOCKER" != "true" ]]; then
|
||||
run_step "golangci-lint-container" \
|
||||
"$CONTAINER_RUNTIME" run --rm \
|
||||
-v "$ROOT_DIR":/app -w /app \
|
||||
-v "$GOCACHE":/gocache -e GOCACHE=/gocache \
|
||||
"$GOLANGCI_IMAGE" golangci-lint run --timeout=10m
|
||||
else
|
||||
log "WARNING: Skipping golangci-lint (not available)"
|
||||
fi
|
||||
}
|
||||
|
||||
run_tests() {
|
||||
log "Creating logs directory for tests..."
|
||||
mkdir -p logs
|
||||
|
||||
run_step "unit-tests" \
|
||||
go test -race -coverprofile="$REPORT_DIR/coverage.out" \
|
||||
-timeout=300s \
|
||||
./pkg/types ./pkg/arbitrage ./pkg/execution ./pkg/arbitrum ./pkg/math ./internal/...
|
||||
|
||||
if [[ -f "$REPORT_DIR/coverage.out" ]]; then
|
||||
run_step "coverage-report" \
|
||||
go tool cover -html="$REPORT_DIR/coverage.out" -o "$REPORT_DIR/coverage.html"
|
||||
|
||||
COVERAGE=$(go tool cover -func="$REPORT_DIR/coverage.out" | tail -1 | awk '{print $3}')
|
||||
log "Test coverage: $COVERAGE"
|
||||
echo "Test coverage: $COVERAGE" > "$REPORT_DIR/coverage-summary.txt"
|
||||
fi
|
||||
}
|
||||
|
||||
run_build() {
|
||||
run_step "build-binary" make build
|
||||
|
||||
if [[ -f bin/mev-bot ]]; then
|
||||
log "✅ Binary built successfully: $(ls -lh bin/mev-bot | awk '{print $5}')"
|
||||
else
|
||||
log "❌ Binary not found after build"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_smoke_test() {
|
||||
export MEV_BOT_ENCRYPTION_KEY="test_key_32_chars_minimum_length_required"
|
||||
run_step "smoke-test" timeout 10s ./bin/mev-bot start || {
|
||||
if [[ $? -eq 124 ]]; then
|
||||
log "✅ Smoke test passed (timeout expected)"
|
||||
return 0
|
||||
else
|
||||
log "❌ Smoke test failed"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
}
|
||||
|
||||
run_math_audit() {
|
||||
if [[ "$SKIP_MATH_AUDIT" == "true" ]]; then
|
||||
log "Skipping math audit (HARNESS_SKIP_MATH_AUDIT=true)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
run_step "math-audit" \
|
||||
go run ./tools/math-audit --vectors default --report reports/math/latest
|
||||
|
||||
if [[ -f reports/math/latest/report.json && -f reports/math/latest/report.md ]]; then
|
||||
log "✅ Math audit completed successfully"
|
||||
cp reports/math/latest/report.md "$REPORT_DIR/math-audit.md"
|
||||
else
|
||||
log "❌ Math audit failed - reports not generated"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_security_checks() {
|
||||
if [[ "$SKIP_SECURITY" == "true" ]]; then
|
||||
log "Skipping security checks (HARNESS_SKIP_SECURITY=true)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if command -v gosec >/dev/null 2>&1; then
|
||||
log "Running gosec security scan (with timeout)..."
|
||||
timeout 60 gosec -fmt sarif -out "$REPORT_DIR/gosec-results.sarif" ./... || {
|
||||
if [[ $? -eq 124 ]]; then
|
||||
log "WARNING: gosec timed out after 60s"
|
||||
else
|
||||
log "WARNING: gosec completed with warnings"
|
||||
fi
|
||||
}
|
||||
else
|
||||
log "WARNING: gosec not available"
|
||||
fi
|
||||
|
||||
if command -v govulncheck >/dev/null 2>&1; then
|
||||
run_step "govulncheck" govulncheck ./...
|
||||
else
|
||||
log "WARNING: govulncheck not available"
|
||||
fi
|
||||
}
|
||||
|
||||
run_docker_build() {
|
||||
if [[ "$SKIP_DOCKER" == "true" || -z "$CONTAINER_RUNTIME" ]]; then
|
||||
log "Skipping Docker build"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Building Docker image with $CONTAINER_RUNTIME..."
|
||||
if timeout 300 "$CONTAINER_RUNTIME" build -t mev-bot:harness-test . &> "$LOG_DIR/docker-build.log"; then
|
||||
log "✅ Docker build successful"
|
||||
|
||||
IMAGE_SIZE=$("$CONTAINER_RUNTIME" images mev-bot:harness-test --format "table {{.Size}}" | tail -1)
|
||||
log "Docker image size: $IMAGE_SIZE"
|
||||
echo "Docker image size: $IMAGE_SIZE" > "$REPORT_DIR/docker-image-size.txt"
|
||||
else
|
||||
log "❌ Docker build failed - see $LOG_DIR/docker-build.log"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
generate_report() {
|
||||
log "Generating final report..."
|
||||
|
||||
REPORT_FILE="$REPORT_DIR/pipeline-report.md"
|
||||
cat > "$REPORT_FILE" << EOF
|
||||
# MEV Bot Local CI Pipeline Report
|
||||
|
||||
**Generated:** $(date -u '+%Y-%m-%d %H:%M:%S UTC')
|
||||
**Branch:** $(git rev-parse --abbrev-ref HEAD)
|
||||
**Commit:** $(git rev-parse --short HEAD)
|
||||
**Go Version:** $(go version)
|
||||
|
||||
## Pipeline Results
|
||||
|
||||
### ✅ Completed Steps
|
||||
EOF
|
||||
|
||||
for logfile in "$LOG_DIR"/*.log; do
|
||||
if [[ -f "$logfile" ]]; then
|
||||
step=$(basename "$logfile" .log)
|
||||
echo "- $step" >> "$REPORT_FILE"
|
||||
fi
|
||||
done
|
||||
|
||||
cat >> "$REPORT_FILE" << EOF
|
||||
|
||||
### 📊 Metrics
|
||||
EOF
|
||||
|
||||
if [[ -f "$REPORT_DIR/coverage-summary.txt" ]]; then
|
||||
echo "- $(cat "$REPORT_DIR/coverage-summary.txt")" >> "$REPORT_FILE"
|
||||
fi
|
||||
|
||||
if [[ -f "$REPORT_DIR/docker-image-size.txt" ]]; then
|
||||
echo "- $(cat "$REPORT_DIR/docker-image-size.txt")" >> "$REPORT_FILE"
|
||||
fi
|
||||
|
||||
cat >> "$REPORT_FILE" << EOF
|
||||
|
||||
### 📁 Generated Artifacts
|
||||
- Coverage report: \`$REPORT_DIR/coverage.html\`
|
||||
- Math audit: \`$REPORT_DIR/math-audit.md\`
|
||||
- Security report: \`$REPORT_DIR/gosec-results.sarif\`
|
||||
- Full logs: \`$LOG_DIR/\`
|
||||
|
||||
### 🔧 Environment
|
||||
- Container runtime: ${CONTAINER_RUNTIME:-"none"}
|
||||
- Parallel jobs: $PARALLEL_JOBS
|
||||
- Go cache: $GOCACHE
|
||||
- Skip flags: docker=$SKIP_DOCKER, math-audit=$SKIP_MATH_AUDIT, security=$SKIP_SECURITY
|
||||
EOF
|
||||
|
||||
log "✅ Report generated: $REPORT_FILE"
|
||||
}
|
||||
|
||||
main() {
|
||||
log "🚀 Starting MEV Bot Local CI Pipeline"
|
||||
log "Working directory: $ROOT_DIR"
|
||||
log "Logs: $LOG_DIR"
|
||||
log "Reports: $REPORT_DIR"
|
||||
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
check_requirements
|
||||
setup_directories
|
||||
|
||||
# Core pipeline steps
|
||||
run_go_deps
|
||||
run_formatting
|
||||
run_static_checks
|
||||
run_tests
|
||||
run_build
|
||||
run_smoke_test
|
||||
|
||||
# Optional steps
|
||||
run_math_audit
|
||||
run_security_checks
|
||||
run_docker_build
|
||||
|
||||
generate_report
|
||||
|
||||
END_TIME=$(date +%s)
|
||||
DURATION=$((END_TIME - START_TIME))
|
||||
|
||||
log "🎉 Pipeline completed successfully in ${DURATION}s"
|
||||
log "📊 View full report: $REPORT_DIR/pipeline-report.md"
|
||||
}
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||||
main "$@"
|
||||
fi
|
||||
139
harness/pipelines/staging.yaml
Normal file
139
harness/pipelines/staging.yaml
Normal file
@@ -0,0 +1,139 @@
|
||||
pipeline:
|
||||
name: Staging Promotion
|
||||
identifier: staging_promotion
|
||||
projectIdentifier: mev_bot
|
||||
orgIdentifier: default
|
||||
tags: {}
|
||||
properties:
|
||||
ci:
|
||||
codebase:
|
||||
connectorRef: account.git
|
||||
repoName: fraktal/mev-beta
|
||||
build:
|
||||
type: branch
|
||||
spec:
|
||||
branch: <+input>
|
||||
stages:
|
||||
- stage:
|
||||
name: Build & Test
|
||||
identifier: build_test
|
||||
description: Run Drone-equivalent checks inside Harness.
|
||||
type: CI
|
||||
spec:
|
||||
cloneCodebase: true
|
||||
platform:
|
||||
os: Linux
|
||||
arch: Amd64
|
||||
execution:
|
||||
steps:
|
||||
- step:
|
||||
type: Run
|
||||
name: Lint
|
||||
identifier: lint
|
||||
spec:
|
||||
shell: Bash
|
||||
command: |
|
||||
go mod download
|
||||
golangci-lint run --timeout=10m
|
||||
- step:
|
||||
type: Run
|
||||
name: UnitTests
|
||||
identifier: unit_tests
|
||||
spec:
|
||||
shell: Bash
|
||||
command: |
|
||||
GOCACHE=$(pwd)/.gocache go test -race -coverprofile=coverage.out ./...
|
||||
test -s coverage.out
|
||||
- step:
|
||||
type: Run
|
||||
name: MathAudit
|
||||
identifier: math_audit
|
||||
spec:
|
||||
shell: Bash
|
||||
command: |
|
||||
GOCACHE=$(pwd)/.gocache go run ./tools/math-audit --vectors default --report reports/math/latest
|
||||
ls -l reports/math/latest
|
||||
- step:
|
||||
type: Run
|
||||
name: ProfitSimulation
|
||||
identifier: profit_simulation
|
||||
spec:
|
||||
shell: Bash
|
||||
command: |
|
||||
./scripts/run_profit_simulation.sh
|
||||
cat reports/simulation/latest/summary.md
|
||||
caching:
|
||||
enabled: true
|
||||
- stage:
|
||||
name: Package
|
||||
identifier: package
|
||||
type: CI
|
||||
spec:
|
||||
cloneCodebase: false
|
||||
platform:
|
||||
os: Linux
|
||||
arch: Amd64
|
||||
execution:
|
||||
steps:
|
||||
- step:
|
||||
type: Run
|
||||
name: DockerBuild
|
||||
identifier: docker_build
|
||||
spec:
|
||||
shell: Bash
|
||||
command: |
|
||||
docker build -t mev-bot:${HARNESS_BUILD_NUMBER} .
|
||||
docker save mev-bot:${HARNESS_BUILD_NUMBER} -o mev-bot.tar
|
||||
- step:
|
||||
type: Run
|
||||
name: PublishArtifact
|
||||
identifier: publish_artifact
|
||||
spec:
|
||||
shell: Bash
|
||||
command: |
|
||||
echo "Uploading mev-bot.tar to artifact store"
|
||||
when:
|
||||
pipelineStatus: Success
|
||||
- stage:
|
||||
name: Deploy to Staging
|
||||
identifier: deploy_staging
|
||||
type: CD
|
||||
spec:
|
||||
deploymentType: Kubernetes
|
||||
service:
|
||||
serviceRef: staging
|
||||
environment:
|
||||
environmentRef: staging
|
||||
execution:
|
||||
steps:
|
||||
- step:
|
||||
type: ShellScript
|
||||
name: HelmUpgrade
|
||||
identifier: helm_upgrade
|
||||
spec:
|
||||
shell: Bash
|
||||
source: Inline
|
||||
script: |
|
||||
helm upgrade --install mev-bot charts/mev-bot \
|
||||
--set image.tag=${HARNESS_BUILD_NUMBER} \
|
||||
--namespace mev-bot-staging
|
||||
- step:
|
||||
type: ShellScript
|
||||
name: PostDeployChecks
|
||||
identifier: post_deploy_checks
|
||||
spec:
|
||||
shell: Bash
|
||||
source: Inline
|
||||
script: |
|
||||
kubectl rollout status deploy/mev-bot -n mev-bot-staging --timeout=120s
|
||||
kubectl logs deploy/mev-bot -n mev-bot-staging | tail -n 100
|
||||
rollbackSteps:
|
||||
- step:
|
||||
type: ShellScript
|
||||
name: Rollback
|
||||
identifier: rollback
|
||||
spec:
|
||||
shell: Bash
|
||||
source: Inline
|
||||
script: |
|
||||
helm rollback mev-bot 1 || true
|
||||
Reference in New Issue
Block a user