logarchiver/internal/config/config.go

// Package config defines logarchiver's configuration and loads it from a YAML
// file with environment-variable overrides, so the same binary is
// k8s-friendly (env/secret-driven) and laptop-friendly (a config file).
//
// Precedence: built-in defaults < YAML file < environment variables.
package config

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"gopkg.in/yaml.v3"
)

// Config is the full logarchiver configuration.
type Config struct {
	NATS    NATSConfig    `yaml:"nats"`
	Batch   BatchConfig   `yaml:"batch"`
	S3      S3Config      `yaml:"s3"`
	Crypto  CryptoConfig  `yaml:"crypto"`
	Index   IndexConfig   `yaml:"index"`
	Metrics MetricsConfig `yaml:"metrics"`
	Log     LogConfig     `yaml:"log"`
}

// NATSConfig configures the JetStream pull consumer that logarchiver binds. It
// mirrors the logging stack's `LOGS` stream / `archiver` durable / `log-consumer`
// user conventions (argocd-apps #296).
type NATSConfig struct {
	URL      string   `yaml:"url"`
	Stream   string   `yaml:"stream"`
	Durable  string   `yaml:"durable"`
	Subjects []string `yaml:"subjects"`
	User     string   `yaml:"user"`
	// Password is the NATS user password. In-cluster it comes from the
	// nats-auth secret via NATS_CONSUMER_PASSWORD (see PasswordEnv).
	Password string `yaml:"password"`
	// PasswordEnv names the env var holding the password when Password is empty.
	PasswordEnv string `yaml:"password_env"`
	// CAFile trusts a custom CA for TLS to NATS (usually unset; in-cluster is plaintext).
	CAFile string `yaml:"ca_file"`
	// FetchBatch is the max messages pulled per Fetch call.
	FetchBatch int `yaml:"fetch_batch"`
	// AckWait is the JetStream redelivery timeout; must exceed a worst-case
	// batch flush (compress+encrypt+S3 PUT+index write).
	AckWait time.Duration `yaml:"ack_wait"`
}

// BatchConfig bounds a single archived object. A per-subject batch is flushed
// when any bound is hit. Keep MaxBytes well under the crypto/engine ceiling so
// even a whole-object decrypt path stays viable; the wrapped-DEK envelope means
// object size is not limited by Vault, but smaller objects retrieve faster.
type BatchConfig struct {
	MaxBytes  int64         `yaml:"max_bytes"`
	MaxEvents int           `yaml:"max_events"`
	MaxAge    time.Duration `yaml:"max_age"`
}

// S3Config targets the Ceph RGW bucket. Credentials are read from the standard
// AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars (cephrgw BucketAccess
// secret logs-archive-s3), so they are intentionally absent here.
type S3Config struct {
	Endpoint  string `yaml:"endpoint"`
	Bucket    string `yaml:"bucket"`
	Region    string `yaml:"region"`
	PathStyle bool   `yaml:"path_style"`
	// KeyPrefix is a text/template with {{.Subject}} {{.Year}} {{.Month}} {{.Day}}.
	KeyPrefix string `yaml:"key_prefix"`
	// CAFile trusts the internal Vault-PKI CA for https://s3.ceph.unkin.net.
	CAFile string `yaml:"ca_file"`
	// EndpointEnv / BucketEnv let the cephrgw secret (S3_ENDPOINT / BUCKET_NAME)
	// override endpoint/bucket without a config edit.
	EndpointEnv string `yaml:"endpoint_env"`
	BucketEnv   string `yaml:"bucket_env"`
}

// PubkeySource selects where the OpenPGP public key is fetched from.
type PubkeySource string

const (
	PubkeyVault PubkeySource = "vault" // read gpg/keys/<name> from the Vault GPG engine
	PubkeyFile  PubkeySource = "file"  // read an armored public key from a mounted file
)

// CryptoConfig controls encryption. The service only ever needs the PUBLIC key;
// decryption (CLI fetch) always goes through the Vault GPG engine.
type CryptoConfig struct {
	KeyName string       `yaml:"key_name"`
	Source  PubkeySource `yaml:"pubkey_source"`
	// PubkeyFile is the armored public key path when Source==file.
	PubkeyFile string `yaml:"pubkey_file"`
	// RefreshInterval re-fetches the public key periodically (rotation aware).
	RefreshInterval time.Duration `yaml:"refresh_interval"`
	// FrameSize is the AES-GCM frame plaintext size in bytes (streaming decrypt).
	FrameSize int         `yaml:"frame_size"`
	Vault     VaultConfig `yaml:"vault"`
}

// VaultConfig configures access to the Vault GPG secrets engine. For the
// service (pubkey fetch) k8s auth is used in-cluster; the CLI relies on the
// operator's ambient VAULT_TOKEN (~/.vault-token), like passv.
type VaultConfig struct {
	Address string `yaml:"address"`
	// Mount is the GPG engine mount path (e.g. "gpg").
	Mount string `yaml:"mount"`
	// AuthMethod is "token" or "kubernetes".
	AuthMethod string `yaml:"auth_method"`
	// K8sRole / K8sMount / K8sJWTPath configure kubernetes auth.
	K8sRole    string `yaml:"k8s_role"`
	K8sMount   string `yaml:"k8s_mount"`
	K8sJWTPath string `yaml:"k8s_jwt_path"`
	CAFile     string `yaml:"ca_file"`
}

// IndexConfig targets the ClickHouse archive index. Credentials come from the
// clickhouse-credentials secret via env by default.
type IndexConfig struct {
	Enabled     bool   `yaml:"enabled"`
	Address     string `yaml:"address"` // host:port for the native protocol (9000)
	Database    string `yaml:"database"`
	Table       string `yaml:"table"`
	Username    string `yaml:"username"`
	Password    string `yaml:"password"`
	PasswordEnv string `yaml:"password_env"`
	TLS         bool   `yaml:"tls"`
}

// MetricsConfig configures the Prometheus /metrics listener.
type MetricsConfig struct {
	Enabled bool   `yaml:"enabled"`
	Address string `yaml:"address"`
}

// LogConfig configures structured logging.
type LogConfig struct {
	Level  string `yaml:"level"`  // debug|info|warn|error
	Format string `yaml:"format"` // json|text
}

// Default returns a Config pre-populated with the logging-stack conventions so
// an in-cluster deployment needs only secrets (creds) supplied via env.
func Default() Config {
	return Config{
		NATS: NATSConfig{
			URL:         "nats://nats.logging.svc.cluster.local:4222",
			Stream:      "LOGS",
			Durable:     "archiver",
			Subjects:    []string{"logs.k8s.vault.>"},
			User:        "log-consumer",
			PasswordEnv: "NATS_CONSUMER_PASSWORD",
			FetchBatch:  512,
			AckWait:     2 * time.Minute,
		},
		Batch: BatchConfig{
			MaxBytes:  64 * 1024 * 1024, // 64 MiB raw NDJSON per object
			MaxEvents: 200000,
			MaxAge:    5 * time.Minute,
		},
		S3: S3Config{
			Endpoint:    "https://s3.ceph.unkin.net",
			Bucket:      "logs-archive",
			Region:      "us-east-1",
			PathStyle:   true,
			KeyPrefix:   "archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/",
			CAFile:      "/etc/vault-ca/ca.crt",
			EndpointEnv: "S3_ENDPOINT",
			BucketEnv:   "BUCKET_NAME",
		},
		Crypto: CryptoConfig{
			KeyName:         "logarchive",
			Source:          PubkeyFile,
			PubkeyFile:      "/etc/logarchiver/pubkey.asc",
			RefreshInterval: time.Hour,
			FrameSize:       1 << 20, // 1 MiB frames
			Vault: VaultConfig{
				Mount:      "gpg",
				AuthMethod: "kubernetes",
				K8sMount:   "k8s/au/syd1",
				K8sRole:    "default",
				K8sJWTPath: "/var/run/secrets/kubernetes.io/serviceaccount/token",
			},
		},
		Index: IndexConfig{
			Enabled:     true,
			Address:     "clickhouse-logs.logging.svc.cluster.local:9000",
			Database:    "logs",
			Table:       "archive_index",
			Username:    "vector",
			PasswordEnv: "CLICKHOUSE_PASSWORD",
			TLS:         false,
		},
		Metrics: MetricsConfig{Enabled: true, Address: ":9090"},
		Log:     LogConfig{Level: "info", Format: "json"},
	}
}

// Load reads defaults, overlays the YAML file at path (if non-empty), then
// applies environment overrides, and validates the result.
func Load(path string) (Config, error) {
	cfg := Default()
	if path != "" {
		data, err := os.ReadFile(path)
		if err != nil {
			return Config{}, fmt.Errorf("read config %s: %w", path, err)
		}
		if err := yaml.Unmarshal(data, &cfg); err != nil {
			return Config{}, fmt.Errorf("parse config %s: %w", path, err)
		}
	}
	cfg.applyEnv()
	cfg.resolveSecretEnvs()
	if err := cfg.Validate(); err != nil {
		return Config{}, err
	}
	return cfg, nil
}

// applyEnv overlays scalar overrides from the environment. Only the knobs an
// operator commonly flips are wired; secrets are handled by resolveSecretEnvs.
func (c *Config) applyEnv() {
	if v := os.Getenv("LOGARCHIVER_NATS_URL"); v != "" {
		c.NATS.URL = v
	}
	if v := os.Getenv("LOGARCHIVER_NATS_DURABLE"); v != "" {
		c.NATS.Durable = v
	}
	if v := os.Getenv("ARCHIVE_SUBJECTS"); v != "" {
		c.NATS.Subjects = splitFields(v)
	}
	if v := os.Getenv("LOGARCHIVER_S3_ENDPOINT"); v != "" {
		c.S3.Endpoint = v
	}
	if v := os.Getenv("LOGARCHIVER_S3_BUCKET"); v != "" {
		c.S3.Bucket = v
	}
	if v := os.Getenv("LOGARCHIVER_KEY_NAME"); v != "" {
		c.Crypto.KeyName = v
	}
	if v := os.Getenv("LOGARCHIVER_PUBKEY_SOURCE"); v != "" {
		c.Crypto.Source = PubkeySource(v)
	}
	if v := os.Getenv("LOGARCHIVER_PUBKEY_FILE"); v != "" {
		c.Crypto.PubkeyFile = v
	}
	if v := os.Getenv("VAULT_ADDR"); v != "" && c.Crypto.Vault.Address == "" {
		c.Crypto.Vault.Address = v
	}
	if v := os.Getenv("LOGARCHIVER_VAULT_MOUNT"); v != "" {
		c.Crypto.Vault.Mount = v
	}
	if v := os.Getenv("LOGARCHIVER_CLICKHOUSE_ADDR"); v != "" {
		c.Index.Address = v
	}
	if v := os.Getenv("CLICKHOUSE_USER"); v != "" {
		c.Index.Username = v
	}
	if v := os.Getenv("LOGARCHIVER_METRICS_ADDR"); v != "" {
		c.Metrics.Address = v
	}
	if v := os.Getenv("LOGARCHIVER_LOG_LEVEL"); v != "" {
		c.Log.Level = v
	}
	if v := os.Getenv("LOGARCHIVER_LOG_FORMAT"); v != "" {
		c.Log.Format = v
	}
	// Endpoint/bucket sourced from the cephrgw secret, if present.
	if c.S3.EndpointEnv != "" {
		if v := os.Getenv(c.S3.EndpointEnv); v != "" {
			c.S3.Endpoint = v
		}
	}
	if c.S3.BucketEnv != "" {
		if v := os.Getenv(c.S3.BucketEnv); v != "" {
			c.S3.Bucket = v
		}
	}
}

// resolveSecretEnvs pulls passwords from their named env vars when not set inline.
func (c *Config) resolveSecretEnvs() {
	if c.NATS.Password == "" && c.NATS.PasswordEnv != "" {
		c.NATS.Password = os.Getenv(c.NATS.PasswordEnv)
	}
	if c.Index.Password == "" && c.Index.PasswordEnv != "" {
		c.Index.Password = os.Getenv(c.Index.PasswordEnv)
	}
}

// Validate checks required fields and coherence.
func (c *Config) Validate() error {
	if c.NATS.URL == "" {
		return fmt.Errorf("nats.url is required")
	}
	if c.NATS.Stream == "" {
		return fmt.Errorf("nats.stream is required")
	}
	if c.NATS.Durable == "" {
		return fmt.Errorf("nats.durable is required")
	}
	if len(c.NATS.Subjects) == 0 {
		return fmt.Errorf("nats.subjects must list at least one filter subject")
	}
	if c.S3.Bucket == "" {
		return fmt.Errorf("s3.bucket is required")
	}
	if c.S3.Endpoint == "" {
		return fmt.Errorf("s3.endpoint is required")
	}
	if c.Crypto.KeyName == "" {
		return fmt.Errorf("crypto.key_name is required")
	}
	switch c.Crypto.Source {
	case PubkeyVault:
		if c.Crypto.Vault.Address == "" {
			return fmt.Errorf("crypto.vault.address is required when pubkey_source=vault")
		}
		if c.Crypto.Vault.Mount == "" {
			return fmt.Errorf("crypto.vault.mount is required when pubkey_source=vault")
		}
	case PubkeyFile:
		if c.Crypto.PubkeyFile == "" {
			return fmt.Errorf("crypto.pubkey_file is required when pubkey_source=file")
		}
	default:
		return fmt.Errorf("crypto.pubkey_source must be 'vault' or 'file', got %q", c.Crypto.Source)
	}
	if c.Crypto.FrameSize <= 0 {
		return fmt.Errorf("crypto.frame_size must be positive")
	}
	if c.Batch.MaxBytes <= 0 && c.Batch.MaxEvents <= 0 && c.Batch.MaxAge <= 0 {
		return fmt.Errorf("batch must set at least one of max_bytes/max_events/max_age")
	}
	if c.Index.Enabled && c.Index.Address == "" {
		return fmt.Errorf("index.address is required when index.enabled")
	}
	return nil
}

func splitFields(s string) []string {
	var out []string
	for _, f := range strings.Fields(s) {
		if f != "" {
			out = append(out, f)
		}
	}
	return out
}

// ParseSize parses a byte size like "64Mi", "128MB", "1024". It is a helper for
// CLI flags; the YAML fields are plain integers.
func ParseSize(s string) (int64, error) {
	s = strings.TrimSpace(s)
	if s == "" {
		return 0, fmt.Errorf("empty size")
	}
	mult := int64(1)
	switch {
	case strings.HasSuffix(s, "Gi"):
		mult, s = 1<<30, strings.TrimSuffix(s, "Gi")
	case strings.HasSuffix(s, "Mi"):
		mult, s = 1<<20, strings.TrimSuffix(s, "Mi")
	case strings.HasSuffix(s, "Ki"):
		mult, s = 1<<10, strings.TrimSuffix(s, "Ki")
	case strings.HasSuffix(s, "GB"):
		mult, s = 1e9, strings.TrimSuffix(s, "GB")
	case strings.HasSuffix(s, "MB"):
		mult, s = 1e6, strings.TrimSuffix(s, "MB")
	case strings.HasSuffix(s, "KB"):
		mult, s = 1e3, strings.TrimSuffix(s, "KB")
	}
	n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
	if err != nil {
		return 0, fmt.Errorf("parse size %q: %w", s, err)
	}
	return n * mult, nil
}