// Package s3store wraps object storage (Ceph RGW via the S3 API) behind an // interface so the archiver and CLI can be tested with a fake. Credentials come // from the standard AWS_* environment (the cephrgw BucketAccess secret // logs-archive-s3); this package only wires the custom endpoint, path-style // addressing, and the internal Vault-PKI CA needed for s3.ceph.unkin.net. package s3store import ( "context" "crypto/tls" "crypto/x509" "fmt" "io" "net/http" "os" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" ) // ObjectStore is the minimal object-storage surface logarchiver needs. type ObjectStore interface { Put(ctx context.Context, key string, body io.Reader, size int64) error Get(ctx context.Context, key string) (io.ReadCloser, error) List(ctx context.Context, prefix string) ([]string, error) Bucket() string } // Config configures the S3 client. type Config struct { Endpoint string Bucket string Region string PathStyle bool CAFile string } // Store is the S3-backed ObjectStore. type Store struct { client *s3.Client bucket string } // New builds a Store, trusting CAFile (in addition to the system roots) when set. func New(ctx context.Context, cfg Config) (*Store, error) { httpClient, err := httpClientWithCA(cfg.CAFile) if err != nil { return nil, err } region := cfg.Region if region == "" { region = "us-east-1" } awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region), awsconfig.WithHTTPClient(httpClient), ) if err != nil { return nil, fmt.Errorf("load aws config: %w", err) } client := s3.NewFromConfig(awsCfg, func(o *s3.Options) { if cfg.Endpoint != "" { o.BaseEndpoint = &cfg.Endpoint } o.UsePathStyle = cfg.PathStyle }) return &Store{client: client, bucket: cfg.Bucket}, nil } // Bucket returns the configured bucket name. func (s *Store) Bucket() string { return s.bucket } // Put uploads body of the given size under key. func (s *Store) Put(ctx context.Context, key string, body io.Reader, size int64) error { _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ Bucket: &s.bucket, Key: &key, Body: body, ContentLength: &size, }) if err != nil { return fmt.Errorf("put s3://%s/%s: %w", s.bucket, key, err) } return nil } // Get streams the object at key. func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) { out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: &s.bucket, Key: &key, }) if err != nil { return nil, fmt.Errorf("get s3://%s/%s: %w", s.bucket, key, err) } return out.Body, nil } // List returns object keys under prefix (paginated). func (s *Store) List(ctx context.Context, prefix string) ([]string, error) { var keys []string p := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{ Bucket: &s.bucket, Prefix: &prefix, }) for p.HasMorePages() { page, err := p.NextPage(ctx) if err != nil { return nil, fmt.Errorf("list s3://%s/%s: %w", s.bucket, prefix, err) } for _, obj := range page.Contents { if obj.Key != nil { keys = append(keys, *obj.Key) } } } return keys, nil } func httpClientWithCA(caFile string) (*http.Client, error) { if caFile == "" { return http.DefaultClient, nil } pem, err := os.ReadFile(caFile) if err != nil { return nil, fmt.Errorf("read s3 ca file %s: %w", caFile, err) } pool, err := x509.SystemCertPool() if err != nil || pool == nil { pool = x509.NewCertPool() } if !pool.AppendCertsFromPEM(pem) { return nil, fmt.Errorf("no certificates parsed from %s", caFile) } return &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, }, }, nil }