unkin-agent
8786636f7c
Four issues from the review of the initial repospawner service, none of which change the shape of a request or the file terraform-git receives. - Encode status checks as one --check flag per context on the server-to-job hop, so a separator inside a context can no longer turn one context into several; ban commas (and cap lengths) in Validate as well, since a real context never holds one. - Fail a merged request that has waited five minutes for a Woodpecker token that vanished after acceptance, surfacing "woodpecker token unavailable" through the API, instead of warning in the log forever from enabling-ci. Advance now leaves a terminal request alone so the failure sticks. - Hold a per-name lock from the duplicate checks through the store write, so two concurrent submissions of one name cannot both be accepted. - Cap the description at 500 characters and the status checks at 20 contexts of 100 characters each, and mirror the first two caps in the form.
292 lines
9.6 KiB
Go
292 lines
9.6 KiB
Go
// Package jobs builds the Kubernetes Jobs that carry out a request and reads
|
|
// their results back. Every job runs repospawner's own image with a different
|
|
// argv, so a job never needs a second image to keep in step.
|
|
package jobs
|
|
|
|
import (
|
|
"encoding/json"
|
|
"path"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
batchv1 "k8s.io/api/batch/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
|
|
"git.unkin.net/unkin/repospawner/internal/config"
|
|
"git.unkin.net/unkin/repospawner/internal/store"
|
|
)
|
|
|
|
// Type is the kind of work a Job performs.
|
|
type Type string
|
|
|
|
const (
|
|
// TypePR opens the terraform-git pull request.
|
|
TypePR Type = "pr"
|
|
// TypeWatch follows that pull request to merge or close.
|
|
TypeWatch Type = "watch"
|
|
// TypeWoodpecker enables the new repository in Woodpecker.
|
|
TypeWoodpecker Type = "woodpecker"
|
|
)
|
|
|
|
// Label and annotation keys. The request label is the join key for both Jobs
|
|
// and their Pods; the annotations carry enough of the request to rebuild the
|
|
// store after a restart.
|
|
const (
|
|
LabelRequest = "repospawner.unkin.net/request"
|
|
LabelType = "repospawner.unkin.net/type"
|
|
LabelApp = "app.kubernetes.io/name"
|
|
AppName = "repospawner"
|
|
AnnoName = "repospawner.unkin.net/name"
|
|
AnnoDescription = "repospawner.unkin.net/description"
|
|
AnnoWoodpecker = "repospawner.unkin.net/woodpecker"
|
|
AnnoStatusChecks = "repospawner.unkin.net/status-checks"
|
|
AnnoCreated = "repospawner.unkin.net/created"
|
|
AnnoPullRequestNo = "repospawner.unkin.net/pr-number"
|
|
AnnoPullRequest = "repospawner.unkin.net/pr-url"
|
|
)
|
|
|
|
const (
|
|
ttlSecondsAfterFinished = int32(3600)
|
|
backoffLimit = int32(2)
|
|
// watchDeadline is generous: a terraform-git PR waits on a human.
|
|
watchDeadline = int64(7 * 24 * 60 * 60)
|
|
shortDeadline = int64(15 * 60)
|
|
)
|
|
|
|
// PRResult is written by the pr job to its termination message.
|
|
type PRResult struct {
|
|
PRNumber int `json:"pr_number"`
|
|
PRURL string `json:"pr_url"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// WatchResult is written by the watch job to its termination message.
|
|
type WatchResult struct {
|
|
Merged bool `json:"merged"`
|
|
Closed bool `json:"closed"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// WoodpeckerResult is written by the woodpecker job to its termination message.
|
|
type WoodpeckerResult struct {
|
|
Enabled bool `json:"enabled"`
|
|
RepoID int64 `json:"repo_id,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// Name is the Job object name for a request and type.
|
|
func Name(t Type, id string) string { return "repospawner-" + string(t) + "-" + id }
|
|
|
|
// PR builds the Job that opens the terraform-git pull request.
|
|
func PR(cfg *config.Config, r store.Request) *batchv1.Job {
|
|
args := []string{
|
|
"job", "pr",
|
|
"--request", r.ID,
|
|
"--name", r.Name,
|
|
"--description", r.Description,
|
|
}
|
|
// One flag per context: a separator inside a check would otherwise turn one
|
|
// context into several on the way back out.
|
|
for _, c := range r.StatusChecks {
|
|
args = append(args, "--check", c)
|
|
}
|
|
return base(cfg, r, TypePR, args, shortDeadline)
|
|
}
|
|
|
|
// Watch builds the Job that follows the pull request to its conclusion.
|
|
func Watch(cfg *config.Config, r store.Request) *batchv1.Job {
|
|
args := []string{"job", "watch", "--repo", cfg.TFGitRepo, "--pr", strconv.Itoa(r.PRNumber)}
|
|
j := base(cfg, r, TypeWatch, args, watchDeadline)
|
|
j.Annotations[AnnoPullRequestNo] = strconv.Itoa(r.PRNumber)
|
|
j.Annotations[AnnoPullRequest] = r.PRURL
|
|
return j
|
|
}
|
|
|
|
// Woodpecker builds the Job that activates the merged repository in CI.
|
|
func Woodpecker(cfg *config.Config, r store.Request) *batchv1.Job {
|
|
j := base(cfg, r, TypeWoodpecker, []string{"job", "woodpecker-enable", "--name", r.Name}, shortDeadline)
|
|
j.Annotations[AnnoPullRequestNo] = strconv.Itoa(r.PRNumber)
|
|
j.Annotations[AnnoPullRequest] = r.PRURL
|
|
mountWoodpeckerToken(cfg, &j.Spec.Template.Spec)
|
|
return j
|
|
}
|
|
|
|
func base(cfg *config.Config, r store.Request, t Type, args []string, deadline int64) *batchv1.Job {
|
|
labels := map[string]string{
|
|
LabelApp: AppName,
|
|
LabelRequest: r.ID,
|
|
LabelType: string(t),
|
|
}
|
|
annotations := map[string]string{
|
|
AnnoName: r.Name,
|
|
AnnoDescription: r.Description,
|
|
AnnoWoodpecker: strconv.FormatBool(r.Woodpecker),
|
|
AnnoStatusChecks: strings.Join(r.StatusChecks, "\n"),
|
|
AnnoCreated: r.Created.UTC().Format(time.RFC3339),
|
|
}
|
|
vaultDir := path.Dir(cfg.VaultSATokenPath)
|
|
|
|
return &batchv1.Job{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: Name(t, r.ID),
|
|
Namespace: cfg.Namespace,
|
|
Labels: labels,
|
|
Annotations: annotations,
|
|
},
|
|
Spec: batchv1.JobSpec{
|
|
BackoffLimit: ptr(backoffLimit),
|
|
TTLSecondsAfterFinished: ptr(ttlSecondsAfterFinished),
|
|
ActiveDeadlineSeconds: ptr(deadline),
|
|
Template: corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: annotations},
|
|
Spec: corev1.PodSpec{
|
|
RestartPolicy: corev1.RestartPolicyNever,
|
|
ServiceAccountName: cfg.JobServiceAccount,
|
|
Containers: []corev1.Container{{
|
|
Name: string(t),
|
|
Image: cfg.Image,
|
|
Args: args,
|
|
Env: env(cfg),
|
|
TerminationMessagePath: corev1.TerminationMessagePathDefault,
|
|
TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
|
|
VolumeMounts: []corev1.VolumeMount{{
|
|
Name: "vault-token",
|
|
MountPath: vaultDir,
|
|
ReadOnly: true,
|
|
}},
|
|
Resources: corev1.ResourceRequirements{
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceMemory: resource.MustParse("64Mi"),
|
|
corev1.ResourceCPU: resource.MustParse("50m"),
|
|
},
|
|
Limits: corev1.ResourceList{
|
|
corev1.ResourceMemory: resource.MustParse("256Mi"),
|
|
corev1.ResourceCPU: resource.MustParse("500m"),
|
|
},
|
|
},
|
|
}},
|
|
Volumes: []corev1.Volume{{
|
|
Name: "vault-token",
|
|
VolumeSource: corev1.VolumeSource{
|
|
Projected: &corev1.ProjectedVolumeSource{
|
|
Sources: []corev1.VolumeProjection{{
|
|
ServiceAccountToken: &corev1.ServiceAccountTokenProjection{
|
|
Path: path.Base(cfg.VaultSATokenPath),
|
|
Audience: "vault",
|
|
ExpirationSeconds: ptr(int64(600)),
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// env passes the server's resolved configuration down to the job so both sides
|
|
// read the same forge, vault mount and credential path.
|
|
func env(cfg *config.Config) []corev1.EnvVar {
|
|
return []corev1.EnvVar{
|
|
{Name: "GITEA_URL", Value: cfg.GiteaURL},
|
|
{Name: "VAULT_ADDR", Value: cfg.VaultAddr},
|
|
{Name: "REPOSPAWNER_TFGIT_REPO", Value: cfg.TFGitRepo},
|
|
{Name: "REPOSPAWNER_VAULT_K8S_MOUNT", Value: cfg.VaultK8sMount},
|
|
{Name: "REPOSPAWNER_VAULT_K8S_ROLE", Value: cfg.VaultK8sRole},
|
|
{Name: "REPOSPAWNER_VAULT_SA_TOKEN_PATH", Value: cfg.VaultSATokenPath},
|
|
{Name: "REPOSPAWNER_GITEA_CREDS_PATH", Value: cfg.GiteaCredsPath},
|
|
{Name: "WOODPECKER_SERVER", Value: cfg.WoodpeckerServer},
|
|
{Name: "REPOSPAWNER_WOODPECKER_TOKEN_FILE", Value: cfg.WoodpeckerTokenFile},
|
|
{Name: "REPOSPAWNER_ALLOWED_GROUPS", Value: strings.Join(cfg.AllowedGroups, ",")},
|
|
}
|
|
}
|
|
|
|
func mountWoodpeckerToken(cfg *config.Config, spec *corev1.PodSpec) {
|
|
dir := path.Dir(cfg.WoodpeckerTokenFile)
|
|
spec.Volumes = append(spec.Volumes, corev1.Volume{
|
|
Name: "woodpecker-token",
|
|
VolumeSource: corev1.VolumeSource{
|
|
Secret: &corev1.SecretVolumeSource{
|
|
SecretName: cfg.WoodpeckerSecret,
|
|
Items: []corev1.KeyToPath{{Key: "token", Path: path.Base(cfg.WoodpeckerTokenFile)}},
|
|
},
|
|
},
|
|
})
|
|
spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, corev1.VolumeMount{
|
|
Name: "woodpecker-token",
|
|
MountPath: dir,
|
|
ReadOnly: true,
|
|
})
|
|
}
|
|
|
|
// View is a Job reduced to what the reconciler reasons about, so the state
|
|
// machine stays testable without a cluster.
|
|
type View struct {
|
|
Type Type
|
|
Request string
|
|
Succeeded bool
|
|
Failed bool
|
|
Annotations map[string]string
|
|
// Result is the pod's termination message, if the pod has terminated.
|
|
Result []byte
|
|
}
|
|
|
|
// ViewOf reduces a Job (and its pod's termination message) to a View.
|
|
func ViewOf(j batchv1.Job, result []byte) View {
|
|
return View{
|
|
Type: Type(j.Labels[LabelType]),
|
|
Request: j.Labels[LabelRequest],
|
|
Succeeded: j.Status.Succeeded > 0,
|
|
Failed: j.Status.Failed >= backoffLimit+1 || failedCondition(j),
|
|
Annotations: j.Annotations,
|
|
Result: result,
|
|
}
|
|
}
|
|
|
|
func failedCondition(j batchv1.Job) bool {
|
|
for _, c := range j.Status.Conditions {
|
|
if c.Type == batchv1.JobFailed && c.Status == corev1.ConditionTrue {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// RequestFrom rebuilds the request fields an annotated Job carries. It is how
|
|
// the server recovers its state after a restart.
|
|
func RequestFrom(v View) store.Request {
|
|
r := store.Request{
|
|
ID: v.Request,
|
|
Name: v.Annotations[AnnoName],
|
|
Description: v.Annotations[AnnoDescription],
|
|
Woodpecker: v.Annotations[AnnoWoodpecker] == "true",
|
|
}
|
|
if s := v.Annotations[AnnoStatusChecks]; s != "" {
|
|
r.StatusChecks = strings.Split(s, "\n")
|
|
}
|
|
if t, err := time.Parse(time.RFC3339, v.Annotations[AnnoCreated]); err == nil {
|
|
r.Created = t
|
|
}
|
|
if n, err := strconv.Atoi(v.Annotations[AnnoPullRequestNo]); err == nil {
|
|
r.PRNumber = n
|
|
}
|
|
r.PRURL = v.Annotations[AnnoPullRequest]
|
|
return r
|
|
}
|
|
|
|
// DecodeResult parses a termination message into out, tolerating the empty
|
|
// message a pod that never wrote one leaves behind.
|
|
func DecodeResult(raw []byte, out any) bool {
|
|
raw = []byte(strings.TrimSpace(string(raw)))
|
|
if len(raw) == 0 {
|
|
return false
|
|
}
|
|
return json.Unmarshal(raw, out) == nil
|
|
}
|
|
|
|
func ptr[T any](v T) *T { return &v }
|