From 2594d3d9466dd909172dca8cf67f32392a01878e Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Fri, 23 Oct 2026 22:00:34 +1100 Subject: [PATCH 1/3] Add colour package for jq/yq-style JSON and YAML highlighting --- README.md | 1 + colour/colour.go | 87 ++++++++++++++ colour/colour_test.go | 232 ++++++++++++++++++++++++++++++++++++ colour/json.go | 88 ++++++++++++++ colour/yaml.go | 268 ++++++++++++++++++++++++++++++++++++++++++ go.mod | 3 +- 6 files changed, 678 insertions(+), 1 deletion(-) create mode 100644 colour/colour.go create mode 100644 colour/colour_test.go create mode 100644 colour/json.go create mode 100644 colour/yaml.go diff --git a/README.md b/README.md index a9fe51f..0d48c4c 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ imports it inherits them. | --- | --- | | `git.unkin.net/unkin/golib/pg` | Postgres: DSN from the environment, pgxpool construction, read/write splitting across a primary and a replica, and the estate's migration runner. | | `git.unkin.net/unkin/golib/pg/pgtest` | A throwaway Postgres container for a consumer's own `_test.go` files. Test-only. | +| `git.unkin.net/unkin/golib/colour` | jq-style JSON and yq-style YAML highlighting for CLI output, preserving the input byte for byte, plus the TTY/`NO_COLOR` check. | ### pg diff --git a/colour/colour.go b/colour/colour.go new file mode 100644 index 0000000..f91f363 --- /dev/null +++ b/colour/colour.go @@ -0,0 +1,87 @@ +// Package colour highlights JSON the way jq does and YAML the way yq does. It +// works on tokens and re-emits the input byte for byte between ANSI escapes, so +// formatting, whitespace and key order survive; input it cannot parse is +// written through uncoloured. +package colour + +import ( + "os" + "strings" + + "golang.org/x/term" +) + +// Mode is a --colour flag's value: Auto decides from the environment. +type Mode int + +const ( + Auto Mode = iota + Always + Never +) + +const reset = "\x1b[0m" + +// Enabled reports whether output to f should be coloured. Auto follows jq: +// f must be a terminal, NO_COLOR (https://no-color.org) must be unset or empty +// and TERM must not be "dumb". +func Enabled(mode Mode, f *os.File) bool { + switch mode { + case Always: + return true + case Never: + return false + } + if os.Getenv("NO_COLOR") != "" || os.Getenv("TERM") == "dumb" { + return false + } + return term.IsTerminal(int(f.Fd())) +} + +// jqDefault is jq's DEFAULT_COLORS: null, false, true, number, string, array, +// object, object key. +var jqDefault = [8]string{"0;90", "0;39", "0;39", "0;39", "0;32", "1;39", "1;39", "1;34"} + +const ( + jqNull = iota + jqFalse + jqTrue + jqNumber + jqString + jqArray + jqObject + jqKey +) + +// jqColours returns jq's palette with JQ_COLORS applied the way jq parses it: +// up to eight colon-separated SGR codes override the defaults in order, a +// trailing empty field is ignored, and anything but digits and ';' discards the +// whole variable. +func jqColours() [8]string { + codes := jqDefault + spec := os.Getenv("JQ_COLORS") + if spec == "" { + return sgr(codes) + } + fields := strings.Split(spec, ":") + if fields[len(fields)-1] == "" { + fields = fields[:len(fields)-1] + } + if len(fields) > len(codes) { + fields = fields[:len(codes)] + } + for i, f := range fields { + if strings.Trim(f, "0123456789;") != "" { + return sgr(jqDefault) + } + codes[i] = f + } + return sgr(codes) +} + +func sgr(codes [8]string) [8]string { + for i, c := range codes { + codes[i] = "\x1b[" + c + "m" + } + return codes +} diff --git a/colour/colour_test.go b/colour/colour_test.go new file mode 100644 index 0000000..d17f1e2 --- /dev/null +++ b/colour/colour_test.go @@ -0,0 +1,232 @@ +package colour + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "regexp" + "testing" +) + +var ansi = regexp.MustCompile("\x1b\\[[0-9;]*m") + +// mark rewrites escapes into readable tags so golden strings stay legible. +func mark(s string) string { + tags := map[string]string{ + reset: "", yqKey: "", yqString: "", yqNumBool: "", + yqAnchor: "", yqComment: "", + "\x1b[0;90m": "", "\x1b[0;39m": "", "\x1b[0;32m": "", + "\x1b[1;39m": "", "\x1b[1;34m": "", + } + return ansi.ReplaceAllStringFunc(s, func(e string) string { + if t, ok := tags[e]; ok { + return t + } + return e + }) +} + +func render(t *testing.T, f func(*bytes.Buffer, []byte) error, in string) string { + t.Helper() + var out bytes.Buffer + if err := f(&out, []byte(in)); err != nil { + t.Fatal(err) + } + if got := ansi.ReplaceAllString(out.String(), ""); got != in { + t.Fatalf("stripped output differs from input:\n%q\n%q", got, in) + } + return mark(out.String()) +} + +func jsonF(b *bytes.Buffer, in []byte) error { return JSON(b, in) } +func yamlF(b *bytes.Buffer, in []byte) error { return YAML(b, in) } + +func TestJSON(t *testing.T) { + t.Setenv("JQ_COLORS", "") + cases := map[string]string{ + `null`: `null`, + `false`: `false`, + `-1.5e3`: `-1.5e3`, + "{\n \"a\\\"b\" : \"x\",\n \"n\": [1, true, {}]\n}\n": "{\n \"a\\\"b\" : \"x\",\n" + + " \"n\": [1, true, {}]\n}\n", + } + for in, want := range cases { + if got := render(t, jsonF, in); got != want { + t.Errorf("JSON(%q)\n got %q\nwant %q", in, got, want) + } + } +} + +func TestJQColors(t *testing.T) { + cases := map[string]string{ + "": "\x1b[0;90mnull\x1b[0m", + "1;31": "\x1b[1;31mnull\x1b[0m", + "1;31:": "\x1b[1;31mnull\x1b[0m", + "x": "\x1b[0;90mnull\x1b[0m", + "0:0:0:0:0:0:0:0:z": "\x1b[0mnull\x1b[0m", + } + for env, want := range cases { + t.Setenv("JQ_COLORS", env) + var out bytes.Buffer + if err := JSON(&out, []byte("null")); err != nil { + t.Fatal(err) + } + if out.String() != want { + t.Errorf("JQ_COLORS=%q: got %q want %q", env, out.String(), want) + } + } + // Empty middle fields become a bare ESC[m, as in jq. + t.Setenv("JQ_COLORS", "::::::7:4") + var out bytes.Buffer + _ = JSON(&out, []byte(`{"k":"v"}`)) + if want := "\x1b[7m{\x1b[0m\x1b[4m\"k\"\x1b[0m\x1b[7m:\x1b[0m\x1b[m\"v\"\x1b[0m\x1b[7m}\x1b[0m"; out.String() != want { + t.Errorf("got %q want %q", out.String(), want) + } +} + +const vaultYAML = `--- +data: + # creds + password: "s3cr:et" # rotated + ttl: 768h + count: 0x1F + ratio: -.5 + on: True + gone: ~ + note: |- + line one + + line: two + folded: > # c + text + tags: [a, "b", 3, null] + meta: {k: v, "q": 1, u: http://x} + 'single': 'it''s' + empty: [] +lease: + - id: &a abc + copy: *a + - - nested + - !!str 42 + - +... +` + +func TestYAML(t *testing.T) { + want := `--- +data: + # creds + password: "s3cr:et" # rotated + ttl: 768h + count: 0x1F + ratio: -.5 + on: True + gone: ~ + note: |- + line one + + line: two + folded: > # c + text + tags: [a, "b", 3, null] + meta: {k: v, "q": 1, u: http://x} + 'single': 'it''s' + empty: [] +lease: + - id: &a abc + copy: *a + - - nested + - !!str 42 + - +... +` + if got := render(t, yamlF, vaultYAML); got != want { + t.Errorf("YAML mismatch\n got:\n%s\nwant:\n%s", got, want) + } +} + +func TestYAMLEdges(t *testing.T) { + cases := map[string]string{ + "a: 1\r\nb:\r\n": "a: 1\r\nb:\r\n", + "k #x: v\n": "k #x: v\n", + "--- a\n": "--- a\n", + "x: &anchor\n y: 1\n": "x: &anchor\n y: 1\n", + "x: !tag\n": "x: !tag\n", + "x: [a] # c\n": "x: [a] # c\n", + "x: |\n a\ny: 2\n": "x: |\n a\ny: 2\n", + "\"k\" : v\n": "\"k\" : v\n", + "x: \"a\\\"b\"\n": "x: \"a\\\"b\"\n", + "- [\"a\", \"b\": 1]\n": "- [\"a\", \"b\": 1]\n", + "x: >\n": "x: >\n", + } + for in, want := range cases { + if got := render(t, yamlF, in); got != want { + t.Errorf("YAML(%q)\n got %q\nwant %q", in, got, want) + } + } +} + +func TestPassthrough(t *testing.T) { + for _, f := range []func(*bytes.Buffer, []byte) error{jsonF, yamlF} { + in := "{\"a\": [1,\n" + var out bytes.Buffer + if err := f(&out, []byte(in)); err != nil { + t.Fatal(err) + } + if out.String() != in { + t.Errorf("invalid input not passed through: %q", out.String()) + } + } + table := "Key Value\n--- -----\npassword s3cret\n" + var out bytes.Buffer + _ = YAML(&out, []byte(table)) + if ansi.MatchString(out.String()) && ansi.ReplaceAllString(out.String(), "") != table { + t.Errorf("table output mangled: %q", out.String()) + } +} + +type failWriter struct{} + +func (failWriter) Write([]byte) (int, error) { return 0, errors.New("boom") } + +func TestWriteError(t *testing.T) { + for _, in := range []string{"{}", "{", "a: 1", "a: ["} { + if JSON(failWriter{}, []byte(in)) == nil || YAML(failWriter{}, []byte(in)) == nil { + t.Errorf("write error swallowed for %q", in) + } + } +} + +func TestEnabled(t *testing.T) { + f, err := os.Create(filepath.Join(t.TempDir(), "out")) + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + t.Setenv("NO_COLOR", "") + t.Setenv("TERM", "xterm") + if Enabled(Auto, f) { + t.Error("Auto coloured a regular file") + } + if !Enabled(Always, f) || Enabled(Never, f) { + t.Error("Always/Never not honoured") + } + tty, err := os.OpenFile("/dev/tty", os.O_WRONLY, 0) + if err != nil { + t.Skip("no controlling terminal") + } + defer func() { _ = tty.Close() }() + if !Enabled(Auto, tty) { + t.Error("Auto did not colour a terminal") + } + t.Setenv("NO_COLOR", "1") + if Enabled(Auto, tty) || !Enabled(Always, tty) { + t.Error("NO_COLOR not honoured, or overrode Always") + } + t.Setenv("NO_COLOR", "") + t.Setenv("TERM", "dumb") + if Enabled(Auto, tty) { + t.Error("TERM=dumb coloured") + } +} diff --git a/colour/json.go b/colour/json.go new file mode 100644 index 0000000..889dd60 --- /dev/null +++ b/colour/json.go @@ -0,0 +1,88 @@ +package colour + +import ( + "bytes" + "encoding/json" + "io" +) + +// JSON writes in to w with jq's colours, honouring JQ_COLORS. Brackets, braces, +// commas and colons take their container's colour, as in jq. +func JSON(w io.Writer, in []byte) error { + if !json.Valid(in) { + _, err := w.Write(in) + return err + } + c := jqColours() + var out bytes.Buffer + var stack []byte + paint := func(col string, tok []byte) { + out.WriteString(col) + out.Write(tok) + out.WriteString(reset) + } + container := func() string { + if len(stack) > 0 && stack[len(stack)-1] == '{' { + return c[jqObject] + } + return c[jqArray] + } + for i := 0; i < len(in); { + switch b := in[i]; b { + case ' ', '\t', '\r', '\n': + out.WriteByte(b) + i++ + case '{', '[': + stack = append(stack, b) + paint(container(), in[i:i+1]) + i++ + case '}', ']': + paint(container(), in[i:i+1]) + stack = stack[:len(stack)-1] + i++ + case ',', ':': + paint(container(), in[i:i+1]) + i++ + case '"': + j := i + 1 + for in[j] != '"' { + if in[j] == '\\' { + j++ + } + j++ + } + j++ + col := c[jqString] + if k := skipSpace(in, j); k < len(in) && in[k] == ':' { + col = c[jqKey] + } + paint(col, in[i:j]) + i = j + default: + j := i + for j < len(in) && !bytes.ContainsRune([]byte(" \t\r\n,]}"), rune(in[j])) { + j++ + } + col := c[jqNumber] + switch string(in[i:j]) { + case "null": + col = c[jqNull] + case "true": + col = c[jqTrue] + case "false": + col = c[jqFalse] + } + paint(col, in[i:j]) + i = j + } + } + _, err := w.Write(out.Bytes()) + return err +} + +func skipSpace(in []byte, i int) int { + for i < len(in) && (in[i] == ' ' || in[i] == '\t' || in[i] == '\r' || in[i] == '\n') { + i++ + } + return i +} diff --git a/colour/yaml.go b/colour/yaml.go new file mode 100644 index 0000000..f963fc6 --- /dev/null +++ b/colour/yaml.go @@ -0,0 +1,268 @@ +package colour + +import ( + "bytes" + "errors" + "io" + "regexp" + "strings" + + "gopkg.in/yaml.v3" +) + +// yq's palette (pkg/yqlib/color_print.go). Null, tags, flow punctuation, +// block-scalar indicators and document markers stay uncoloured, as in yq. +const ( + yqKey = "\x1b[36m" + yqString = "\x1b[32m" + yqNumBool = "\x1b[95m" + yqAnchor = "\x1b[93m" + yqComment = "\x1b[90m" +) + +// YAML 1.2 core schema int and float forms. +var yamlNumber = regexp.MustCompile(`^(?:[-+]?[0-9]+|0o[0-7]+|0x[0-9a-fA-F]+|[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$`) + +// YAML writes in to w with yq's colours. It handles block mappings and +// sequences, plain and quoted scalars, '|' and '>' blocks, single-line flow +// collections, anchors, aliases and comments. +func YAML(w io.Writer, in []byte) error { + if !validYAML(in) { + _, err := w.Write(in) + return err + } + var out bytes.Buffer + block := -1 + for _, line := range strings.SplitAfter(string(in), "\n") { + body := strings.TrimRight(line, "\r\n") + eol := line[len(body):] + trimmed := strings.TrimLeft(body, " ") + indent := len(body) - len(trimmed) + if block >= 0 { + if trimmed == "" || indent > block { + out.WriteString(body[:indent]) + if trimmed != "" { + paint(&out, yqString, trimmed) + } + out.WriteString(eol) + continue + } + block = -1 + } + out.WriteString(body[:indent]) + if trimmed == "---" || trimmed == "..." || strings.HasPrefix(trimmed, "--- ") { + out.WriteString(trimmed) + } else if yamlNode(&out, trimmed) { + block = indent + } + out.WriteString(eol) + } + _, err := w.Write(out.Bytes()) + return err +} + +func validYAML(in []byte) bool { + dec := yaml.NewDecoder(bytes.NewReader(in)) + for { + var n yaml.Node + err := dec.Decode(&n) + if errors.Is(err, io.EOF) { + return true + } + if err != nil { + return false + } + } +} + +func paint(out *bytes.Buffer, col, s string) { + if col == "" { + out.WriteString(s) + return + } + out.WriteString(col) + out.WriteString(s) + out.WriteString(reset) +} + +// yamlNode colours one line's content after its indentation and reports +// whether it opens a block scalar. +func yamlNode(out *bytes.Buffer, s string) bool { + for strings.HasPrefix(s, "- ") || s == "-" { + n := len(s) - len(strings.TrimLeft(s[1:], " ")) // dash plus spaces + out.WriteString(s[:n]) + s = s[n:] + } + if s == "" { + return false + } + if s[0] == '#' { + paint(out, yqComment, s) + return false + } + if k := keyEnd(s); k > 0 { + paint(out, yqKey, s[:k]) + rest := s[k:] + colon := strings.IndexByte(rest, ':') + 1 + out.WriteString(rest[:colon]) + rest = rest[colon:] + value := strings.TrimLeft(rest, " ") + out.WriteString(rest[:len(rest)-len(value)]) + s = value + } + return yamlValue(out, s) +} + +// keyEnd returns the length of a mapping key at the start of s, or 0 when s is +// not a key/value pair. +func keyEnd(s string) int { + end := 0 + switch s[0] { + case '"', '\'': + end = quoteEnd(s) + case '[', '{', '&', '*', '!', '|', '>', '?', '%', '@', '`': + return 0 + default: + for i := 0; i < len(s); i++ { + if s[i] == ':' && (i+1 == len(s) || s[i+1] == ' ') { + return len(strings.TrimRight(s[:i], " ")) + } + if s[i] == '#' && i > 0 && s[i-1] == ' ' { + return 0 + } + } + return 0 + } + rest := strings.TrimLeft(s[end:], " ") + if strings.HasPrefix(rest, ":") && (len(rest) == 1 || rest[1] == ' ') { + return end + } + return 0 +} + +// quoteEnd returns the index just past the quoted scalar starting s, or len(s) +// when it is unterminated on this line. +func quoteEnd(s string) int { + q := s[0] + for i := 1; i < len(s); i++ { + switch { + case q == '"' && s[i] == '\\': + i++ + case s[i] == q && q == '\'' && i+1 < len(s) && s[i+1] == '\'': + i++ + case s[i] == q: + return i + 1 + } + } + return len(s) +} + +// yamlValue colours a scalar or flow collection with any trailing comment. +func yamlValue(out *bytes.Buffer, s string) bool { + if s == "" { + return false + } + for s[0] == '&' || s[0] == '!' { + n := strings.IndexByte(s, ' ') + if n < 0 { + n = len(s) + } + if s[0] == '&' { + paint(out, yqAnchor, s[:n]) + } else { + out.WriteString(s[:n]) + } + v := strings.TrimLeft(s[n:], " ") + out.WriteString(s[n : len(s)-len(v)]) + if s = v; s == "" { + return false + } + } + switch s[0] { + case '#': + paint(out, yqComment, s) + return false + case '|', '>': + n := strings.IndexByte(s, ' ') + if n < 0 { + n = len(s) + } + out.WriteString(s[:n]) + comment(out, s[n:]) + return true + case '[', '{': + flow(out, s) + return false + } + end := len(s) + if s[0] == '"' || s[0] == '\'' { + end = quoteEnd(s) + } else if i := strings.Index(s, " #"); i >= 0 { + end = i + } + val := strings.TrimRight(s[:end], " ") + paint(out, scalarColour(val), val) + comment(out, s[len(val):]) + return false +} + +// comment writes rest, colouring a trailing "# ..." comment. +func comment(out *bytes.Buffer, rest string) { + i := strings.IndexByte(rest, '#') + if i < 0 { + out.WriteString(rest) + return + } + out.WriteString(rest[:i]) + paint(out, yqComment, rest[i:]) +} + +func scalarColour(v string) string { + switch v[0] { + case '"', '\'': + return yqString + case '*': + return yqAnchor + } + switch v { + case "null", "Null", "NULL", "~": + return "" + case "true", "True", "TRUE", "false", "False", "FALSE": + return yqNumBool + } + if yamlNumber.MatchString(v) { + return yqNumBool + } + return yqString +} + +// flow colours a single-line flow collection: punctuation plain, keys and +// scalars as in block style. +func flow(out *bytes.Buffer, s string) { + for len(s) > 0 { + switch c := s[0]; { + case strings.IndexByte("[]{},: ", c) >= 0: + out.WriteByte(c) + s = s[1:] + case c == '#': + paint(out, yqComment, s) + return + default: + end, scope := len(s), s + if c == '"' || c == '\'' { + end = quoteEnd(s) + } else if i := strings.IndexAny(s, ",]}"); i >= 0 { + end, scope = i, s[:i] + } + tok := s[:end] + if k := keyEnd(scope); k > 0 { + paint(out, yqKey, tok[:k]) + s = s[k:] + continue + } + tok = strings.TrimRight(tok, " ") + paint(out, scalarColour(tok), tok) + s = s[len(tok):] + } + } +} diff --git a/go.mod b/go.mod index 05682a4..3afe97c 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require ( github.com/jackc/pgx/v5 v5.9.2 github.com/testcontainers/testcontainers-go v0.44.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 + golang.org/x/term v0.45.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -63,5 +65,4 @@ require ( golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.40.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) From 37112ddf6883893de97fba1ac59896f997f80300 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Fri, 23 Oct 2026 22:16:42 +1100 Subject: [PATCH 2/3] colour: fix block scalars under list items, multi-line scalars, yq numbers and JQ_COLORS parsing --- colour/colour.go | 40 +++++------ colour/colour_test.go | 66 ++++++++++++++---- colour/yaml.go | 159 +++++++++++++++++++++++++++++------------- 3 files changed, 186 insertions(+), 79 deletions(-) diff --git a/colour/colour.go b/colour/colour.go index f91f363..cc8c038 100644 --- a/colour/colour.go +++ b/colour/colour.go @@ -22,9 +22,9 @@ const ( const reset = "\x1b[0m" -// Enabled reports whether output to f should be coloured. Auto follows jq: -// f must be a terminal, NO_COLOR (https://no-color.org) must be unset or empty -// and TERM must not be "dumb". +// Enabled reports whether output to f should be coloured. Auto requires f to +// be a terminal and NO_COLOR (https://no-color.org) to be unset or empty, as jq +// does, and additionally TERM not to be "dumb". func Enabled(mode Mode, f *os.File) bool { switch mode { case Always: @@ -53,28 +53,28 @@ const ( jqKey ) -// jqColours returns jq's palette with JQ_COLORS applied the way jq parses it: -// up to eight colon-separated SGR codes override the defaults in order, a -// trailing empty field is ignored, and anything but digits and ';' discards the -// whole variable. +// jqColours returns jq's palette with JQ_COLORS applied as jq 1.8.1's +// jq_set_colors parses it: colon-separated runs of digits and ';' override the +// defaults in order, a trailing empty field is ignored, the eighth field ends at +// its first other character and anything after it is ignored, and any other +// character in the first seven fields discards the whole variable. func jqColours() [8]string { codes := jqDefault spec := os.Getenv("JQ_COLORS") - if spec == "" { - return sgr(codes) - } - fields := strings.Split(spec, ":") - if fields[len(fields)-1] == "" { - fields = fields[:len(fields)-1] - } - if len(fields) > len(codes) { - fields = fields[:len(codes)] - } - for i, f := range fields { - if strings.Trim(f, "0123456789;") != "" { + for i := 0; ; i++ { + rest := strings.TrimLeft(spec, "0123456789;") + field := spec[:len(spec)-len(rest)] + if rest == "" || i+1 == len(codes) { + if field != "" { + codes[i] = field + } + break + } + if rest[0] != ':' { return sgr(jqDefault) } - codes[i] = f + codes[i] = field + spec = rest[1:] } return sgr(codes) } diff --git a/colour/colour_test.go b/colour/colour_test.go index d17f1e2..745eb14 100644 --- a/colour/colour_test.go +++ b/colour/colour_test.go @@ -65,6 +65,9 @@ func TestJQColors(t *testing.T) { "1;31:": "\x1b[1;31mnull\x1b[0m", "x": "\x1b[0;90mnull\x1b[0m", "0:0:0:0:0:0:0:0:z": "\x1b[0mnull\x1b[0m", + "1:0:0:0:0:0:0:0z!": "\x1b[1mnull\x1b[0m", + "1:0:0:0:0:0:0:z": "\x1b[1mnull\x1b[0m", + "1:z:0": "\x1b[0;90mnull\x1b[0m", } for env, want := range cases { t.Setenv("JQ_COLORS", env) @@ -91,6 +94,8 @@ data: password: "s3cr:et" # rotated ttl: 768h count: 0x1F + big: 1_000 + exp: 1e3 ratio: -.5 on: True gone: ~ @@ -119,7 +124,9 @@ func TestYAML(t *testing.T) { # creds password: "s3cr:et" # rotated ttl: 768h - count: 0x1F + count: 0x1F + big: 1_000 + exp: 1e3 ratio: -.5 on: True gone: ~ @@ -148,17 +155,23 @@ func TestYAML(t *testing.T) { func TestYAMLEdges(t *testing.T) { cases := map[string]string{ - "a: 1\r\nb:\r\n": "a: 1\r\nb:\r\n", - "k #x: v\n": "k #x: v\n", - "--- a\n": "--- a\n", - "x: &anchor\n y: 1\n": "x: &anchor\n y: 1\n", - "x: !tag\n": "x: !tag\n", - "x: [a] # c\n": "x: [a] # c\n", - "x: |\n a\ny: 2\n": "x: |\n a\ny: 2\n", - "\"k\" : v\n": "\"k\" : v\n", - "x: \"a\\\"b\"\n": "x: \"a\\\"b\"\n", - "- [\"a\", \"b\": 1]\n": "- [\"a\", \"b\": 1]\n", - "x: >\n": "x: >\n", + "a: 1\r\nb:\r\n": "a: 1\r\nb:\r\n", + "k #x: v\n": "k #x: v\n", + "--- a\n": "--- a\n", + "x: &anchor\n y: 1\n": "x: &anchor\n y: 1\n", + "x: !tag\n": "x: !tag\n", + "x: [a] # c\n": "x: [a] # c\n", + "x: |\n a\ny: 2\n": "x: |\n a\ny: 2\n", + "\"k\" : v\n": "\"k\" : v\n", + "x: \"a\\\"b\"\n": "x: \"a\\\"b\"\n", + "- [\"a\", \"b\": 1]\n": "- [\"a\", \"b\": 1]\n", + "x: >\n": "x: >\n", + "- b: |\n x\n c: 2\n": "- b: |\n x\n c: 2\n", + "- - |\n y\n - z: 1\n": "- - |\n y\n - z: 1\n", + "- certs:\n - pem: |\n -----BEGIN CERTIFICATE-----\n MIIB\n ttl: 768h\n n: 1\n": "- certs:\n" + + " - pem: |\n -----BEGIN CERTIFICATE-----\n MIIB\n ttl: 768h\n n: 1\n", + "w: one two\n\n 42\n# c\nn: 1\n": "w: one two\n\n 42\n# c\nn: 1\n", + "q: 'it''s\n ok: y' # c\n": "q: 'it''s\n ok: y' # c\n", } for in, want := range cases { if got := render(t, yamlF, in); got != want { @@ -167,6 +180,35 @@ func TestYAMLEdges(t *testing.T) { } } +// vault -format=yaml (yaml.v2) folds long strings at 80 columns. +func TestYAMLWrapped(t *testing.T) { + part := "policy grants read: on secret/data/foo policy grants read: on secret/data/foo" + in := "data:\n warning: \"" + part + "\n " + part + "\n \\\"done\\\"\"\n next: 1\n" + want := "data:\n warning: \"" + part + "\n " + part + + "\n \\\"done\\\"\"\n next: 1\n" + if got := render(t, yamlF, in); got != want { + t.Errorf("YAML mismatch\n got %q\nwant %q", got, want) + } +} + +func TestYAMLNumbers(t *testing.T) { + cases := map[string]string{ + "1_000": "", "-.5": "", "+1": "", "1.0e+3": "", "-1": "", + "1e3": "", "09": "", "_1": "", "1.2.3": "", "+.inf": "", + "99999999999999999999": "", "1.5e400": "", + "0o17": "", "012": "", "-012": "", "0x1F": "", "0b101": "", ".inf": "", "-.Inf": "", ".nan": "", + } + for v, tag := range cases { + want := "k: " + tag + v + if tag != "" { + want += "" + } + if got := render(t, yamlF, "k: "+v+"\n"); got != want+"\n" { + t.Errorf("%s: got %q want %q", v, got, want) + } + } +} + func TestPassthrough(t *testing.T) { for _, f := range []func(*bytes.Buffer, []byte) error{jsonF, yamlF} { in := "{\"a\": [1,\n" diff --git a/colour/yaml.go b/colour/yaml.go index f963fc6..03404e0 100644 --- a/colour/yaml.go +++ b/colour/yaml.go @@ -4,7 +4,7 @@ import ( "bytes" "errors" "io" - "regexp" + "strconv" "strings" "gopkg.in/yaml.v3" @@ -20,40 +20,49 @@ const ( yqComment = "\x1b[90m" ) -// YAML 1.2 core schema int and float forms. -var yamlNumber = regexp.MustCompile(`^(?:[-+]?[0-9]+|0o[0-7]+|0x[0-9a-fA-F]+|[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$`) +// Kinds of scalar left open at the end of a line. +const ( + openNone = 0 + openBlock = '|' + openPlain = 'p' +) // YAML writes in to w with yq's colours. It handles block mappings and -// sequences, plain and quoted scalars, '|' and '>' blocks, single-line flow -// collections, anchors, aliases and comments. +// sequences, plain and quoted scalars (including multi-line ones), '|' and +// '>' blocks, single-line flow collections, anchors, aliases and comments. func YAML(w io.Writer, in []byte) error { if !validYAML(in) { _, err := w.Write(in) return err } var out bytes.Buffer - block := -1 + // open is the scalar continuing onto the next line: a block or plain + // scalar owned by the node at column parent, or an unterminated quote. + open, parent := byte(openNone), 0 for _, line := range strings.SplitAfter(string(in), "\n") { body := strings.TrimRight(line, "\r\n") eol := line[len(body):] trimmed := strings.TrimLeft(body, " ") indent := len(body) - len(trimmed) - if block >= 0 { - if trimmed == "" || indent > block { - out.WriteString(body[:indent]) - if trimmed != "" { - paint(&out, yqString, trimmed) - } - out.WriteString(eol) - continue - } - block = -1 - } out.WriteString(body[:indent]) - if trimmed == "---" || trimmed == "..." || strings.HasPrefix(trimmed, "--- ") { + switch { + case open == '"' || open == '\'': + if end := quoteClose(trimmed, open); end < 0 { + paint(&out, yqString, trimmed) + } else { + paint(&out, yqString, trimmed[:end]) + comment(&out, trimmed[end:]) + open = openNone + } + case open != openNone && (trimmed == "" || indent > parent && (open == openBlock || trimmed[0] != '#')): + paint(&out, yqString, trimmed) + case trimmed == "---" || trimmed == "..." || strings.HasPrefix(trimmed, "--- "): out.WriteString(trimmed) - } else if yamlNode(&out, trimmed) { - block = indent + open = openNone + default: + var col int + col, open = yamlNode(&out, trimmed) + parent = indent + col } out.WriteString(eol) } @@ -76,7 +85,7 @@ func validYAML(in []byte) bool { } func paint(out *bytes.Buffer, col, s string) { - if col == "" { + if col == "" || s == "" { out.WriteString(s) return } @@ -85,22 +94,25 @@ func paint(out *bytes.Buffer, col, s string) { out.WriteString(reset) } -// yamlNode colours one line's content after its indentation and reports -// whether it opens a block scalar. -func yamlNode(out *bytes.Buffer, s string) bool { +// yamlNode colours one line's content after its indentation. It returns the +// column, relative to s, of the innermost node (the last '-' or the key) and +// the kind of scalar left open at the end of the line. +func yamlNode(out *bytes.Buffer, s string) (int, byte) { + col, off := 0, 0 for strings.HasPrefix(s, "- ") || s == "-" { n := len(s) - len(strings.TrimLeft(s[1:], " ")) // dash plus spaces out.WriteString(s[:n]) - s = s[n:] + col, off, s = off, off+n, s[n:] } if s == "" { - return false + return col, openNone } if s[0] == '#' { paint(out, yqComment, s) - return false + return col, openNone } if k := keyEnd(s); k > 0 { + col = off paint(out, yqKey, s[:k]) rest := s[k:] colon := strings.IndexByte(rest, ':') + 1 @@ -110,7 +122,7 @@ func yamlNode(out *bytes.Buffer, s string) bool { out.WriteString(rest[:len(rest)-len(value)]) s = value } - return yamlValue(out, s) + return col, yamlValue(out, s) } // keyEnd returns the length of a mapping key at the start of s, or 0 when s is @@ -143,8 +155,16 @@ func keyEnd(s string) int { // quoteEnd returns the index just past the quoted scalar starting s, or len(s) // when it is unterminated on this line. func quoteEnd(s string) int { - q := s[0] - for i := 1; i < len(s); i++ { + if end := quoteClose(s[1:], s[0]); end >= 0 { + return end + 1 + } + return len(s) +} + +// quoteClose returns the index just past the quote q closing a scalar whose +// opening quote precedes s, or -1 when s does not close it. +func quoteClose(s string, q byte) int { + for i := 0; i < len(s); i++ { switch { case q == '"' && s[i] == '\\': i++ @@ -154,13 +174,14 @@ func quoteEnd(s string) int { return i + 1 } } - return len(s) + return -1 } -// yamlValue colours a scalar or flow collection with any trailing comment. -func yamlValue(out *bytes.Buffer, s string) bool { +// yamlValue colours a scalar or flow collection with any trailing comment and +// returns the kind of scalar it leaves open. +func yamlValue(out *bytes.Buffer, s string) byte { if s == "" { - return false + return openNone } for s[0] == '&' || s[0] == '!' { n := strings.IndexByte(s, ' ') @@ -175,13 +196,13 @@ func yamlValue(out *bytes.Buffer, s string) bool { v := strings.TrimLeft(s[n:], " ") out.WriteString(s[n : len(s)-len(v)]) if s = v; s == "" { - return false + return openNone } } switch s[0] { case '#': paint(out, yqComment, s) - return false + return openNone case '|', '>': n := strings.IndexByte(s, ' ') if n < 0 { @@ -189,21 +210,28 @@ func yamlValue(out *bytes.Buffer, s string) bool { } out.WriteString(s[:n]) comment(out, s[n:]) - return true + return openBlock case '[', '{': flow(out, s) - return false + return openNone } - end := len(s) - if s[0] == '"' || s[0] == '\'' { - end = quoteEnd(s) - } else if i := strings.Index(s, " #"); i >= 0 { - end = i + end, open := len(s), byte(openPlain) + switch s[0] { + case '"', '\'': + if end, open = quoteClose(s[1:], s[0])+1, openNone; end == 0 { + end, open = len(s), s[0] + } + case '*': + open = openNone + default: + if i := strings.Index(s, " #"); i >= 0 { + end, open = i, openNone + } } val := strings.TrimRight(s[:end], " ") paint(out, scalarColour(val), val) comment(out, s[len(val):]) - return false + return open } // comment writes rest, colouring a trailing "# ..." comment. @@ -225,17 +253,54 @@ func scalarColour(v string) string { return yqAnchor } switch v { - case "null", "Null", "NULL", "~": + case "null", "Null", "NULL", "~", ".inf", ".Inf", ".INF", "-.inf", "-.Inf", "-.INF", ".nan", ".NaN", ".NAN": return "" case "true", "True", "TRUE", "false", "False", "FALSE": return yqNumBool } - if yamlNumber.MatchString(v) { - return yqNumBool + if col, ok := number(v); ok { + return col } return yqString } +// number mirrors goccy/go-yaml v1.19.2 token.toNumber, which yq's lexer uses: +// decimal ints and floats get the number colour; hex, octal (0o or a leading +// 0) and binary ints are typed but yq leaves them uncoloured. +func number(v string) (string, bool) { + if v == "" || v[0] == '_' || strings.Count(v, ".") > 1 { + return "", false + } + neg := v[0] == '-' + n := strings.ReplaceAll(strings.TrimPrefix(strings.TrimPrefix(v, "+"), "-"), "_", "") + base, col := 10, "" + switch { + case strings.HasPrefix(n, "0x"): + n, base = n[2:], 16 + case strings.HasPrefix(n, "0o"): + n, base = n[2:], 8 + case strings.HasPrefix(n, "0b"): + n, base = n[2:], 2 + case strings.HasPrefix(n, "0") && len(n) > 1 && !strings.Contains(v, "."): + base = 8 + case strings.Contains(v, "."): + if neg { + n = "-" + n + } + _, err := strconv.ParseFloat(n, 64) + return yqNumBool, err == nil + default: + col = yqNumBool + } + var err error + if neg { + _, err = strconv.ParseInt("-"+n, base, 64) + } else { + _, err = strconv.ParseUint(n, base, 64) + } + return col, err == nil +} + // flow colours a single-line flow collection: punctuation plain, keys and // scalars as in block style. func flow(out *bytes.Buffer, s string) { From bd61a52463dab8850a5204bb58f6bfc094d6f1be Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Fri, 23 Oct 2026 22:33:53 +1100 Subject: [PATCH 3/3] colour: treat ? as a YAML key indicator only before space or EOL --- colour/colour_test.go | 2 ++ colour/yaml.go | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/colour/colour_test.go b/colour/colour_test.go index 745eb14..c9de789 100644 --- a/colour/colour_test.go +++ b/colour/colour_test.go @@ -172,6 +172,8 @@ func TestYAMLEdges(t *testing.T) { " - pem: |\n -----BEGIN CERTIFICATE-----\n MIIB\n ttl: 768h\n n: 1\n", "w: one two\n\n 42\n# c\nn: 1\n": "w: one two\n\n 42\n# c\nn: 1\n", "q: 'it''s\n ok: y' # c\n": "q: 'it''s\n ok: y' # c\n", + "?q: ?v\n?n:\n a: 1\n b: |\n x\nd: 2\n": "?q: ?v\n?n:\n a: 1\n" + + " b: |\n x\nd: 2\n", } for in, want := range cases { if got := render(t, yamlF, in); got != want { diff --git a/colour/yaml.go b/colour/yaml.go index 03404e0..04f6b6f 100644 --- a/colour/yaml.go +++ b/colour/yaml.go @@ -129,10 +129,12 @@ func yamlNode(out *bytes.Buffer, s string) (int, byte) { // not a key/value pair. func keyEnd(s string) int { end := 0 - switch s[0] { - case '"', '\'': + switch { + case s[0] == '"' || s[0] == '\'': end = quoteEnd(s) - case '[', '{', '&', '*', '!', '|', '>', '?', '%', '@', '`': + case s[0] == '?' && (len(s) == 1 || s[1] == ' '): + return 0 + case strings.IndexByte("[{&*!|>%@`", s[0]) >= 0: return 0 default: for i := 0; i < len(s); i++ {