puppet-catalog-tools/internal/impact/index.go

package impact

import (
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"gopkg.in/yaml.v3"
)

// manifestRoots are the modulepath entries that live in this repo.
// external_modules/ is Puppetfile-managed and not checked in.
var manifestRoots = []string{"site", "modules"}

var (
	commentRe    = regexp.MustCompile(`(?m)#.*$`)
	includeRe    = regexp.MustCompile(`\b(?:include|contain|require|inherits)\s*\(?\s*['"]?(?:::)?([a-z][a-z0-9_]*(?:::[a-z0-9_]+)*)['"]?`)
	classParamRe = regexp.MustCompile(`\bclass\s*\{\s*['"](?:::)?([a-z][a-z0-9_]*(?:::[a-z0-9_]+)*)['"]`)
	classRefRe   = regexp.MustCompile(`\bClass\s*\[\s*['"](?:::)?([a-z][a-z0-9_]*(?:::[a-z0-9_]+)*)['"]`)
	defineUseRe  = regexp.MustCompile(`(?m)(?:^|[^\w:$.])(?:::)?([a-z][a-z0-9_]*(?:::[a-z0-9_]+)+)\s*\{`)
	// $mod::params::thing reads a class variable, which is a dependency on the
	// class that declares it.
	scopeVarRe = regexp.MustCompile(`\$(?:::)?([a-z][a-z0-9_]*(?:::[a-z0-9_]+)+)::[a-z_][a-z0-9_]*`)
	// create_resources('mod::type', ...) names a defined type in a string.
	createResRe   = regexp.MustCompile(`create_resources\s*\(\s*['"](?:::)?([a-z][a-z0-9_]*(?:::[a-z0-9_]+)*)['"]`)
	resourceRefRe = regexp.MustCompile(`\b([A-Z][a-zA-Z0-9_]*(?:::[A-Z][a-zA-Z0-9_]*)+)\s*\[`)
	assetRe       = regexp.MustCompile(`(?:template|epp|file)\s*\(\s*['"]([a-z0-9_]+/[^'"]+)['"]|puppet:///modules/([a-z0-9_]+/[^'"\s]+)`)
	hieraBlockRe  = regexp.MustCompile(`(?m)^(hiera_include|hiera_exclude):\s*$((?:\n\s+-\s*\S+)*)`)
	hieraItemRe   = regexp.MustCompile(`(?m)^\s+-\s*['"]?([^'"\s]+)['"]?\s*$`)
)

// Index is a static view of the control repo: which file defines which class,
// which classes reference which, which hieradata files pull classes in through
// hiera_include, and which manifest owns which template or file.
type Index struct {
	Root string

	// includedBy maps a class name to the classes that declare it.
	includedBy map[string]map[string]bool
	// moduleClasses maps a module name to the classes it defines.
	moduleClasses map[string][]string
	// assetOwners maps "<module>/<path>" to the classes referencing it.
	assetOwners map[string][]string
	// hieraInclude and hieraExclude map a class name to the hieradata files
	// listing it.
	hieraInclude map[string][]string
	hieraExclude map[string][]string
	// known holds every class this repo defines.
	known map[string]bool
}

// BuildIndex walks the control repo at root and indexes it.
func BuildIndex(root string) (*Index, error) {
	idx := &Index{
		Root:          root,
		includedBy:    map[string]map[string]bool{},
		moduleClasses: map[string][]string{},
		assetOwners:   map[string][]string{},
		hieraInclude:  map[string][]string{},
		hieraExclude:  map[string][]string{},
		known:         map[string]bool{},
	}

	for _, base := range manifestRoots {
		dir := filepath.Join(root, base)
		if _, err := os.Stat(dir); err != nil {
			continue
		}
		if err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
			if err != nil || d.IsDir() || !strings.HasSuffix(p, ".pp") {
				return err
			}
			rel, relErr := filepath.Rel(root, p)
			if relErr != nil {
				return relErr
			}
			return idx.addManifest(filepath.ToSlash(rel), p)
		}); err != nil {
			return nil, err
		}
	}

	hieraDir := filepath.Join(root, "hieradata")
	if _, err := os.Stat(hieraDir); err == nil {
		if err := filepath.WalkDir(hieraDir, func(p string, d fs.DirEntry, err error) error {
			if err != nil || d.IsDir() {
				return err
			}
			if ext := filepath.Ext(p); ext != ".yaml" && ext != ".eyaml" {
				return nil
			}
			rel, relErr := filepath.Rel(root, p)
			if relErr != nil {
				return relErr
			}
			return idx.addHieradata(filepath.ToSlash(rel), p)
		}); err != nil {
			return nil, err
		}
	}

	for module := range idx.moduleClasses {
		sort.Strings(idx.moduleClasses[module])
	}
	return idx, nil
}

func (i *Index) addManifest(rel, full string) error {
	parts := strings.Split(rel, "/")
	if len(parts) < 4 || parts[2] != "manifests" {
		return nil
	}
	module := parts[1]
	declared := ManifestClasses(module, parts[3:])
	for _, name := range declared {
		i.known[name] = true
		i.moduleClasses[module] = append(i.moduleClasses[module], name)
	}

	data, err := os.ReadFile(full) //nolint:gosec // paths come from walking the given repo
	if err != nil {
		return fmt.Errorf("reading %s: %w", rel, err)
	}
	body := commentRe.ReplaceAllString(string(data), "")

	self := map[string]bool{}
	for _, name := range declared {
		self[name] = true
	}
	for _, ref := range references(body) {
		if self[ref] {
			continue
		}
		if i.includedBy[ref] == nil {
			i.includedBy[ref] = map[string]bool{}
		}
		for _, name := range declared {
			i.includedBy[ref][name] = true
		}
	}

	for _, asset := range assetReferences(body) {
		i.assetOwners[asset] = append(i.assetOwners[asset], declared...)
	}
	return nil
}

// references extracts every class or defined-type name a manifest body names,
// across all the declaration forms this repo uses.
func references(body string) []string {
	seen := map[string]bool{}
	add := func(name string) {
		if name != "" && name != "class" && name != "define" {
			seen[name] = true
		}
	}
	for _, re := range []*regexp.Regexp{includeRe, classParamRe, classRefRe, defineUseRe, scopeVarRe, createResRe} {
		for _, m := range re.FindAllStringSubmatch(body, -1) {
			add(m[1])
		}
	}
	// Resource references (Profiles::Foo::Bar['x']) name a defined type in
	// capitalised form.
	for _, m := range resourceRefRe.FindAllStringSubmatch(body, -1) {
		add(strings.ToLower(m[1]))
	}

	out := make([]string, 0, len(seen))
	for name := range seen {
		out = append(out, name)
	}
	sort.Strings(out)
	return out
}

func assetReferences(body string) []string {
	var out []string
	for _, m := range assetRe.FindAllStringSubmatch(body, -1) {
		for _, g := range m[1:] {
			if g != "" {
				out = append(out, g)
			}
		}
	}
	return out
}

func (i *Index) addHieradata(rel, full string) error {
	data, err := os.ReadFile(full) //nolint:gosec // paths come from walking the given repo
	if err != nil {
		return fmt.Errorf("reading %s: %w", rel, err)
	}
	include, exclude := parseHieraLists(data)
	for _, name := range include {
		i.hieraInclude[name] = append(i.hieraInclude[name], rel)
	}
	for _, name := range exclude {
		i.hieraExclude[name] = append(i.hieraExclude[name], rel)
	}
	return nil
}

// parseHieraLists reads hiera_include and hiera_exclude. eyaml files can carry
// block scalars a strict parser chokes on, so a regex scan backs the YAML up.
func parseHieraLists(data []byte) (include, exclude []string) {
	var doc map[string]any
	if err := yaml.Unmarshal(data, &doc); err == nil && doc != nil {
		return stringList(doc["hiera_include"]), stringList(doc["hiera_exclude"])
	}
	for _, m := range hieraBlockRe.FindAllStringSubmatch(string(data), -1) {
		var items []string
		for _, item := range hieraItemRe.FindAllStringSubmatch(m[2], -1) {
			items = append(items, item[1])
		}
		if m[1] == "hiera_include" {
			include = append(include, items...)
		} else {
			exclude = append(exclude, items...)
		}
	}
	return include, exclude
}

func stringList(v any) []string {
	items, ok := v.([]any)
	if !ok {
		return nil
	}
	var out []string
	for _, item := range items {
		if s, ok := item.(string); ok && s != "" {
			out = append(out, s)
		}
	}
	return out
}

// IncludedBy returns the classes that declare the given class, sorted.
func (i *Index) IncludedBy(class string) []string {
	names := make([]string, 0, len(i.includedBy[class]))
	for name := range i.includedBy[class] {
		names = append(names, name)
	}
	sort.Strings(names)
	return names
}

// ModuleClasses returns every class the module defines, sorted.
func (i *Index) ModuleClasses(module string) []string { return i.moduleClasses[module] }

// AssetOwners returns the classes referencing a "<module>/<path>" template or
// file, sorted and deduplicated.
func (i *Index) AssetOwners(asset string) []string { return uniqueSorted(i.assetOwners[asset]) }

// HieraInclude returns the hieradata files listing the class under
// hiera_include, sorted.
func (i *Index) HieraInclude(class string) []string { return uniqueSorted(i.hieraInclude[class]) }

// HieraExclude returns the hieradata files listing the class under
// hiera_exclude, sorted.
func (i *Index) HieraExclude(class string) []string { return uniqueSorted(i.hieraExclude[class]) }

// Known reports whether the repo defines the class.
func (i *Index) Known(class string) bool { return i.known[class] }

func uniqueSorted(in []string) []string {
	if len(in) == 0 {
		return nil
	}
	seen := map[string]bool{}
	out := make([]string, 0, len(in))
	for _, s := range in {
		if !seen[s] {
			seen[s] = true
			out = append(out, s)
		}
	}
	sort.Strings(out)
	return out
}