puppet-catalog-tools/internal/impact/index.go

package impact

import (
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"gopkg.in/yaml.v3"
)

// manifestRoots are the modulepath entries that live in this repo.
// external_modules/ is Puppetfile-managed and not checked in.
var manifestRoots = []string{"site", "modules"}

var (
	// include/contain/require are functions taking a list of names; the
	// argument list is parsed rather than matched so every name is seen.
	declFuncRe = regexp.MustCompile(`(?:^|[^\w:$.])(?:include|contain|require)`)
	// inherits takes exactly one name, so a list parse does not apply.
	inheritsRe   = regexp.MustCompile(`\binherits\s+(?:::)?([a-z][a-z0-9_]*(?:::[a-z0-9_]+)*)`)
	classParamRe = regexp.MustCompile(`\bclass\s*\{`)
	classRefRe   = regexp.MustCompile(`\bClass\s*\[`)
	classNameRe  = regexp.MustCompile(`^(?:::)?[a-z][a-z0-9_]*(?:::[a-z0-9_]+)*$`)
	// a class name is case insensitive in a type reference or a resource
	// title, so Class['Mysql::server'] and include Mysql::Server both name
	// mysql::server.
	anyCaseNameRe = regexp.MustCompile(`^(?:::)?[A-Za-z][A-Za-z0-9_]*(?:::[A-Za-z0-9_]+)*$`)
	defineUseRe   = regexp.MustCompile(`(?m)(?:^|[^\w:$.])(?:::)?([a-z][a-z0-9_]*(?:::[a-z0-9_]+)+)\s*\{`)
	// $mod::params::thing reads a class variable, which is a dependency on the
	// class that declares it.
	scopeVarRe = regexp.MustCompile(`\$(?:::)?([a-z][a-z0-9_]*(?:::[a-z0-9_]+)+)::[a-z_][a-z0-9_]*`)
	// create_resources('mod::type', ...) names a defined type in a string.
	createResRe   = regexp.MustCompile(`create_resources\s*\(\s*['"](?:::)?([a-z][a-z0-9_]*(?:::[a-z0-9_]+)*)['"]`)
	resourceRefRe = regexp.MustCompile(`\b([A-Z][a-zA-Z0-9_]*(?:::[A-Z][a-zA-Z0-9_]*)+)\s*\[`)
	assetRe       = regexp.MustCompile(`(?:template|epp|file)\s*\(\s*['"]([a-z0-9_]+/[^'"]+)['"]|puppet:///modules/([a-z0-9_]+/[^'"\s]+)`)
	hieraBlockRe  = regexp.MustCompile(`(?m)^(hiera_include|hiera_exclude):\s*$((?:\n\s+-\s*\S+)*)`)
	hieraItemRe   = regexp.MustCompile(`(?m)^\s+-\s*['"]?([^'"\s]+)['"]?\s*$`)
)

// Index is a static view of the control repo: which file defines which class,
// which classes reference which, which hieradata files pull classes in through
// hiera_include, and which manifest owns which template or file.
type Index struct {
	Root string

	// includedBy maps a class name to the classes that reference it, and how.
	includedBy map[string]map[string]refKind
	// moduleClasses maps a module name to the classes it defines.
	moduleClasses map[string][]string
	// assetOwners maps "<module>/<path>" to the classes referencing it.
	assetOwners map[string][]string
	// hieraInclude and hieraExclude map a class name to the hieradata files
	// listing it.
	hieraInclude map[string][]string
	hieraExclude map[string][]string
	// known holds every class this repo defines.
	known map[string]bool
	// unresolved maps a manifest file to the class references in it that no
	// static resolution could name.
	unresolved map[string][]UnresolvedRef
	// unresolvedIncluders records, for each unresolved reference, the classes
	// whose manifest holds it, so a class the reference might have named still
	// reaches them.
	unresolvedIncluders []UnresolvedIncluder
}

// UnresolvedIncluder is an unresolved class reference together with the classes
// declared by the manifest holding it. Any class the reference could have named
// is treated as reaching those classes, because a dropped edge loses hosts.
type UnresolvedIncluder struct {
	Ref     UnresolvedRef
	Classes []string
}

// BuildIndex walks the control repo at root and indexes it.
func BuildIndex(root string) (*Index, error) {
	idx := &Index{
		Root:          root,
		includedBy:    map[string]map[string]refKind{},
		moduleClasses: map[string][]string{},
		assetOwners:   map[string][]string{},
		hieraInclude:  map[string][]string{},
		hieraExclude:  map[string][]string{},
		known:         map[string]bool{},
		unresolved:    map[string][]UnresolvedRef{},
	}

	for _, base := range manifestRoots {
		dir := filepath.Join(root, base)
		if _, err := os.Stat(dir); err != nil {
			continue
		}
		if err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
			if err != nil || d.IsDir() || !strings.HasSuffix(p, ".pp") {
				return err
			}
			rel, relErr := filepath.Rel(root, p)
			if relErr != nil {
				return relErr
			}
			return idx.addManifest(filepath.ToSlash(rel), p)
		}); err != nil {
			return nil, err
		}
	}

	hieraDir := filepath.Join(root, "hieradata")
	if _, err := os.Stat(hieraDir); err == nil {
		if err := filepath.WalkDir(hieraDir, func(p string, d fs.DirEntry, err error) error {
			if err != nil || d.IsDir() {
				return err
			}
			if ext := filepath.Ext(p); ext != ".yaml" && ext != ".eyaml" {
				return nil
			}
			rel, relErr := filepath.Rel(root, p)
			if relErr != nil {
				return relErr
			}
			return idx.addHieradata(filepath.ToSlash(rel), p)
		}); err != nil {
			return nil, err
		}
	}

	for module := range idx.moduleClasses {
		sort.Strings(idx.moduleClasses[module])
	}
	return idx, nil
}

func (i *Index) addManifest(rel, full string) error {
	parts := strings.Split(rel, "/")
	if len(parts) < 4 || parts[2] != "manifests" {
		return nil
	}
	module := parts[1]
	declared := ManifestClasses(module, parts[3:])
	for _, name := range declared {
		i.known[name] = true
		i.moduleClasses[module] = append(i.moduleClasses[module], name)
	}

	data, err := os.ReadFile(full) //nolint:gosec // paths come from walking the given repo
	if err != nil {
		return fmt.Errorf("reading %s: %w", rel, err)
	}
	body := stripSource(string(data))

	self := map[string]bool{}
	for _, name := range declared {
		self[name] = true
	}
	refs, unresolved := scanReferences(body)
	for ref, kind := range refs {
		if self[ref] {
			continue
		}
		if i.includedBy[ref] == nil {
			i.includedBy[ref] = map[string]refKind{}
		}
		for _, name := range declared {
			if old, ok := i.includedBy[ref][name]; !ok || kind > old {
				i.includedBy[ref][name] = kind
			}
		}
	}

	for idx := range unresolved {
		unresolved[idx].File = rel
	}
	if len(unresolved) > 0 && len(declared) > 0 {
		i.unresolved[rel] = unresolved
		for _, ref := range unresolved {
			i.unresolvedIncluders = append(i.unresolvedIncluders,
				UnresolvedIncluder{Ref: ref, Classes: declared})
		}
	}

	for _, asset := range assetReferences(body) {
		i.assetOwners[asset] = append(i.assetOwners[asset], declared...)
	}
	return nil
}

func assetReferences(body string) []string {
	var out []string
	for _, m := range assetRe.FindAllStringSubmatch(body, -1) {
		for _, g := range m[1:] {
			if g != "" {
				out = append(out, g)
			}
		}
	}
	return out
}

func (i *Index) addHieradata(rel, full string) error {
	data, err := os.ReadFile(full) //nolint:gosec // paths come from walking the given repo
	if err != nil {
		return fmt.Errorf("reading %s: %w", rel, err)
	}
	include, exclude := parseHieraLists(data)
	for _, name := range include {
		i.hieraInclude[name] = append(i.hieraInclude[name], rel)
	}
	for _, name := range exclude {
		i.hieraExclude[name] = append(i.hieraExclude[name], rel)
	}
	return nil
}

// parseHieraLists reads hiera_include and hiera_exclude. eyaml files can carry
// block scalars a strict parser chokes on, so a regex scan backs the YAML up.
func parseHieraLists(data []byte) (include, exclude []string) {
	var doc map[string]any
	if err := yaml.Unmarshal(data, &doc); err == nil && doc != nil {
		return stringList(doc["hiera_include"]), stringList(doc["hiera_exclude"])
	}
	for _, m := range hieraBlockRe.FindAllStringSubmatch(string(data), -1) {
		var items []string
		for _, item := range hieraItemRe.FindAllStringSubmatch(m[2], -1) {
			items = append(items, item[1])
		}
		if m[1] == "hiera_include" {
			include = append(include, items...)
		} else {
			exclude = append(exclude, items...)
		}
	}
	return include, exclude
}

func stringList(v any) []string {
	items, ok := v.([]any)
	if !ok {
		return nil
	}
	var out []string
	for _, item := range items {
		if s, ok := item.(string); ok && s != "" {
			out = append(out, s)
		}
	}
	return out
}

// IncludedBy returns the classes that reference the given class, sorted.
func (i *Index) IncludedBy(class string) []string {
	names := make([]string, 0, len(i.includedBy[class]))
	for name := range i.includedBy[class] {
		names = append(names, name)
	}
	sort.Strings(names)
	return names
}

// EdgeNote describes an edge that is not a declaration, so an explanation can
// say why the class is reached. It is empty for a plain declaration.
func (i *Index) EdgeNote(class, parent string) string {
	if i.includedBy[class][parent] == refDeclare {
		return ""
	}
	return " (ordering or data reference, not a declaration)"
}

// UnresolvedIncluders returns the unresolved references that could have named
// the given class, matched on whatever literal prefix was recoverable.
func (i *Index) UnresolvedIncluders(class string) []UnresolvedIncluder {
	var out []UnresolvedIncluder
	for _, u := range i.unresolvedIncluders {
		if strings.HasPrefix(class, u.Ref.Prefix) {
			out = append(out, u)
		}
	}
	return out
}

// UnresolvedIn returns the class references in a manifest file that no static
// resolution could name.
func (i *Index) UnresolvedIn(file string) []UnresolvedRef { return i.unresolved[file] }

// ModuleClasses returns every class the module defines, sorted.
func (i *Index) ModuleClasses(module string) []string { return i.moduleClasses[module] }

// AssetOwners returns the classes referencing a "<module>/<path>" template or
// file, sorted and deduplicated.
func (i *Index) AssetOwners(asset string) []string { return uniqueSorted(i.assetOwners[asset]) }

// HieraInclude returns the hieradata files listing the class under
// hiera_include, sorted.
func (i *Index) HieraInclude(class string) []string { return uniqueSorted(i.hieraInclude[class]) }

// HieraExclude returns the hieradata files listing the class under
// hiera_exclude, sorted.
func (i *Index) HieraExclude(class string) []string { return uniqueSorted(i.hieraExclude[class]) }

// Known reports whether the repo defines the class.
func (i *Index) Known(class string) bool { return i.known[class] }

func uniqueSorted(in []string) []string {
	if len(in) == 0 {
		return nil
	}
	seen := map[string]bool{}
	out := make([]string, 0, len(in))
	for _, s := range in {
		if !seen[s] {
			seen[s] = true
			out = append(out, s)
		}
	}
	sort.Strings(out)
	return out
}