Kinds: Generate k8 resources without use kindys/thema (#83310)

Generate k8 resources reading cue file directly instead of use thema/kindsys binding

kinds/gen.go

@@ -15,6 +15,7 @@ import (
 	"sort"
 	"strings"
 
+	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/errors"
 	"github.com/grafana/codejen"
 	"github.com/grafana/cuetsy"
@@ -38,8 +39,6 @@ func main() {
 
 	// All the jennies that comprise the core kinds generator pipeline
 	coreKindsGen.Append(
-		&codegen.ResourceGoTypesJenny{},
-		&codegen.SubresourceGoTypesJenny{},
 		codegen.CoreKindJenny(cuectx.GoCoreKindParentPath, nil),
 		codegen.BaseCoreRegistryJenny(filepath.Join("pkg", "registry", "corekind"), cuectx.GoCoreKindParentPath),
 		codegen.LatestMajorsOrXJenny(
@@ -95,6 +94,16 @@ func main() {
 		die(err)
 	}
 
+	// Merging k8 resources
+	k8Resources, err := genK8Resources(kinddirs)
+	if err != nil {
+		die(err)
+	}
+
+	if err = jfs.Merge(k8Resources); err != nil {
+		die(err)
+	}
+
 	if _, set := os.LookupEnv("CODEGEN_VERIFY"); set {
 		if err = jfs.Verify(context.Background(), groot); err != nil {
 			die(fmt.Errorf("generated code is out of sync with inputs:\n%s\nrun `make gen-cue` to regenerate", err))
@@ -181,3 +190,44 @@ func die(err error) {
 	fmt.Fprint(os.Stderr, err, "\n")
 	os.Exit(1)
 }
+
+func genK8Resources(dirs []os.DirEntry) (*codejen.FS, error) {
+	jenny := codejen.JennyListWithNamer[[]cue.Value](func(_ []cue.Value) string {
+		return "K8Resources"
+	})
+
+	jenny.Append(&codegen.K8ResourcesJenny{})
+
+	header := codegen.SlashHeaderMapper("kinds/gen.go")
+	jenny.AddPostprocessors(header)
+
+	return jenny.GenerateFS(loadCueFiles(dirs))
+}
+
+func loadCueFiles(dirs []os.DirEntry) []cue.Value {
+	ctx := cuectx.GrafanaCUEContext()
+	values := make([]cue.Value, 0)
+	for _, dir := range dirs {
+		if !dir.IsDir() {
+			continue
+		}
+
+		entries, err := os.ReadDir(dir.Name())
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "error opening %s directory: %s", dir, err)
+			os.Exit(1)
+		}
+
+		// It's assuming that we only have one file in each folder
+		entry := filepath.Join(dir.Name(), entries[0].Name())
+		cueFile, err := os.ReadFile(entry)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to open %s/%s file: %s", dir, entries[0].Name(), err)
+			os.Exit(1)
+		}
+
+		values = append(values, ctx.CompileBytes(cueFile))
+	}
+
+	return values
+}

pkg/codegen/jenny_go_resources.go

@@ -1,128 +0,0 @@
-package codegen
-
-import (
-	"bytes"
-	"fmt"
-	"go/format"
-	"strings"
-
-	"cuelang.org/go/cue"
-	"github.com/dave/dst/dstutil"
-	"github.com/grafana/codejen"
-	"github.com/grafana/kindsys"
-	"github.com/grafana/thema/encoding/gocode"
-	"github.com/grafana/thema/encoding/openapi"
-)
-
-var schPath = cue.MakePath(cue.Hid("_#schema", "github.com/grafana/thema"))
-
-type ResourceGoTypesJenny struct {
-	ApplyFuncs       []dstutil.ApplyFunc
-	ExpandReferences bool
-}
-
-func (*ResourceGoTypesJenny) JennyName() string {
-	return "GoTypesJenny"
-}
-
-func (ag *ResourceGoTypesJenny) Generate(kind kindsys.Kind) (*codejen.File, error) {
-	comm := kind.Props().Common()
-	sfg := SchemaForGen{
-		Name:    comm.Name,
-		Schema:  kind.Lineage().Latest(),
-		IsGroup: comm.LineageIsGroup,
-	}
-	sch := sfg.Schema
-
-	iter, err := sch.Underlying().LookupPath(schPath).Fields()
-	if err != nil {
-		return nil, err
-	}
-
-	var subr []string
-	for iter.Next() {
-		subr = append(subr, typeNameFromKey(iter.Selector().String()))
-	}
-
-	buf := new(bytes.Buffer)
-	mname := kind.Props().Common().MachineName
-	if err := tmpls.Lookup("core_resource.tmpl").Execute(buf, tvars_resource{
-		PackageName:      mname,
-		KindName:         kind.Props().Common().Name,
-		Version:          strings.Replace(sfg.Schema.Version().String(), ".", "-", -1),
-		SubresourceNames: subr,
-	}); err != nil {
-		return nil, fmt.Errorf("failed executing core resource template: %w", err)
-	}
-
-	if err != nil {
-		return nil, err
-	}
-
-	content, err := format.Source(buf.Bytes())
-	if err != nil {
-		return nil, err
-	}
-
-	return codejen.NewFile(fmt.Sprintf("pkg/kinds/%s/%s_gen.go", mname, mname), content, ag), nil
-}
-
-type SubresourceGoTypesJenny struct {
-	ApplyFuncs       []dstutil.ApplyFunc
-	ExpandReferences bool
-}
-
-func (*SubresourceGoTypesJenny) JennyName() string {
-	return "GoResourceTypes"
-}
-
-func (g *SubresourceGoTypesJenny) Generate(kind kindsys.Kind) (codejen.Files, error) {
-	comm := kind.Props().Common()
-	sfg := SchemaForGen{
-		Name:    comm.Name,
-		Schema:  kind.Lineage().Latest(),
-		IsGroup: comm.LineageIsGroup,
-	}
-	sch := sfg.Schema
-
-	// Iterate through all top-level fields and make go types for them
-	// (this should consist of "spec" and arbitrary subresources)
-	i, err := sch.Underlying().LookupPath(schPath).Fields()
-	if err != nil {
-		return nil, err
-	}
-	files := make(codejen.Files, 0)
-	for i.Next() {
-		str := i.Selector().String()
-
-		b, err := gocode.GenerateTypesOpenAPI(sch, &gocode.TypeConfigOpenAPI{
-			// TODO will need to account for sanitizing e.g. dashes here at some point
-			Config: &openapi.Config{
-				Group:    false, // TODO: better
-				RootName: typeNameFromKey(str),
-				Subpath:  cue.MakePath(cue.Str(str)),
-			},
-			PackageName: sfg.Schema.Lineage().Name(),
-			ApplyFuncs:  append(g.ApplyFuncs, PrefixDropper(sfg.Name)),
-		})
-		if err != nil {
-			return nil, err
-		}
-
-		name := sfg.Schema.Lineage().Name()
-		files = append(files, codejen.File{
-			RelativePath: fmt.Sprintf("pkg/kinds/%s/%s_%s_gen.go", name, name, strings.ToLower(str)),
-			Data:         b,
-			From:         []codejen.NamedJenny{g},
-		})
-	}
-
-	return files, nil
-}
-
-func typeNameFromKey(key string) string {
-	if len(key) > 0 {
-		return strings.ToUpper(key[:1]) + key[1:]
-	}
-	return strings.ToUpper(key)
-}

pkg/codegen/jenny_k8_resources.go

@@ -0,0 +1,130 @@
+package codegen
+
+import (
+	"bytes"
+	"fmt"
+	"go/format"
+	"strings"
+
+	"cuelang.org/go/cue"
+	"github.com/grafana/codejen"
+)
+
+// K8ResourcesJenny generates resource, metadata and status for each file.
+type K8ResourcesJenny struct {
+}
+
+func (jenny *K8ResourcesJenny) JennyName() string {
+	return "K8ResourcesJenny"
+}
+
+func (jenny *K8ResourcesJenny) Generate(cueFiles []cue.Value) (codejen.Files, error) {
+	files := make(codejen.Files, 0)
+	for _, val := range cueFiles {
+		pkg, err := getPackageName(val)
+		if err != nil {
+			return nil, err
+		}
+
+		resource, err := jenny.genResource(pkg, val)
+		if err != nil {
+			return nil, err
+		}
+
+		metadata, err := jenny.genMetadata(pkg)
+		if err != nil {
+			return nil, err
+		}
+
+		status, err := jenny.genStatus(pkg)
+		if err != nil {
+			return nil, err
+		}
+
+		files = append(files, resource)
+		files = append(files, metadata)
+		files = append(files, status)
+	}
+
+	return files, nil
+}
+
+func (jenny *K8ResourcesJenny) genResource(pkg string, val cue.Value) (codejen.File, error) {
+	version, err := getVersion(val)
+	if err != nil {
+		return codejen.File{}, err
+	}
+
+	pkgName := strings.ToLower(pkg)
+
+	buf := new(bytes.Buffer)
+	if err := tmpls.Lookup("core_resource.tmpl").Execute(buf, tvars_resource{
+		PackageName: pkgName,
+		KindName:    pkg,
+		Version:     version,
+	}); err != nil {
+		return codejen.File{}, fmt.Errorf("failed executing core resource template: %w", err)
+	}
+
+	content, err := format.Source(buf.Bytes())
+	if err != nil {
+		return codejen.File{}, err
+	}
+
+	return *codejen.NewFile(fmt.Sprintf("pkg/kinds/%s/%s_gen.go", pkgName, pkgName), content, jenny), nil
+}
+
+func (jenny *K8ResourcesJenny) genMetadata(pkg string) (codejen.File, error) {
+	pkg = strings.ToLower(pkg)
+
+	buf := new(bytes.Buffer)
+	if err := tmpls.Lookup("core_metadata.tmpl").Execute(buf, tvars_metadata{
+		PackageName: pkg,
+	}); err != nil {
+		return codejen.File{}, fmt.Errorf("failed executing core resource template: %w", err)
+	}
+
+	return *codejen.NewFile(fmt.Sprintf("pkg/kinds/%s/%s_metadata_gen.go", pkg, pkg), buf.Bytes(), jenny), nil
+}
+
+func (jenny *K8ResourcesJenny) genStatus(pkg string) (codejen.File, error) {
+	pkg = strings.ToLower(pkg)
+
+	buf := new(bytes.Buffer)
+	if err := tmpls.Lookup("core_status.tmpl").Execute(buf, tvars_status{
+		PackageName: pkg,
+	}); err != nil {
+		return codejen.File{}, fmt.Errorf("failed executing core resource template: %w", err)
+	}
+
+	return *codejen.NewFile(fmt.Sprintf("pkg/kinds/%s/%s_status_gen.go", pkg, pkg), buf.Bytes(), jenny), nil
+}
+
+func getPackageName(val cue.Value) (string, error) {
+	name := val.LookupPath(cue.ParsePath("name"))
+	pkg, err := name.String()
+	if err != nil {
+		return "", fmt.Errorf("file doesn't have name field set: %s", err)
+	}
+	return pkg, nil
+}
+
+func getVersion(val cue.Value) (string, error) {
+	val = val.LookupPath(cue.ParsePath("lineage.schemas[0].version"))
+	versionValues, err := val.List()
+	if err != nil {
+		return "", fmt.Errorf("missing version in schema: %s", err)
+	}
+
+	version := make([]int64, 0)
+	for versionValues.Next() {
+		v, err := versionValues.Value().Int64()
+		if err != nil {
+			return "", fmt.Errorf("version should be a list of two elements: %s", err)
+		}
+
+		version = append(version, v)
+	}
+
+	return fmt.Sprintf("%d-%d", version[0], version[1]), nil
+}

pkg/codegen/jenny_tsveneerindex.go

@@ -18,6 +18,8 @@ import (
 	"github.com/grafana/thema/encoding/typescript"
 )
 
+var schPath = cue.MakePath(cue.Hid("_#schema", "github.com/grafana/thema"))
+
 // TSVeneerIndexJenny generates an index.gen.ts file with references to all
 // generated TS types. Elements with the attribute @grafana(TSVeneer="type") are
 // exported from a handwritten file, rather than the raw generated types.

pkg/codegen/tmpl.go

@@ -39,9 +39,16 @@ type (
 		Kinds             []kindsys.Core
 	}
 	tvars_resource struct {
-		PackageName      string
+		PackageName string
-		KindName         string
+		KindName    string
-		Version          string
+		Version     string
-		SubresourceNames []string
+	}
+
+	tvars_metadata struct {
+		PackageName string
+	}
+
+	tvars_status struct {
+		PackageName string
 	}
 )

pkg/codegen/tmpl/core_metadata.tmpl

@@ -0,0 +1,33 @@
+package {{ .PackageName }}
+
+import (
+	"time"
+)
+
+// Metadata defines model for Metadata.
+type Metadata struct {
+	CreatedBy         string     `json:"createdBy"`
+	CreationTimestamp time.Time  `json:"creationTimestamp"`
+	DeletionTimestamp *time.Time `json:"deletionTimestamp,omitempty"`
+
+	// extraFields is reserved for any fields that are pulled from the API server metadata but do not have concrete fields in the CUE metadata
+	ExtraFields     map[string]any    `json:"extraFields"`
+	Finalizers      []string          `json:"finalizers"`
+	Labels          map[string]string `json:"labels"`
+	ResourceVersion string            `json:"resourceVersion"`
+	Uid             string            `json:"uid"`
+	UpdateTimestamp time.Time         `json:"updateTimestamp"`
+	UpdatedBy       string            `json:"updatedBy"`
+}
+
+// _kubeObjectMetadata is metadata found in a kubernetes object's metadata field.
+// It is not exhaustive and only includes fields which may be relevant to a kind's implementation,
+// As it is also intended to be generic enough to function with any API Server.
+type KubeObjectMetadata struct {
+	CreationTimestamp time.Time         `json:"creationTimestamp"`
+	DeletionTimestamp *time.Time        `json:"deletionTimestamp,omitempty"`
+	Finalizers        []string          `json:"finalizers"`
+	Labels            map[string]string `json:"labels"`
+	ResourceVersion   string            `json:"resourceVersion"`
+	Uid               string            `json:"uid"`
+}

pkg/codegen/tmpl/core_resource.tmpl

@@ -28,6 +28,7 @@ func NewK8sResource(name string, s *Spec) K8sResource {
 // Resource is the wire representation of {{ .KindName }}.
 // It currently will soon be merged into the k8s flavor (TODO be better)
 type Resource struct {
-    {{- range .SubresourceNames }}
+    Metadata Metadata `json:"metadata"`
-	{{ . }} {{ . }} `json:"{{ . | ToLower }}"`{{end}}
+    Spec     Spec     `json:"spec"`
+    Status   Status   `json:"status"`
 }

pkg/codegen/tmpl/core_status.tmpl

@@ -0,0 +1,65 @@
+package {{ .PackageName }}
+
+// Defines values for OperatorStateState.
+const (
+	OperatorStateStateFailed     OperatorStateState = "failed"
+	OperatorStateStateInProgress OperatorStateState = "in_progress"
+	OperatorStateStateSuccess    OperatorStateState = "success"
+)
+
+// Defines values for StatusOperatorStateState.
+const (
+	StatusOperatorStateStateFailed     StatusOperatorStateState = "failed"
+	StatusOperatorStateStateInProgress StatusOperatorStateState = "in_progress"
+	StatusOperatorStateStateSuccess    StatusOperatorStateState = "success"
+)
+
+// OperatorState defines model for OperatorState.
+type OperatorState struct {
+	// descriptiveState is an optional more descriptive state field which has no requirements on format
+	DescriptiveState *string `json:"descriptiveState,omitempty"`
+
+	// details contains any extra information that is operator-specific
+	Details map[string]any `json:"details,omitempty"`
+
+	// lastEvaluation is the ResourceVersion last evaluated
+	LastEvaluation string `json:"lastEvaluation"`
+
+	// state describes the state of the lastEvaluation.
+	// It is limited to three possible states for machine evaluation.
+	State OperatorStateState `json:"state"`
+}
+
+// OperatorStateState state describes the state of the lastEvaluation.
+// It is limited to three possible states for machine evaluation.
+type OperatorStateState string
+
+// Status defines model for Status.
+type Status struct {
+	// additionalFields is reserved for future use
+	AdditionalFields map[string]any `json:"additionalFields,omitempty"`
+
+	// operatorStates is a map of operator ID to operator state evaluations.
+	// Any operator which consumes this kind SHOULD add its state evaluation information to this field.
+	OperatorStates map[string]StatusOperatorState `json:"operatorStates,omitempty"`
+}
+
+// StatusOperatorState defines model for status.#OperatorState.
+type StatusOperatorState struct {
+	// descriptiveState is an optional more descriptive state field which has no requirements on format
+	DescriptiveState *string `json:"descriptiveState,omitempty"`
+
+	// details contains any extra information that is operator-specific
+	Details map[string]any `json:"details,omitempty"`
+
+	// lastEvaluation is the ResourceVersion last evaluated
+	LastEvaluation string `json:"lastEvaluation"`
+
+	// state describes the state of the lastEvaluation.
+	// It is limited to three possible states for machine evaluation.
+	State StatusOperatorStateState `json:"state"`
+}
+
+// StatusOperatorStateState state describes the state of the lastEvaluation.
+// It is limited to three possible states for machine evaluation.
+type StatusOperatorStateState string

pkg/kinds/accesspolicy/accesspolicy_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoTypesJenny
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/accesspolicy/accesspolicy_metadata_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/accesspolicy/accesspolicy_status_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/dashboard/dashboard_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoTypesJenny
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/dashboard/dashboard_metadata_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/dashboard/dashboard_status_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/librarypanel/librarypanel_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoTypesJenny
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/librarypanel/librarypanel_metadata_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/librarypanel/librarypanel_status_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/preferences/preferences_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoTypesJenny
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/preferences/preferences_metadata_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/preferences/preferences_status_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/publicdashboard/publicdashboard_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoTypesJenny
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/publicdashboard/publicdashboard_metadata_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/publicdashboard/publicdashboard_status_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/role/role_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoTypesJenny
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/role/role_metadata_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/role/role_status_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/rolebinding/rolebinding_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoTypesJenny
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/rolebinding/rolebinding_metadata_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/rolebinding/rolebinding_status_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/team/team_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoTypesJenny
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/team/team_metadata_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/team/team_status_gen.go

@@ -3,7 +3,7 @@
 // Generated by:
 //     kinds/gen.go
 // Using jennies:
-//     GoResourceTypes
+//     K8ResourcesJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.