From 42baad837a6241acbd59b266311d291345a1e4a3 Mon Sep 17 00:00:00 2001 From: sam boyer Date: Tue, 22 Nov 2022 09:00:29 -0500 Subject: [PATCH] codegen: Refactor core jennies for reusability, add version-picking metajennies (#58995) * Add each-major jenny, refactor TS jenny for it * Introduce LatestJenny, refactor GoTypesJenny for it --- kinds/gen.go | 11 +-- .../raw/dashboard/x/dashboard_types.gen.ts | 1 + .../src/raw/playlist/x/playlist_types.gen.ts | 1 + pkg/codegen/generators.go | 48 ++++++---- pkg/codegen/jenny_eachmajor.go | 74 +++++++++++++++ pkg/codegen/jenny_gotypes.go | 89 +++---------------- pkg/codegen/jenny_tstypes.go | 79 +++------------- pkg/codegen/latest_jenny.go | 56 ++++++++++++ pkg/codegen/tmpl.go | 7 ++ pkg/codegen/tmpl/gen_header.tmpl | 11 +++ pkg/kinds/dashboard/dashboard_types_gen.go | 1 + pkg/kinds/playlist/playlist_types_gen.go | 1 + 12 files changed, 206 insertions(+), 173 deletions(-) create mode 100644 pkg/codegen/jenny_eachmajor.go create mode 100644 pkg/codegen/latest_jenny.go create mode 100644 pkg/codegen/tmpl/gen_header.tmpl diff --git a/kinds/gen.go b/kinds/gen.go index 41f0d527049..f9981203e17 100644 --- a/kinds/gen.go +++ b/kinds/gen.go @@ -21,8 +21,6 @@ import ( "github.com/grafana/grafana/pkg/kindsys" ) -const sep = string(filepath.Separator) - func main() { if len(os.Args) > 1 { fmt.Fprintf(os.Stderr, "plugin thema code generator does not currently accept any arguments\n, got %q", os.Args) @@ -37,16 +35,11 @@ func main() { // All the jennies that comprise the core kinds generator pipeline coreKindsGen.Append( - codegen.GoTypesJenny(kindsys.GoCoreKindParentPath, nil), + codegen.LatestJenny(kindsys.GoCoreKindParentPath, codegen.GoTypesJenny{}), codegen.CoreStructuredKindJenny(kindsys.GoCoreKindParentPath, nil), codegen.RawKindJenny(kindsys.GoCoreKindParentPath, nil), codegen.BaseCoreRegistryJenny(filepath.Join("pkg", "registry", "corekind"), kindsys.GoCoreKindParentPath), - codegen.TSTypesJenny(kindsys.TSCoreKindParentPath, &codegen.TSTypesGeneratorConfig{ - GenDirName: func(decl *codegen.DeclForGen) string { - // FIXME this hardcodes always generating to experimental dir. OK for now, but need generator fanout - return filepath.Join(decl.Meta.Common().MachineName, "x") - }, - }), + codegen.LatestMajorsOrXJenny(kindsys.TSCoreKindParentPath, codegen.TSTypesJenny{}), codegen.TSVeneerIndexJenny(filepath.Join("packages", "grafana-schema", "src")), ) diff --git a/packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts b/packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts index 7bcf55da260..f89a0ae4abf 100644 --- a/packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts +++ b/packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts @@ -4,6 +4,7 @@ // kinds/gen.go // Using jennies: // TSTypesJenny +// LatestMajorsOrXJenny // // Run 'make gen-cue' from repository root to regenerate. diff --git a/packages/grafana-schema/src/raw/playlist/x/playlist_types.gen.ts b/packages/grafana-schema/src/raw/playlist/x/playlist_types.gen.ts index 1e1ceeabd8d..a4fe7fc3d15 100644 --- a/packages/grafana-schema/src/raw/playlist/x/playlist_types.gen.ts +++ b/packages/grafana-schema/src/raw/playlist/x/playlist_types.gen.ts @@ -4,6 +4,7 @@ // kinds/gen.go // Using jennies: // TSTypesJenny +// LatestMajorsOrXJenny // // Run 'make gen-cue' from repository root to regenerate. diff --git a/pkg/codegen/generators.go b/pkg/codegen/generators.go index 11820333f7a..1a56f71a5b4 100644 --- a/pkg/codegen/generators.go +++ b/pkg/codegen/generators.go @@ -57,7 +57,6 @@ func StructuredForGen(k kindsys.Structured) *DeclForGen { type DeclForGen struct { *kindsys.SomeDecl lin thema.Lineage - sch thema.Lineage } // Lineage returns the [thema.Lineage] for the underlying [kindsys.SomeDecl]. @@ -65,10 +64,15 @@ func (decl *DeclForGen) Lineage() thema.Lineage { return decl.lin } -// Schema returns the [thema.Schema] that a jenny should operate against, for those -// jennies that target a single schema. -func (decl *DeclForGen) Schema() thema.Lineage { - return decl.sch +// ForLatestSchema returns a [SchemaForGen] for the latest schema in this +// DeclForGen's lineage. +func (decl *DeclForGen) ForLatestSchema() SchemaForGen { + comm := decl.Meta.Common() + return SchemaForGen{ + Name: comm.Name, + Schema: decl.Lineage().Latest(), + IsGroup: comm.LineageIsGroup, + } } // SlashHeaderMapper produces a FileMapper that injects a comment header onto @@ -82,22 +86,28 @@ func SlashHeaderMapper(maingen string) codejen.FileMapper { case ".json", ".yml", ".yaml": return f, nil default: - b := new(bytes.Buffer) - fmt.Fprintf(b, headerTmpl, filepath.ToSlash(maingen), f.FromString()) - fmt.Fprint(b, string(f.Data)) - f.Data = b.Bytes() + buf := new(bytes.Buffer) + if err := tmpls.Lookup("gen_header.tmpl").Execute(buf, tvars_gen_header{ + MainGenerator: maingen, + Using: f.From, + }); err != nil { + return codejen.File{}, fmt.Errorf("failed executing gen header template: %w", err) + } + fmt.Fprint(buf, string(f.Data)) + f.Data = buf.Bytes() } return f, nil } } -var headerTmpl = `// THIS FILE IS GENERATED. EDITING IS FUTILE. -// -// Generated by: -// %s -// Using jennies: -// %s -// -// Run 'make gen-cue' from repository root to regenerate. - -` +// SchemaForGen is an intermediate values type for jennies that holds both a thema.Schema, +// and values relevant to generating the schema that should properly, eventually, be in +// thema itself. +type SchemaForGen struct { + // The PascalCase name of the schematized type. + Name string + // The schema to be rendered for the type itself. + Schema thema.Schema + // Whether the schema is grouped. See https://github.com/grafana/thema/issues/62 + IsGroup bool +} diff --git a/pkg/codegen/jenny_eachmajor.go b/pkg/codegen/jenny_eachmajor.go new file mode 100644 index 00000000000..766fcad5bf0 --- /dev/null +++ b/pkg/codegen/jenny_eachmajor.go @@ -0,0 +1,74 @@ +package codegen + +import ( + "fmt" + "path/filepath" + + "github.com/grafana/codejen" + "github.com/grafana/grafana/pkg/kindsys" +) + +// LatestMajorsOrXJenny returns a jenny that repeats the input for the latest in each major version, +func LatestMajorsOrXJenny(parentdir string, inner codejen.OneToOne[SchemaForGen]) OneToMany { + if inner == nil { + panic("inner jenny must not be nil") + } + + return &lmox{ + parentdir: parentdir, + inner: inner, + } +} + +type lmox struct { + parentdir string + inner codejen.OneToOne[SchemaForGen] +} + +func (j *lmox) JennyName() string { + return "LatestMajorsOrXJenny" +} + +func (j *lmox) Generate(decl *DeclForGen) (codejen.Files, error) { + if decl.IsRaw() { + return nil, nil + } + comm := decl.Meta.Common() + sfg := SchemaForGen{ + Name: comm.Name, + IsGroup: comm.LineageIsGroup, + } + + do := func(sfg SchemaForGen, infix string) (codejen.Files, error) { + f, err := j.inner.Generate(sfg) + if err != nil { + return nil, fmt.Errorf("%s jenny failed on %s schema for %s: %w", j.inner.JennyName(), sfg.Schema.Version(), decl.Meta.Common().Name, err) + } + if f == nil || !f.Exists() { + return nil, nil + } + + f.RelativePath = filepath.Join(j.parentdir, comm.MachineName, infix, f.RelativePath) + f.From = append(f.From, j) + return codejen.Files{*f}, nil + } + + if comm.Maturity.Less(kindsys.MaturityStable) { + sfg.Schema = decl.Lineage().Latest() + return do(sfg, "x") + } + + var fl codejen.Files + for sch := decl.Lineage().First(); sch != nil; sch.Successor() { + sfg.Schema = sch.LatestInMajor() + files, err := do(sfg, fmt.Sprintf("v%v", sch.Version()[0])) + if err != nil { + return nil, err + } + fl = append(fl, files...) + } + if fl.Validate() != nil { + return nil, fl.Validate() + } + return fl, nil +} diff --git a/pkg/codegen/jenny_gotypes.go b/pkg/codegen/jenny_gotypes.go index 5d933ee6857..c662455eda4 100644 --- a/pkg/codegen/jenny_gotypes.go +++ b/pkg/codegen/jenny_gotypes.go @@ -1,98 +1,29 @@ package codegen import ( - "fmt" - "path/filepath" - "github.com/grafana/codejen" - "github.com/grafana/thema" "github.com/grafana/thema/encoding/gocode" "golang.org/x/tools/go/ast/astutil" ) -// GoTypesJenny creates a [OneToOne] that produces Go types for the latest -// Thema schema in a structured kind's lineage. -// -// At minimum, a gokindsdir must be provided. This should be the path to the parent -// directory of the directory in which the types should be generated, relative -// to the project root. For example, if the types for a kind named "foo" -// should live at pkg/kind/foo/foo_gen.go, relpath should be "pkg/kind". -// -// This generator is a no-op for raw kinds. -func GoTypesJenny(gokindsdir string, cfg *GoTypesGeneratorConfig) OneToOne { - if cfg == nil { - cfg = new(GoTypesGeneratorConfig) - } - if cfg.GenDirName == nil { - cfg.GenDirName = func(decl *DeclForGen) string { - return decl.Meta.Common().MachineName - } - } +// GoTypesJenny creates a [OneToOne] that produces Go types for the provided +// [thema.Schema]. +type GoTypesJenny struct{} - return &genGoTypes{ - gokindsdir: gokindsdir, - cfg: cfg, - } -} - -// GoTypesGeneratorConfig holds configuration options for [GoTypesJenny]. -type GoTypesGeneratorConfig struct { - // Apply is an optional AST manipulation func that, if provided, will be run - // against the generated Go file prior to running it through goimports. - Apply astutil.ApplyFunc - - // GenDirName returns the name of the parent directory in which the type file - // should be generated. If nil, the DeclForGen.Lineage().Name() will be used. - GenDirName func(*DeclForGen) string - - // Version of the schema to generate. If nil, latest is generated. - Version *thema.SyntacticVersion -} - -type genGoTypes struct { - gokindsdir string - cfg *GoTypesGeneratorConfig -} - -func (gen *genGoTypes) JennyName() string { +func (j GoTypesJenny) JennyName() string { return "GoTypesJenny" } -func (gen *genGoTypes) Generate(decl *DeclForGen) (*codejen.File, error) { - if decl.IsRaw() { - return nil, nil - } - - var sch thema.Schema - var err error - - lin := decl.Lineage() - if gen.cfg.Version == nil { - sch = lin.Latest() - } else { - sch, err = lin.Schema(*gen.cfg.Version) - if err != nil { - return nil, fmt.Errorf("error in configured version for %s generator: %w", *gen.cfg.Version, err) - } - } - - // always drop prefixes. - var appf []astutil.ApplyFunc - if gen.cfg.Apply != nil { - appf = append(appf, gen.cfg.Apply) - } - appf = append(appf, PrefixDropper(decl.Meta.Common().Name)) - - pdir := gen.cfg.GenDirName(decl) - fpath := filepath.Join(gen.gokindsdir, pdir, lin.Name()+"_types_gen.go") +func (j GoTypesJenny) Generate(sfg SchemaForGen) (*codejen.File, error) { // TODO allow using name instead of machine name in thema generator - b, err := gocode.GenerateTypesOpenAPI(sch, &gocode.TypeConfigOpenAPI{ - PackageName: filepath.Base(pdir), - ApplyFuncs: appf, + b, err := gocode.GenerateTypesOpenAPI(sfg.Schema, &gocode.TypeConfigOpenAPI{ + // TODO will need to account for sanitizing e.g. dashes here at some point + PackageName: sfg.Schema.Lineage().Name(), + ApplyFuncs: []astutil.ApplyFunc{PrefixDropper(sfg.Name)}, }) if err != nil { return nil, err } - return codejen.NewFile(fpath, b, gen), nil + return codejen.NewFile(sfg.Schema.Lineage().Name()+"_types_gen.go", b, j), nil } diff --git a/pkg/codegen/jenny_tstypes.go b/pkg/codegen/jenny_tstypes.go index 110ec60ff1a..aa02b922eab 100644 --- a/pkg/codegen/jenny_tstypes.go +++ b/pkg/codegen/jenny_tstypes.go @@ -1,85 +1,32 @@ package codegen import ( - "fmt" - "path/filepath" - "github.com/grafana/codejen" - "github.com/grafana/thema" "github.com/grafana/thema/encoding/typescript" ) -// TSTypesJenny creates a [OneToOne] that produces TypeScript types and -// defaults for the latest Thema schema in a structured kind's lineage. +// TSTypesJenny is a [OneToOne] that produces TypeScript types and +// defaults for a Thema schema. // -// At minimum, a tskindsdir must be provided. This should be the path to the parent -// directory of the directory in which the types should be generated, relative -// to the project root. For example, if the types for a kind named "foo" -// should live at packages/grafana-schema/src/raw/foo, relpath should be "pkg/kind". -// -// This generator is a no-op for raw kinds. -func TSTypesJenny(tskindsdir string, cfg *TSTypesGeneratorConfig) OneToOne { - if cfg == nil { - cfg = new(TSTypesGeneratorConfig) - } - if cfg.GenDirName == nil { - cfg.GenDirName = func(decl *DeclForGen) string { - return decl.Meta.Common().MachineName - } - } +// Thema's generic TS jenny will be able to replace this one once +// https://github.com/grafana/thema/issues/89 is complete. +type TSTypesJenny struct{} - return &genTSTypes{ - tskindsdir: tskindsdir, - cfg: cfg, - } -} +var _ codejen.OneToOne[SchemaForGen] = &TSTypesJenny{} -// TSTypesGeneratorConfig holds configuration options for [TSTypesJenny]. -type TSTypesGeneratorConfig struct { - // GenDirName returns the name of the parent directory in which the type file - // should be generated. If nil, the DeclForGen.Lineage().Name() will be used. - GenDirName func(*DeclForGen) string - - // Version of the schema to generate. If nil, latest is generated. - Version *thema.SyntacticVersion -} - -type genTSTypes struct { - tskindsdir string - cfg *TSTypesGeneratorConfig -} - -func (gen *genTSTypes) JennyName() string { +func (j TSTypesJenny) JennyName() string { return "TSTypesJenny" } -func (gen *genTSTypes) Generate(decl *DeclForGen) (*codejen.File, error) { - if decl.IsRaw() { - return nil, nil - } - var sch thema.Schema - var err error - - lin := decl.Lineage() - if gen.cfg.Version == nil { - sch = lin.Latest() - } else { - sch, err = lin.Schema(*gen.cfg.Version) - if err != nil { - return nil, fmt.Errorf("error in configured version for %s generator: %w", *gen.cfg.Version, err) - } - } - +func (j TSTypesJenny) Generate(sfg SchemaForGen) (*codejen.File, error) { // TODO allow using name instead of machine name in thema generator - f, err := typescript.GenerateTypes(sch, &typescript.TypeConfig{ - RootName: decl.Meta.Common().Name, - Group: decl.Meta.Common().LineageIsGroup, + f, err := typescript.GenerateTypes(sfg.Schema, &typescript.TypeConfig{ + RootName: sfg.Name, + Group: sfg.IsGroup, }) if err != nil { return nil, err } - return codejen.NewFile( - filepath.Join(gen.tskindsdir, gen.cfg.GenDirName(decl), lin.Name()+"_types.gen.ts"), - []byte(f.String()), - gen), nil + + return codejen.NewFile(sfg.Schema.Lineage().Name()+"_types.gen.ts", []byte(f.String()), j), nil } diff --git a/pkg/codegen/latest_jenny.go b/pkg/codegen/latest_jenny.go new file mode 100644 index 00000000000..5f24052a46a --- /dev/null +++ b/pkg/codegen/latest_jenny.go @@ -0,0 +1,56 @@ +package codegen + +import ( + "fmt" + "path/filepath" + + "github.com/grafana/codejen" +) + +// LatestJenny returns a jenny that runs another jenny for only the latest +// schema in a DeclForGen, and prefixes the resulting file with the provided +// parentdir (e.g. "pkg/kinds/") and with a directory based on the kind's +// machine name (e.g. "dashboard/"). +func LatestJenny(parentdir string, inner codejen.OneToOne[SchemaForGen]) OneToOne { + if inner == nil { + panic("inner jenny must not be nil") + } + + return &latestj{ + parentdir: parentdir, + inner: inner, + } +} + +type latestj struct { + parentdir string + inner codejen.OneToOne[SchemaForGen] +} + +func (j *latestj) JennyName() string { + return "LatestJenny" +} + +func (j *latestj) Generate(decl *DeclForGen) (*codejen.File, error) { + if decl.IsRaw() { + return nil, nil + } + comm := decl.Meta.Common() + sfg := SchemaForGen{ + Name: comm.Name, + Schema: decl.Lineage().Latest(), + IsGroup: comm.LineageIsGroup, + } + + f, err := j.inner.Generate(sfg) + if err != nil { + return nil, fmt.Errorf("%s jenny failed on %s schema for %s: %w", j.inner.JennyName(), sfg.Schema.Version(), decl.Meta.Common().Name, err) + } + if f == nil || !f.Exists() { + return nil, nil + } + + f.RelativePath = filepath.Join(j.parentdir, comm.MachineName, f.RelativePath) + f.From = append(f.From, j) + return f, nil +} diff --git a/pkg/codegen/tmpl.go b/pkg/codegen/tmpl.go index 5b4f8212f4c..a2639faa58b 100644 --- a/pkg/codegen/tmpl.go +++ b/pkg/codegen/tmpl.go @@ -5,6 +5,8 @@ import ( "embed" "text/template" "time" + + "github.com/grafana/codejen" ) // All the parsed templates in the tmpl subdirectory @@ -29,6 +31,11 @@ type ( LineageCUEPath string GenLicense bool } + tvars_gen_header struct { + MainGenerator string + Using []codejen.NamedJenny + From string + } tvars_kind_registry struct { // Header tvars_autogen_header NumRaw, NumStructured int diff --git a/pkg/codegen/tmpl/gen_header.tmpl b/pkg/codegen/tmpl/gen_header.tmpl new file mode 100644 index 00000000000..d32ee6f24b3 --- /dev/null +++ b/pkg/codegen/tmpl/gen_header.tmpl @@ -0,0 +1,11 @@ +// THIS FILE IS GENERATED. EDITING IS FUTILE. +// +// Generated by: +// {{ .MainGenerator }} +// Using jennies: +{{- range .Using }} +// {{ .JennyName }} +{{- end }} +// +// Run 'make gen-cue' from repository root to regenerate. + diff --git a/pkg/kinds/dashboard/dashboard_types_gen.go b/pkg/kinds/dashboard/dashboard_types_gen.go index e4d63c1a16b..c8faba1a7a5 100644 --- a/pkg/kinds/dashboard/dashboard_types_gen.go +++ b/pkg/kinds/dashboard/dashboard_types_gen.go @@ -4,6 +4,7 @@ // kinds/gen.go // Using jennies: // GoTypesJenny +// LatestJenny // // Run 'make gen-cue' from repository root to regenerate. diff --git a/pkg/kinds/playlist/playlist_types_gen.go b/pkg/kinds/playlist/playlist_types_gen.go index d45a61f3e2c..9b94aa11107 100644 --- a/pkg/kinds/playlist/playlist_types_gen.go +++ b/pkg/kinds/playlist/playlist_types_gen.go @@ -4,6 +4,7 @@ // kinds/gen.go // Using jennies: // GoTypesJenny +// LatestJenny // // Run 'make gen-cue' from repository root to regenerate.