Feat: Add command line app to validate go.mod (#67796)

* chore: start modowners

* read go.mod, parse modfile, iterate through requires; add dummy go.mod

* make BEP owners of all grafana dependencies 😱

* push attempt at logging the require comments

* shrink dummy modfile

* revert changes in go.mod

* access comments suffix

* add Module struct; attempt to separate ParseGoMod functionality into its own func; add owner (third) for loop when interating modfile

* feat: print all owners in modfile

* add additional question in comment

* feat: add subcommands: check, owners, modules; chunk out some functions

* chunk out subcommand functions

* add flags

* start tests for common element

* refactor: test for common element

* attempt #1 to refactor modules to accept multiple args

* refactor: refactor modfule func to take 1+ owner arguments (0 arguments not working atm)

* chore: remove debug logging

* refine existing comments

* comment out indirect flag stuff, add example cli command for modules

* unsuccessful attempt #2 to refactor modules to accept -o and -i flags

* refactor funcs to take filesystem and logger

* test: add test for check when all modules have owners

* fail attempt 1 to get TestModules to work

* assert expected log result in TestModules; unsure if properly reading logs

* test: add TestModules to test modules func without any flags returns direct dependencies

* test: add TestInvalidCheck for scenario when some dependencies are missing an owner

* attempt 1 at refactoring TestCheck into a table

* chore: clean TestCheck

* chore: clean up comments for func check

* move files under scripts/modowners

* revert go.mod and go.sum
This commit is contained in:
Kat Yang 2023-05-30 11:18:05 -04:00 committed by GitHub
parent 778f054419
commit f3f3b1494d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 340 additions and 0 deletions

1
scripts/modowners/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
modowners

12
scripts/modowners/go.mod Normal file
View File

@ -0,0 +1,12 @@
module modowners
go 1.19
require golang.org/x/mod v0.10.0
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/testify v1.8.3 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

11
scripts/modowners/go.sum Normal file
View File

@ -0,0 +1,11 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

23
scripts/modowners/go.txd Normal file
View File

@ -0,0 +1,23 @@
module github.com/grafana/grafana
go 1.19
require (
cloud.google.com/go/storage v1.28.1 // @backend-platform
cuelang.org/go v0.5.0 // @as-code @backend-platform
github.com/Azure/azure-sdk-for-go v65.0.0+incompatible // indirect, @delivery
github.com/Masterminds/semver v1.5.0 // @delivery @backend-platform
)
require (
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
)
require (
cloud.google.com/go/kms v1.4.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // @backend-platform
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.9.0 // @delivery
)

View File

@ -0,0 +1,191 @@
package main
import (
"errors"
"flag"
"fmt"
"io"
"io/fs"
"log"
"os"
"strings"
"golang.org/x/mod/modfile"
)
type Module struct {
Name string
Owners []string
Indirect bool
}
func parseModule(mod *modfile.Require) Module {
m := Module{Name: mod.Mod.String()}
// For each require, access the comment.
for _, comment := range mod.Syntax.Comments.Suffix {
owners := strings.Fields(comment.Token)
// For each comment, determine if it contains owner(s).
for _, owner := range owners {
if strings.Contains(owner, "indirect") {
m.Indirect = true
}
// If there is an owner, add to owners list.
if strings.Contains(owner, "@") {
m.Owners = append(m.Owners, owner)
}
}
}
return m
}
func parseGoMod(fileSystem fs.FS, name string) ([]Module, error) {
file, err := fileSystem.Open(name)
if err != nil {
return nil, err
}
defer file.Close()
// Turn modfile into array of bytes.
data, err := io.ReadAll(file)
if err != nil {
return nil, err
}
// Parse modfile.
modFile, err := modfile.Parse(name, data, nil)
if err != nil {
return nil, err
}
modules := []Module{}
// Iterate through requires in modfile.
for _, mod := range modFile.Require {
m := parseModule(mod)
modules = append(modules, m)
}
return modules, nil
}
// Validate that each module has an owner.
// An example CLI command is `go run dummy/modowners.go check dummy/go.txd`
// TODO: replace above example with final filepath in the end
func check(fileSystem fs.FS, logger *log.Logger, args []string) error {
m, err := parseGoMod(fileSystem, args[0])
if err != nil {
return err
}
fail := false
for _, mod := range m {
if !mod.Indirect && len(mod.Owners) == 0 {
logger.Println(mod.Name)
fail = true
}
}
if fail {
return errors.New("modfile is invalid")
}
return nil
}
// TODO: owners and modules may optionally take a list (modules for owners, owners for modules)
// TODO: test with go test
// Print owners.
func owners(fileSystem fs.FS, logger *log.Logger, args []string) error {
fs := flag.NewFlagSet("owners", flag.ExitOnError)
count := fs.Bool("c", false, "print count of dependencies per owner")
fs.Parse(args)
m, err := parseGoMod(fileSystem, fs.Arg(0))
if err != nil {
return err
}
owners := map[string]int{}
for _, mod := range m {
if mod.Indirect == false {
for _, owner := range mod.Owners {
owners[owner]++
}
}
}
for owner, n := range owners {
if *count {
fmt.Println(owner, n)
} else {
fmt.Println(owner)
}
}
return nil
}
/*
GOAL:
1. if no flags, print all direct dependencies
2. if -i, print all dependencies (direct + indirect)
3. if -o, print dependencies owned by the owner(s) listed
4. if -i and -o, print all dependencies owned by the owner(s) listed
print all dependencies for each owner listed in CLI after -o flag
check each dependency's owners
if it match one of the owners in the flag/CLI, print it
if not skip
CURRENT ISSUE:
owner flag logic not working well with indirect flag logic
not sure how to check for both flags
mod.Owners := [bep, as-code, delivery]
flag := [gaas, delivery]
*/
// Print dependencies. Can specify direct / multiple owners.
// Example CLI command `go run dummy/modowners.go modules -m dummy/go.txd -o @as-code,@delivery`
func modules(fileSystem fs.FS, logger *log.Logger, args []string) error {
fs := flag.NewFlagSet("modules", flag.ExitOnError)
indirect := fs.Bool("i", false, "print indirect dependencies") // NOTE: indirect is a pointer bc we dont want to lose value after changing it
modfile := fs.String("m", "go.txd", "use specified modfile")
owner := fs.String("o", "", "one or more owners")
fs.Parse(args)
m, err := parseGoMod(fileSystem, *modfile) // NOTE: give me the string that's the first positional argument; fs.Arg works only after fs.Parse
if err != nil {
return err
}
ownerFlags := strings.Split(*owner, ",")
for _, mod := range m {
// If there are owner flags or modfile's dependency has an owner to compare
// Else if -i is present and current dependency is indirect
if len(*owner) > 0 && hasCommonElement(mod.Owners, ownerFlags) {
logger.Println(mod.Name)
} else if *indirect && !mod.Indirect {
logger.Println(mod.Name)
}
}
return nil
}
func hasCommonElement(a []string, b []string) bool {
for _, u := range a {
for _, v := range b {
if u == v {
return true
}
}
}
return false
}
func main() {
if len(os.Args) < 2 {
fmt.Println("usage: modowners subcommand go.mod...")
os.Exit(1)
}
type CmdFunc func(fs.FS, *log.Logger, []string) error
cmds := map[string]CmdFunc{"check": check, "owners": owners, "modules": modules}
if f, ok := cmds[os.Args[1]]; !ok { // NOTE: both f and ok are visible inside the if / else if statement, but not outside; chaining of ifs very common in go when checking errors and calling multiple funcs
log.Fatal("invalid command")
} else if err := f(os.DirFS("."), log.Default(), os.Args[2:]); err != nil {
log.Fatal(err)
}
}

View File

@ -0,0 +1,102 @@
package main
import (
"bytes"
"log"
"strings"
"testing"
"testing/fstest"
)
func TestCommonElement(t *testing.T) {
for _, test := range []struct {
A []string
B []string
Result bool
}{
{nil, nil, false},
{[]string{"a"}, []string{"a"}, true},
{[]string{"a", "b"}, []string{"a"}, true},
{[]string{"a"}, []string{"b"}, false},
} {
if hasCommonElement(test.A, test.B) != test.Result {
t.Error(test)
}
}
}
func TestCheck(t *testing.T) {
for _, test := range []struct {
description string
fileName string
contents string
args []string
valid bool
expectedOutput string
}{
{"Test valid modfile", "go.mod", `
require (
cloud.google.com/go/storage v1.28.1 // @delivery
cuelang.org/go v0.5.0 // @as-code @backend-platform
github.com/Azure/azure-sdk-for-go v65.0.0+incompatible // indirect, @delivery
github.com/Masterminds/semver v1.5.0 // @delivery @backend-platform
)
`, []string{"go.mod"}, true, ""},
{"Test invalid modfile", "go.mod", `
require (
cloud.google.com/go/storage v1.28.1
cuelang.org/go v0.5.0 // @as-code @backend-platform
github.com/Azure/azure-sdk-for-go v65.0.0+incompatible // indirect, @delivery
github.com/Masterminds/semver v1.5.0 // @delivery @backend-platform
)
`, []string{"go.mod"}, false, "cloud.google.com/go/storage@v1.28.1\n"},
} {
buf := &bytes.Buffer{}
logger := log.New(buf, "", 0)
filesystem := fstest.MapFS{test.fileName: &fstest.MapFile{Data: []byte(test.contents)}}
err := check(filesystem, logger, test.args)
if test.valid && err != nil {
t.Error(test.description, err)
} else if !test.valid && err == nil {
t.Error(test.description, "error expected")
}
if buf.String() != test.expectedOutput {
t.Error(test.description, buf.String())
}
}
}
func TestModules(t *testing.T) {
buf := &bytes.Buffer{}
logger := log.New(buf, "", 0)
filesystem := fstest.MapFS{"go.txd": &fstest.MapFile{Data: []byte(`
require (
cloud.google.com/go/storage v1.28.1
cuelang.org/go v0.5.0 // @as-code @backend-platform
github.com/Azure/azure-sdk-for-go v65.0.0+incompatible // indirect, @delivery
github.com/Masterminds/semver v1.5.0 // @delivery @backend-platform
)
`)}}
err := modules(filesystem, logger, []string{"-m", "go.txd"}) // NOTE: pass various flags, these are cmd line arguments
if err != nil {
t.Error(err, buf.String())
}
logs := buf.String()
// Expected results
expectedModules := []string{
"cloud.google.com/go/storage v1.28.1",
"cuelang.org/go v0.5.0",
"github.com/Azure/azure-sdk-for-go v65.0.0+incompatible",
"github.com/Masterminds/semver v1.5.0",
}
expectedResults := strings.Join(expectedModules, "\n")
// Compare logs to expected results
if logs != expectedResults {
t.Error(err)
}
}