infra/go: remove unused source code and config
Packages have been successfully moved to infra/infra.
BUG=chromium:1185285
TEST=none
Change-Id: I666ea109eaf268e65a29384bfc96699a6a997cd9
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/infra/go/+/2762600
Commit-Queue: Jack Neus <jackneus@google.com>
Tested-by: Jack Neus <jackneus@google.com>
Reviewed-by: George Engelbrecht <engeg@google.com>
Reviewed-by: Dhanya Ganesh <dhanyaganesh@chromium.org>
diff --git a/cmd/build_plan_generator/build_plan_generator.go b/cmd/build_plan_generator/build_plan_generator.go
deleted file mode 100644
index 33e9b81..0000000
--- a/cmd/build_plan_generator/build_plan_generator.go
+++ /dev/null
@@ -1,381 +0,0 @@
-// Copyright 2019 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-package main
-
-import (
- "bytes"
- "context"
- "flag"
- "fmt"
- "io/ioutil"
- "log"
- "os"
-
- "github.com/golang/protobuf/jsonpb"
- "github.com/golang/protobuf/proto"
- "github.com/maruel/subcommands"
- "go.chromium.org/chromiumos/infra/go/internal/build_plan"
- igerrit "go.chromium.org/chromiumos/infra/go/internal/gerrit"
- "go.chromium.org/chromiumos/infra/go/internal/repo"
- cros_pb "go.chromium.org/chromiumos/infra/proto/go/chromiumos"
- testplans_pb "go.chromium.org/chromiumos/infra/proto/go/testplans"
- "go.chromium.org/luci/auth"
- "go.chromium.org/luci/auth/client/authcli"
- bbproto "go.chromium.org/luci/buildbucket/proto"
- "go.chromium.org/luci/common/api/gerrit"
- "go.chromium.org/luci/common/cli"
- "go.chromium.org/luci/hardcoded/chromeinfra"
-)
-
-const (
- buildIrrelevanceConfigPath = "buildplanconfig/generated/build_irrelevance_config.binaryproto"
- builderConfigsPath = "generated/builder_configs.binaryproto"
- slimBuildConfigPath = "buildplanconfig/generated/slim_build_config.binaryproto"
- targetTestRequirementsConfigPath = "testingconfig/generated/target_test_requirements.binaryproto"
-)
-
-var (
- unmarshaler = jsonpb.Unmarshaler{AllowUnknownFields: true}
-)
-
-func cmdCheckSkip(authOpts auth.Options) *subcommands.Command {
- return &subcommands.Command{
- UsageLine: "generate-plan --input_json=/path/to/input.json --output_json=/path/to/output.json",
- ShortDesc: "Checks which of the provided BuilderConfigs must be run",
- LongDesc: "Checks which of the provided BuilderConfigs must be run.",
- CommandRun: func() subcommands.CommandRun {
- c := &checkBuild{}
- c.authFlags = authcli.Flags{}
- c.authFlags.Register(c.GetFlags(), authOpts)
- c.Flags.StringVar(&c.inputJson, "input_json", "",
- "Path to JSON proto representing a GenerateBuildPlanRequest")
- c.Flags.StringVar(&c.outputJson, "output_json", "",
- "Path to file to write output GenerateBuildPlanResponse JSON proto")
- c.Flags.StringVar(&c.inputTextPb, "input_text_pb", "",
- "Path to text proto representing a GenerateBuildPlanRequest")
- c.Flags.StringVar(&c.outputTextPb, "output_text_pb", "",
- "Path to file to write output GenerateBuildPlanResponse text proto")
- c.Flags.StringVar(&c.inputBinaryPb, "input_binary_pb", "",
- "Path to binaryproto file representing a GenerateTestPlanRequest")
- c.Flags.StringVar(&c.outputBinaryPb, "output_binary_pb", "",
- "Path to file to write output GenerateTestPlanResponse binaryproto")
- c.Flags.StringVar(&c.manifestFile, "manifest_file", "",
- "Path to local manifest file. If given, will be used instead of default snapshot.xml")
- return c
- }}
-}
-
-type fetchConfigResult struct {
- buildIrrelevanceCfg *testplans_pb.BuildIrrelevanceCfg
- slimBuildCfg *testplans_pb.SlimBuildCfg
- testReqsCfg *testplans_pb.TargetTestRequirementsCfg
- builderConfigs *cros_pb.BuilderConfigs
-}
-
-func (c *checkBuild) Run(a subcommands.Application, args []string, env subcommands.Env) int {
- flag.Parse()
-
- req, err := c.readInput()
- if err != nil {
- log.Print(err)
- return 1
- }
-
- configs, err := c.fetchConfigFromGitiles()
- if err != nil {
- log.Print(err)
- return 2
- }
-
- changes, err := readGerritChanges(req.GerritChanges)
- if err != nil {
- log.Print(err)
- return 3
- }
-
- changeRevs, err := c.fetchGerritData(changes)
- if err != nil {
- log.Print(err)
- return 4
- }
-
- var repoToSrcRoot *map[string]map[string]string
- // If we have a local manifest file provided, use that. Else get it from Gerrit.
- if c.manifestFile == "" {
- gitilesCommit, err := readGitilesCommit(req.GitilesCommit)
- if err != nil {
- log.Print(err)
- return 5
- }
-
- repoToSrcRoot, err = c.getRepoToSourceRoot(gitilesCommit)
- if err != nil {
- log.Print(err)
- return 6
- }
- } else {
- log.Printf("Reading local manifest from %s", c.manifestFile)
- repoToSrcRootMap, err := repo.GetRepoToRemoteBranchToSourceRootFromManifestFile(c.manifestFile)
- if err != nil {
- log.Print(err)
- return 9
- }
- repoToSrcRoot = &repoToSrcRootMap
- }
-
- checkBuildersInput := build_plan.CheckBuildersInput{
- Builders: req.BuilderConfigs,
- Changes: changes,
- ChangeRevs: changeRevs,
- RepoToBranchToSrcRoot: *repoToSrcRoot,
- BuildIrrelevanceCfg: configs.buildIrrelevanceCfg,
- SlimBuildCfg: configs.slimBuildCfg,
- TestReqsCfg: configs.testReqsCfg,
- BuilderConfigs: configs.builderConfigs,
- }
- resp, err := checkBuildersInput.CheckBuilders()
- if err != nil {
- log.Printf("Error checking which builds can be skipped:\n%v", err)
- return 7
- }
-
- if err = c.writeOutput(resp); err != nil {
- log.Print(err)
- return 8
- }
- return 0
-}
-
-type checkBuild struct {
- subcommands.CommandRunBase
- authFlags authcli.Flags
- inputJson string
- outputJson string
- inputTextPb string
- outputTextPb string
- inputBinaryPb string
- outputBinaryPb string
- manifestFile string
-}
-
-func nonEmptyCount(strs ...string) int64 {
- var count int64
- for _, str := range strs {
- if len(str) > 0 {
- count++
- }
- }
- return count
-}
-
-func (c *checkBuild) readInput() (*cros_pb.GenerateBuildPlanRequest, error) {
- numInputs := nonEmptyCount(c.inputBinaryPb, c.inputTextPb, c.inputJson)
- if numInputs != 1 {
- return nil, fmt.Errorf("expected 1 input-related flag, found %v", numInputs)
- }
- // use input_binary_pb if it's specified
- if len(c.inputBinaryPb) > 0 {
- inputPb, err := ioutil.ReadFile(c.inputBinaryPb)
- if err != nil {
- return nil, fmt.Errorf("Failed reason input_binary_pb\n%v", err)
- }
- req := &cros_pb.GenerateBuildPlanRequest{}
- if err := proto.Unmarshal(inputPb, req); err != nil {
- return nil, fmt.Errorf("Failed parsing input_binary_pb as proto\n%v", err)
- }
- return req, nil
- }
- if len(c.inputTextPb) > 0 {
- inputBytes, err := ioutil.ReadFile(c.inputTextPb)
- log.Printf("Request is:\n%s", string(inputBytes))
- if err != nil {
- return nil, fmt.Errorf("Failed reading input_text_pb\n%v", err)
- }
- req := &cros_pb.GenerateBuildPlanRequest{}
- if err := proto.UnmarshalText(string(inputBytes), req); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a chromiumos.GenerateBuildPlanRequest\n%v", c.inputJson, err)
- }
- return req, nil
- }
- inputBytes, err := ioutil.ReadFile(c.inputJson)
- log.Printf("Request is:\n%s", string(inputBytes))
- if err != nil {
- return nil, fmt.Errorf("Failed reading input_json\n%v", err)
- }
- req := &cros_pb.GenerateBuildPlanRequest{}
- if err := unmarshaler.Unmarshal(bytes.NewReader(inputBytes), req); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a chromiumos.GenerateBuildPlanRequest\n%v", c.inputJson, err)
- }
- return req, nil
-}
-
-func (c *checkBuild) fetchConfigFromGitiles() (*fetchConfigResult, error) {
- // Create an authenticated client for Gerrit RPCs, then fetch all required CL data from Gerrit.
- ctx := context.Background()
- authOpts, err := c.authFlags.Options()
- if err != nil {
- return nil, err
- }
- authedClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, authOpts).Client()
- if err != nil {
- return nil, err
- }
- m, err := igerrit.FetchFilesFromGitiles(authedClient, ctx,
- "chrome-internal.googlesource.com",
- "chromeos/infra/config",
- "main",
- []string{buildIrrelevanceConfigPath, slimBuildConfigPath, targetTestRequirementsConfigPath, builderConfigsPath})
- if err != nil {
- return nil, err
- }
- buildIrrelevanceConfig := &testplans_pb.BuildIrrelevanceCfg{}
- if err := proto.Unmarshal([]byte((*m)[buildIrrelevanceConfigPath]), buildIrrelevanceConfig); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a BuildIrrelevanceCfg\n%v", (*m)[buildIrrelevanceConfigPath], err)
- }
- slimBuildConfig := &testplans_pb.SlimBuildCfg{}
- if err := proto.Unmarshal([]byte((*m)[slimBuildConfigPath]), slimBuildConfig); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a SlimBuildCfg\n%v", (*m)[slimBuildConfigPath], err)
- }
- targetTestRequirementsConfig := &testplans_pb.TargetTestRequirementsCfg{}
- if err := proto.Unmarshal([]byte((*m)[targetTestRequirementsConfigPath]), targetTestRequirementsConfig); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a TargetTestRequirementsCfg\n%v", (*m)[targetTestRequirementsConfigPath], err)
- }
- builderConfigs := &cros_pb.BuilderConfigs{}
- if err := proto.Unmarshal([]byte((*m)[builderConfigsPath]), builderConfigs); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a BuilderConfigs\n%v", (*m)[builderConfigsPath], err)
- }
- log.Printf("Fetched config from Gitiles:\n%s\n\n%s\n\n%s",
- proto.MarshalTextString(buildIrrelevanceConfig), proto.MarshalTextString(targetTestRequirementsConfig), proto.MarshalTextString(builderConfigs))
- return &fetchConfigResult{
- buildIrrelevanceCfg: buildIrrelevanceConfig,
- slimBuildCfg: slimBuildConfig,
- testReqsCfg: targetTestRequirementsConfig,
- builderConfigs: builderConfigs,
- }, nil
-}
-
-func readGerritChanges(changeBytes []*cros_pb.ProtoBytes) ([]*bbproto.GerritChange, error) {
- changes := make([]*bbproto.GerritChange, 0)
- for i, c := range changeBytes {
- gc := &bbproto.GerritChange{}
- if err := proto.Unmarshal(c.SerializedProto, gc); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a GerritChange\n%v", c.String(), err)
- }
- log.Printf("Got GerritChange %d proto:\n%s", i, proto.MarshalTextString(gc))
- changes = append(changes, gc)
- }
- return changes, nil
-}
-
-func (c *checkBuild) fetchGerritData(changes []*bbproto.GerritChange) (*igerrit.ChangeRevData, error) {
- // Create an authenticated client for Gerrit RPCs, then fetch all required CL data from Gerrit.
- ctx := context.Background()
- authOpts, err := c.authFlags.Options()
- if err != nil {
- return nil, err
- }
- authedClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, authOpts).Client()
- if err != nil {
- return nil, err
- }
- changeIds := make([]igerrit.ChangeRevKey, 0)
- for _, ch := range changes {
- changeIds = append(changeIds, igerrit.ChangeRevKey{Host: ch.Host, ChangeNum: ch.Change, Revision: int32(ch.Patchset)})
- }
- chRevData, err := igerrit.GetChangeRevData(authedClient, ctx, changeIds)
- if err != nil {
- return nil, fmt.Errorf("Failed to fetch CL data from Gerrit. "+
- "Note that a NotFound error may indicate authorization issues.\n%v", err)
- }
- return chRevData, nil
-}
-
-func readGitilesCommit(gitilesBytes *cros_pb.ProtoBytes) (*bbproto.GitilesCommit, error) {
- gc := &bbproto.GitilesCommit{}
- if err := proto.Unmarshal(gitilesBytes.SerializedProto, gc); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a GitilesCommit\n%v", gitilesBytes.String(), err)
- }
- log.Printf("Got GitilesCommit proto:\n%s", proto.MarshalTextString(gc))
- return gc, nil
-}
-
-func (c *checkBuild) getRepoToSourceRoot(gc *bbproto.GitilesCommit) (*map[string]map[string]string, error) {
- ctx := context.Background()
- authOpts, err := c.authFlags.Options()
- if err != nil {
- return nil, err
- }
- authedClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, authOpts).Client()
- if err != nil {
- return nil, err
- }
- if gc.Id == "" {
- log.Print("No manifest commit provided. Using 'snapshot' instead.")
- gc.Id = "snapshot"
- }
- repoToRemoteBranchToSrcRoot, err := repo.GetRepoToRemoteBranchToSourceRootFromManifests(authedClient, ctx, gc)
- if err != nil {
- return nil, fmt.Errorf("Error with repo tool call\n%v", err)
- }
- return &repoToRemoteBranchToSrcRoot, nil
-}
-
-func (c *checkBuild) writeOutput(resp *cros_pb.GenerateBuildPlanResponse) error {
- log.Printf("Full output =\n%s", proto.MarshalTextString(resp))
-
- if len(c.outputJson) > 0 {
- marshal := &jsonpb.Marshaler{EmitDefaults: true, Indent: " "}
- jsonOutput, err := marshal.MarshalToString(resp)
- if err != nil {
- return fmt.Errorf("Failed to marshal JSON %v\n%v", resp, err)
- }
- if err = ioutil.WriteFile(c.outputJson, []byte(jsonOutput), 0644); err != nil {
- return fmt.Errorf("Failed to write output JSON!\n%v", err)
- }
- log.Printf("Wrote JSON output to %s", c.outputJson)
- }
-
- if len(c.outputTextPb) > 0 {
- if err := ioutil.WriteFile(c.outputTextPb, []byte(proto.MarshalTextString(resp)), 0644); err != nil {
- return fmt.Errorf("Failed to write output text proto!\n%v", err)
- }
- }
-
- if len(c.outputBinaryPb) > 0 {
- binaryOutput, err := proto.Marshal(resp)
- if err != nil {
- return fmt.Errorf("Failed to marshal binaryproto %v\n%v", resp, err)
- }
- if err = ioutil.WriteFile(c.outputBinaryPb, binaryOutput, 0644); err != nil {
- return fmt.Errorf("Failed to write output binary proto!\n%v", err)
- }
- log.Printf("Wrote output binary proto to %s", c.outputBinaryPb)
- }
-
- return nil
-}
-
-func GetApplication(authOpts auth.Options) *cli.Application {
- return &cli.Application{
- Name: "build_plan_generator",
-
- Context: func(ctx context.Context) context.Context {
- return ctx
- },
-
- Commands: []*subcommands.Command{
- authcli.SubcommandInfo(authOpts, "auth-info", false),
- authcli.SubcommandLogin(authOpts, "auth-login", false),
- authcli.SubcommandLogout(authOpts, "auth-logout", false),
- cmdCheckSkip(authOpts),
- },
- }
-}
-
-func main() {
- opts := chromeinfra.DefaultAuthOptions()
- opts.Scopes = []string{gerrit.OAuthScope, auth.OAuthScopeEmail}
- app := GetApplication(opts)
- os.Exit(subcommands.Run(app, nil))
-}
diff --git a/cmd/build_plan_generator/sample/input.json b/cmd/build_plan_generator/sample/input.json
deleted file mode 100644
index ba9f930..0000000
--- a/cmd/build_plan_generator/sample/input.json
+++ /dev/null
@@ -1,95 +0,0 @@
-{
- "gerritChanges": [
- {
- "serializedProto": "CiBjaHJvbWl1bS1yZXZpZXcuZ29vZ2xlc291cmNlLmNvbRIeY2hyb21pdW1vcy9wbGF0Zm9ybS90YXN0LXRlc3RzGNHebSAB"
- }
- ],
- "gitiles_commit": {
- "serializedProto": "CiBjaHJvbWUtaW50ZXJuYWwuZ29vZ2xlc291cmNlLmNvbRIaY2hyb21lb3MvbWFuaWZlc3QtaW50ZXJuYWwaKDg2NmUyYTMxMzI0MWJmNmVkOTMxMTQ4ZWQ0M2VhMTIyMzVhNGZiYzU="
- },
- "manifestCommit": "866e2a313241bf6ed931148ed43ea12235a4fbc5",
- "builderConfigs": [
- {
- "id": {
- "name": "amd64-generic-bisect",
- "branch": "master",
- "type": "POSTSUBMIT"
- },
- "general": {
- "critical": true,
- "environment": "PRODUCTION",
- "runWhen": {
- "mode": "ONLY_RUN_ON_FILE_MATCH",
- "filePatterns": [
- "src/project/dedede/boten/sw_build_config/**"
- ]
- }
- },
- "artifacts": {
- "prebuilts": "PUBLIC",
- "artifactTypes": [
- "IMAGE_ZIP",
- "EBUILD_LOGS",
- "CPE_REPORT"
- ]
- },
- "chrome": {},
- "build": {
- "installPackages": {
- "runSpec": "RUN_EXIT"
- }
- },
- "unitTests": {
- "ebuildsRunSpec": "RUN"
- }
- },
- {
- "id": {
- "name": "amd64-generic-cq",
- "branch": "master",
- "type": "CQ"
- },
- "general": {
- "critical": true,
- "environment": "PRODUCTION",
- "runWhen": {
- "mode": "ALWAYS_RUN"
- }
- },
- "artifacts": {
- "prebuilts": "NONE",
- "artifactTypes": [
- "IMAGE_ZIP",
- "AUTOTEST_FILES",
- "TAST_FILES",
- "PINNED_GUEST_IMAGES",
- "EBUILD_LOGS",
- "TEST_UPDATE_PAYLOAD"
- ],
- "artifactsGsBucket": "chromeos-image-archive"
- },
- "chrome": {},
- "build": {
- "buildImages": {
- "imageTypes": [
- "TEST",
- "TEST_VM"
- ]
- },
- "installPackages": {
- "runSpec": "RUN"
- },
- "applyGerritChanges": true
- },
- "unitTests": {
- "packageBlacklist": [
- {
- "packageName": "chromite",
- "category": "chromeos-base"
- }
- ],
- "ebuildsRunSpec": "RUN"
- }
- }
- ]
-}
diff --git a/cmd/pointless_build_checker/pointless_build_checker.go b/cmd/pointless_build_checker/pointless_build_checker.go
deleted file mode 100644
index 20af684..0000000
--- a/cmd/pointless_build_checker/pointless_build_checker.go
+++ /dev/null
@@ -1,306 +0,0 @@
-// Copyright 2019 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-package main
-
-import (
- "bytes"
- "context"
- "flag"
- "fmt"
- "io/ioutil"
- "log"
- "os"
-
- "github.com/golang/protobuf/jsonpb"
- "github.com/golang/protobuf/proto"
- "github.com/maruel/subcommands"
- igerrit "go.chromium.org/chromiumos/infra/go/internal/gerrit"
- "go.chromium.org/chromiumos/infra/go/internal/pointless"
- "go.chromium.org/chromiumos/infra/go/internal/repo"
- testplans_pb "go.chromium.org/chromiumos/infra/proto/go/testplans"
- "go.chromium.org/luci/auth"
- "go.chromium.org/luci/auth/client/authcli"
- bbproto "go.chromium.org/luci/buildbucket/proto"
- "go.chromium.org/luci/common/api/gerrit"
- "go.chromium.org/luci/common/cli"
- "go.chromium.org/luci/hardcoded/chromeinfra"
-)
-
-const (
- buildIrrelevanceConfigPath = "buildplanconfig/generated/build_irrelevance_config.binaryproto"
-)
-
-var (
- unmarshaler = jsonpb.Unmarshaler{AllowUnknownFields: true}
-)
-
-func cmdCheckBuild(authOpts auth.Options) *subcommands.Command {
- return &subcommands.Command{
- UsageLine: "check-build --input_json=/path/to/input.json --output_json=/path/to/output.json",
- ShortDesc: "Checks if the current build is pointless",
- LongDesc: "Checks if the current build is pointless, e.g. if the commits in the CQ run can't " +
- "actually affect the outcome of the build.",
- CommandRun: func() subcommands.CommandRun {
- c := &checkBuild{}
- c.authFlags = authcli.Flags{}
- c.authFlags.Register(c.GetFlags(), authOpts)
- c.Flags.StringVar(&c.inputJson, "input_json", "",
- "Path to JSON proto representing a PointlessBuildCheckRequest")
- c.Flags.StringVar(&c.outputJson, "output_json", "",
- "Path to file to write output PointlessBuildCheckResponse JSON proto")
- c.Flags.StringVar(&c.inputBinaryPb, "input_binary_pb", "",
- "Path to binaryproto file representing a PointlessBuildCheckRequest")
- c.Flags.StringVar(&c.outputBinaryPb, "output_binary_pb", "",
- "Path to file to write output PointlessBuildCheckResponse binaryproto")
- c.Flags.StringVar(&c.manifestFile, "manifest_file", "",
- "Path to local manifest file. If given, will be used instead of default snapshot.xml")
- return c
- }}
-}
-
-func (c *checkBuild) Run(a subcommands.Application, args []string, env subcommands.Env) int {
- flag.Parse()
-
- req, err := c.readInput()
- if err != nil {
- log.Print(err)
- return 1
- }
-
- cfg, err := c.fetchConfigFromGitiles()
- if err != nil {
- log.Print(err)
- return 2
- }
-
- changes, err := readGerritChanges(req.GerritChanges)
- if err != nil {
- log.Print(err)
- return 3
- }
-
- changeRevs, err := c.fetchGerritData(changes)
- if err != nil {
- log.Print(err)
- return 4
- }
-
- var repoToSrcRoot *map[string]map[string]string
- if c.manifestFile == "" {
- gitilesCommit, err := readGitilesCommit(req.GitilesCommit)
- if err != nil {
- log.Print(err)
- return 5
- }
-
- repoToSrcRoot, err = c.getRepoToSourceRoot(gitilesCommit)
- if err != nil {
- log.Print(err)
- return 6
- }
- } else {
- log.Printf("Reading local manifest from %s", c.manifestFile)
- repoToSrcRootMap, err := repo.GetRepoToRemoteBranchToSourceRootFromManifestFile(c.manifestFile)
- if err != nil {
- log.Print(err)
- return 9
- }
- repoToSrcRoot = &repoToSrcRootMap
- }
-
- resp, err := pointless.CheckBuilder(changes, changeRevs, req.RelevantPaths, *repoToSrcRoot, cfg)
- if err != nil {
- log.Printf("Error checking if build is pointless:\n%v", err)
- return 7
- }
-
- if err = c.writeOutput(resp); err != nil {
- log.Print(err)
- return 8
- }
- return 0
-}
-
-type checkBuild struct {
- subcommands.CommandRunBase
- authFlags authcli.Flags
- inputJson string
- outputJson string
- inputBinaryPb string
- outputBinaryPb string
- manifestFile string
-}
-
-func (c *checkBuild) readInput() (*testplans_pb.PointlessBuildCheckRequest, error) {
- // use input_binary_pb if it's specified
- if len(c.inputBinaryPb) > 0 {
- inputPb, err := ioutil.ReadFile(c.inputBinaryPb)
- if err != nil {
- return nil, fmt.Errorf("Failed reason input_binary_pb\n%v", err)
- }
- req := &testplans_pb.PointlessBuildCheckRequest{}
- if err := proto.Unmarshal(inputPb, req); err != nil {
- return nil, fmt.Errorf("Failed parsing input_binary_pb as proto\n%v", err)
- }
- return req, nil
- // otherwise use input_json
- } else {
- inputBytes, err := ioutil.ReadFile(c.inputJson)
- log.Printf("Request is:\n%s", string(inputBytes))
- if err != nil {
- return nil, fmt.Errorf("Failed reading input_json\n%v", err)
- }
- req := &testplans_pb.PointlessBuildCheckRequest{}
- if err := unmarshaler.Unmarshal(bytes.NewReader(inputBytes), req); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a chromiumos.PointlessBuildCheckRequest\n%v", c.inputJson, err)
- }
- return req, nil
- }
-}
-
-func (c *checkBuild) fetchConfigFromGitiles() (*testplans_pb.BuildIrrelevanceCfg, error) {
- // Create an authenticated client for Gerrit RPCs, then fetch all required CL data from Gerrit.
- ctx := context.Background()
- authOpts, err := c.authFlags.Options()
- if err != nil {
- return nil, err
- }
- authedClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, authOpts).Client()
- if err != nil {
- return nil, err
- }
- m, err := igerrit.FetchFilesFromGitiles(authedClient, ctx,
- "chrome-internal.googlesource.com",
- "chromeos/infra/config",
- "main",
- []string{buildIrrelevanceConfigPath})
- if err != nil {
- return nil, err
- }
- buildIrrelevanceConfig := &testplans_pb.BuildIrrelevanceCfg{}
- if err := proto.Unmarshal([]byte((*m)[buildIrrelevanceConfigPath]), buildIrrelevanceConfig); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a BuildIrrelevanceCfg\n%v", (*m)[buildIrrelevanceConfigPath], err)
- }
- log.Printf("Fetched config from Gitiles:\n%s\n", proto.MarshalTextString(buildIrrelevanceConfig))
- return buildIrrelevanceConfig, nil
-}
-
-func readGerritChanges(changeBytes []*testplans_pb.ProtoBytes) ([]*bbproto.GerritChange, error) {
- changes := make([]*bbproto.GerritChange, 0)
- for i, c := range changeBytes {
- gc := &bbproto.GerritChange{}
- if err := proto.Unmarshal(c.SerializedProto, gc); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a GerritChange\n%v", c.String(), err)
- }
- log.Printf("Got GerritChange %d proto:\n%s", i, proto.MarshalTextString(gc))
- changes = append(changes, gc)
- }
- return changes, nil
-}
-
-func (c *checkBuild) fetchGerritData(changes []*bbproto.GerritChange) (*igerrit.ChangeRevData, error) {
- // Create an authenticated client for Gerrit RPCs, then fetch all required CL data from Gerrit.
- ctx := context.Background()
- authOpts, err := c.authFlags.Options()
- if err != nil {
- return nil, err
- }
- authedClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, authOpts).Client()
- if err != nil {
- return nil, err
- }
- changeIds := make([]igerrit.ChangeRevKey, 0)
- for _, ch := range changes {
- changeIds = append(changeIds, igerrit.ChangeRevKey{Host: ch.Host, ChangeNum: ch.Change, Revision: int32(ch.Patchset)})
- }
- chRevData, err := igerrit.GetChangeRevData(authedClient, ctx, changeIds)
- if err != nil {
- return nil, fmt.Errorf("Failed to fetch CL data from Gerrit. "+
- "Note that a NotFound error may indicate authorization issues.\n%v", err)
- }
- return chRevData, nil
-}
-
-func readGitilesCommit(gitilesBytes *testplans_pb.ProtoBytes) (*bbproto.GitilesCommit, error) {
- gc := &bbproto.GitilesCommit{}
- if err := proto.Unmarshal(gitilesBytes.SerializedProto, gc); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a GitilesCommit\n%v", gitilesBytes.String(), err)
- }
- log.Printf("Got GitilesCommit proto:\n%s", proto.MarshalTextString(gc))
- return gc, nil
-}
-
-func (c *checkBuild) getRepoToSourceRoot(gc *bbproto.GitilesCommit) (*map[string]map[string]string, error) {
- ctx := context.Background()
- authOpts, err := c.authFlags.Options()
- if err != nil {
- return nil, err
- }
- authedClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, authOpts).Client()
- if err != nil {
- return nil, err
- }
- if gc.Id == "" {
- log.Print("No manifest commit provided. Using 'snapshot' instead.")
- gc.Id = "snapshot"
- }
- repoToRemoteBranchToSrcRoot, err := repo.GetRepoToRemoteBranchToSourceRootFromManifests(authedClient, ctx, gc)
- if err != nil {
- return nil, fmt.Errorf("Error with repo tool call\n%v", err)
- }
- return &repoToRemoteBranchToSrcRoot, nil
-}
-
-func (c *checkBuild) writeOutput(resp *testplans_pb.PointlessBuildCheckResponse) error {
- log.Printf("Full output =\n%s", proto.MarshalTextString(resp))
-
- if len(c.outputJson) > 0 {
- marshal := &jsonpb.Marshaler{EmitDefaults: true, Indent: " "}
- jsonOutput, err := marshal.MarshalToString(resp)
- if err != nil {
- return fmt.Errorf("Failed to marshal JSON %v\n%v", resp, err)
- }
- if err = ioutil.WriteFile(c.outputJson, []byte(jsonOutput), 0644); err != nil {
- return fmt.Errorf("Failed to write output JSON!\n%v", err)
- }
- log.Printf("Wrote output to %s", c.outputJson)
- }
-
- if len(c.outputBinaryPb) > 0 {
- binaryOutput, err := proto.Marshal(resp)
- if err != nil {
- return fmt.Errorf("Failed to marshal binaryproto %v\n%v", resp, err)
- }
- if err = ioutil.WriteFile(c.outputBinaryPb, binaryOutput, 0644); err != nil {
- return fmt.Errorf("Failed to write output binary proto!\n%v", err)
- }
- log.Printf("Wrote output binary proto to %s", c.outputBinaryPb)
- }
-
- return nil
-}
-
-func GetApplication(authOpts auth.Options) *cli.Application {
- return &cli.Application{
- Name: "pointless_build_checker",
-
- Context: func(ctx context.Context) context.Context {
- return ctx
- },
-
- Commands: []*subcommands.Command{
- authcli.SubcommandInfo(authOpts, "auth-info", false),
- authcli.SubcommandLogin(authOpts, "auth-login", false),
- authcli.SubcommandLogout(authOpts, "auth-logout", false),
- cmdCheckBuild(authOpts),
- },
- }
-}
-
-func main() {
- opts := chromeinfra.DefaultAuthOptions()
- opts.Scopes = []string{gerrit.OAuthScope, auth.OAuthScopeEmail}
- app := GetApplication(opts)
- os.Exit(subcommands.Run(app, nil))
-}
diff --git a/cmd/pointless_build_checker/sample/checker_input.json b/cmd/pointless_build_checker/sample/checker_input.json
deleted file mode 100644
index f58c2f7..0000000
--- a/cmd/pointless_build_checker/sample/checker_input.json
+++ /dev/null
@@ -1,160 +0,0 @@
-{
- "depGraph": {
- "buildTarget": {
- "name": "tatl"
- },
- "packageDeps": [
- {
- "dependencyPackages": [
- {
- "category": "app-portage",
- "packageName": "elt-patches",
- "version": "20170815"
- },
- {
- "category": "sys-devel",
- "packageName": "autoconf",
- "version": "2.69"
- },
- {
- "category": "sys-devel",
- "packageName": "automake",
- "version": "1.15.1-r2"
- },
- {
- "category": "sys-devel",
- "packageName": "bison",
- "version": "3.0.4-r1"
- },
- {
- "category": "sys-devel",
- "packageName": "libtool",
- "version": "2.4.6-r6"
- },
- {
- "category": "virtual",
- "packageName": "pkgconfig",
- "version": "0-r1"
- }
- ],
- "dependencySourcePaths": [
- {
- "path": "src/third_party/chromiumos-overlay/eclass"
- },
- {
- "path": "src/third_party/chromiumos-overlay/x11-libs/libxkbcommon/libxkbcommon-0.8.2-r1.ebuild"
- }
- ],
- "packageInfo": {
- "category": "x11-libs",
- "packageName": "libxkbcommon",
- "version": "0.8.2-r1"
- }
- },
- {
- "dependencyPackages": [
- {
- "category": "dev-libs",
- "packageName": "libltdl",
- "version": "2.4.6"
- },
- {
- "category": "media-libs",
- "packageName": "alsa-lib",
- "version": "1.1.6-r3"
- },
- {
- "category": "media-libs",
- "packageName": "libogg",
- "version": "1.3.0"
- },
- {
- "category": "media-libs",
- "packageName": "libvorbis",
- "version": "1.3.5"
- },
- {
- "category": "media-sound",
- "packageName": "gsm",
- "version": "1.0.13"
- },
- {
- "category": "sys-devel",
- "packageName": "autoconf",
- "version": "2.69"
- },
- {
- "category": "sys-devel",
- "packageName": "automake",
- "version": "1.15.1-r2"
- },
- {
- "category": "sys-devel",
- "packageName": "libtool",
- "version": "2.4.6-r6"
- },
- {
- "category": "virtual",
- "packageName": "pkgconfig",
- "version": "0-r1"
- }
- ],
- "dependencySourcePaths": [
- {
- "path": "src/overlays/overlay-tatl"
- },
- {
- "path": "src/overlays/project-termina"
- },
- {
- "path": "src/private-overlays/chromeos-overlay"
- },
- {
- "path": "src/private-overlays/chromeos-partner-overlay"
- },
- {
- "path": "src/third_party/chromiumos-overlay"
- },
- {
- "path": "src/third_party/eclass-overlay"
- },
- {
- "path": "src/third_party/portage-stable"
- }
- ],
- "packageInfo": {
- "category": "media-sound",
- "packageName": "sox",
- "version": "14.4.2-r1"
- }
- },
- {
- "dependencySourcePaths": [
- {
- "path": "src/third_party/chromiumos-overlay/eclass"
- },
- {
- "path": "src/third_party/portage-stable/sys-apps/which/which-2.21.ebuild"
- }
- ],
- "packageInfo": {
- "category": "sys-apps",
- "packageName": "which",
- "version": "2.21"
- }
- }
- ]
- },
- "gerritChanges": [
- {
- "serializedProto": "CiBjaHJvbWl1bS1yZXZpZXcuZ29vZ2xlc291cmNlLmNvbRITY2hyb21pdW1vcy9jaHJvbWl0ZRiolGYgAw=="
- },
- {
- "serializedProto": "CiBjaHJvbWl1bS1yZXZpZXcuZ29vZ2xlc291cmNlLmNvbRIdY2hyb21pdW1vcy90aGlyZF9wYXJ0eS9rZXJuZWwYhpZ1IAE="
- }
- ],
- "gitiles_commit": {
- "serializedProto": "CiBjaHJvbWUtaW50ZXJuYWwuZ29vZ2xlc291cmNlLmNvbRIaY2hyb21lb3MvbWFuaWZlc3QtaW50ZXJuYWwaKGE3N2VlYjUzMTcyMjFlYzhiNmFlMGMwNjI3OTczY2M3YjczYjNlZWY="
- },
- "manifestCommit": "a77eeb5317221ec8b6ae0c0627973cc7b73b3eef"
-}
diff --git a/cmd/test_plan_generator/README.md b/cmd/test_plan_generator/README.md
deleted file mode 100644
index 3e68aa2..0000000
--- a/cmd/test_plan_generator/README.md
+++ /dev/null
@@ -1,16 +0,0 @@
-# Test plan generator
-
-The test plan generator determines which tests need to run for a given set of
-Chrome OS builds.
-
-This is generally intended to be executed by a LUCI recipe. A service account
-credentials file is needed in order to fetch CL data from Gerrit.
-
-```shell
-go run cmd/test_plan_generator/main.go gen-test-plan \
- --input_json=/path/to/input.json \
- --output_json=/path/for/output.json \
- --service-account-json=/path/to/service-account-json-file.json
-```
-
-See the sample directory for data that can be used for local runs.
diff --git a/cmd/test_plan_generator/sample/README.md b/cmd/test_plan_generator/sample/README.md
deleted file mode 100644
index 5b8171a..0000000
--- a/cmd/test_plan_generator/sample/README.md
+++ /dev/null
@@ -1,44 +0,0 @@
-# Test plan generator sample
-
-So you want to try running the test plan generator locally? Cool! Alright,
-you're going to have to do a bit of setup, and you'll have to be a Googler.
-
-1. Have the test_planner repo (you're in it now) checked out (see
- https://chromium.googlesource.com/chromiumos/infra/test_planner).
-1. Have a local chromiumos repo checkout.
-1. Have depot_tools (in particular, the repo command) on your PATH.
-1. Have Golang >=1.12 installed.
-
-OK, now edit gen_test_plan_input.json and replace the REPLACE strings.
-
-This might look something like
-
-```json
-{
- "chromiumos_checkout_root": "/home/sean/chromiumos",
- "buildbucket_protos": [
- {
- "serialized_proto": "SomeBase64EncodedBuildBucketBuildProto"
- },
- {
- "serialized_proto": "SomeOtherBase64EncodedBuildBucketBuildProto"
- }
- ]
-}
-```
-
-Alright, now you'll need to get OAuth credentials to run the program:
-
-```shell
-# Get to this folder in the repo
-# cd test_planner/src/testplans/
-go run cmd/test_plan_generator/main.go auth-login
-```
-
-And now you can actually run it:
-
-```shell
-go run cmd/test_plan_generator/main.go gen-test-plan \
- --input_json=$PWD/cmd/test_plan_generator/sample/input.json \
- --output_json=$PWD/cmd/test_plan_generator/sample/output.json
-```
diff --git a/cmd/test_plan_generator/sample/build_bucket_build_1.cfg b/cmd/test_plan_generator/sample/build_bucket_build_1.cfg
deleted file mode 100644
index 3291f60..0000000
--- a/cmd/test_plan_generator/sample/build_bucket_build_1.cfg
+++ /dev/null
@@ -1,46 +0,0 @@
-{
- "status": "SUCCESS",
- "updateTime": "2019-04-03T23:06:03.857944Z",
- "createdBy": "user:chromeos-postsubmit@chromeos-bot.iam.gserviceaccount.com",
- "builder": {
- "project": "chromeos",
- "builder": "wolf-postsubmit",
- "bucket": "postsubmit"
- },
- "number": 152,
- "id": "8917161684567016032",
- "startTime": "2019-04-03T22:02:54.342805Z",
- "input": {
- "gitilesCommit": {
- "project": "chromeos/manifest-internal",
- "host": "chrome-internal.googlesource.com",
- "ref": "refs/heads/snapshot",
- "id": "a62682ab3278e994f51317a42b99d6409d2cb1ad"
- },
- "properties": {
- "$recipe_engine/runtime": {
- "is_experimental": false,
- "is_luci": true
- },
- "$kitchen": {
- "devshell": false,
- "git_auth": false
- },
- "buildnumber": 256,
- "buildername": "wolf-postsubmit"
- }
- },
- "output": {
- "properties": {
- "build_target": {
- "name": "wolf"
- },
- "artifacts": {
- "gs_bucket": "gs://some-bucket-name",
- "gs_path": "wolf-fake/R1-0.0.0"
- }
- }
- },
- "endTime": "2019-04-03T23:06:03.176817Z",
- "createTime": "2019-04-03T22:02:28.394159Z"
-}
diff --git a/cmd/test_plan_generator/sample/build_bucket_build_2.cfg b/cmd/test_plan_generator/sample/build_bucket_build_2.cfg
deleted file mode 100644
index a8db568..0000000
--- a/cmd/test_plan_generator/sample/build_bucket_build_2.cfg
+++ /dev/null
@@ -1,52 +0,0 @@
-{
- "status": "SUCCESS",
- "updateTime": "2019-04-03T23:06:03.857944Z",
- "createdBy": "user:chromeos@chromeos-bot.iam.gserviceaccount.com",
- "builder": {
- "project": "chromeos",
- "builder": "reef-postsubmit",
- "bucket": "postsubmit"
- },
- "number": 152,
- "id": "8917161684567016032",
- "startTime": "2019-04-03T22:02:54.342805Z",
- "input": {
- "gerritChanges": [
- {
- "host": "chromium-review.googlesource.com",
- "change": "1384752",
- "patchset": "1"
- },
- {
- "host": "chrome-internal-review.googlesource.com",
- "change": "1072090",
- "patchset": "1"
- }
- ],
- "properties": {
- "$recipe_engine/runtime": {
- "is_experimental": false,
- "is_luci": true
- },
- "$kitchen": {
- "devshell": false,
- "git_auth": false
- },
- "buildnumber": 256,
- "buildername": "reef-paladin"
- }
- },
- "output": {
- "properties": {
- "build_target": {
- "name": "reef"
- },
- "artifacts": {
- "gs_bucket": "gs://some-bucket-name",
- "gs_path": "reef-fake/R1-0.0.0"
- }
- }
- },
- "endTime": "2019-04-03T23:06:03.176817Z",
- "createTime": "2019-04-03T22:02:28.394159Z"
-}
diff --git a/cmd/test_plan_generator/sample/gen_test_plan_input.json b/cmd/test_plan_generator/sample/gen_test_plan_input.json
deleted file mode 100644
index 1f87681..0000000
--- a/cmd/test_plan_generator/sample/gen_test_plan_input.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "chromiumos_checkout_root": "REPLACE_WITH_PATH_TO_YOUR_CHROMIUMOS_REPO",
- "buildbucket_protos": [
- {
- "serialized_proto": "COCE3arp6YfgexInCghjaHJvbWVvcxIKcG9zdHN1Ym1pdBoPd29sZi1wb3N0c3VibWl0GJgBIj11c2VyOmNocm9tZW9zLXBvc3RzdWJtaXRAY2hyb21lb3MtYm90LmlhbS5nc2VydmljZWFjY291bnQuY29tMgwI9NqU5QUQmMf5uwE6DAiO25TlBRCIlLujAUILCNv4lOUFEOiGqFRKDAjb+JTlBRDA34yZA2AMerICCrABCi4KCCRraXRjaGVuEiIqIAoOCghkZXZzaGVsbBICIAAKDgoIZ2l0X2F1dGgSAiAAChgKC2J1aWxkbnVtYmVyEgkRAAAAAAAAcEAKIAoLYnVpbGRlcm5hbWUSERoPd29sZi1wb3N0c3VibWl0CkIKFiRyZWNpcGVfZW5naW5lL3J1bnRpbWUSKComChUKD2lzX2V4cGVyaW1lbnRhbBICIAAKDQoHaXNfbHVjaRICIAESfQogY2hyb21lLWludGVybmFsLmdvb2dsZXNvdXJjZS5jb20SGmNocm9tZW9zL21hbmlmZXN0LWludGVybmFsGihhNjI2ODJhYjMyNzhlOTk0ZjUxMzE3YTQyYjk5ZDY0MDlkMmNiMWFkIhNyZWZzL2hlYWRzL3NuYXBzaG90ggF/Cn0KIgoMYnVpbGRfdGFyZ2V0EhIqEAoOCgRuYW1lEgYaBHdvbGYKVwoJYXJ0aWZhY3RzEkoqSAokCglnc19idWNrZXQSFxoVZ3M6Ly9zb21lLWJ1Y2tldC1uYW1lCiAKB2dzX3BhdGgSFRoTd29sZi1kdW1teS9SMS0wLjAuMA=="
- },
- {
- "serialized_proto": "COCE3arp6YfgexInCghjaHJvbWVvcxIKcG9zdHN1Ym1pdBoPcmVlZi1wb3N0c3VibWl0GJgBIjJ1c2VyOmNocm9tZW9zQGNocm9tZW9zLWJvdC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbTIMCPTalOUFEJjH+bsBOgwIjtuU5QUQiJS7owFCCwjb+JTlBRDohqhUSgwI2/iU5QUQwN+MmQNgDHqLAgqtAQpCChYkcmVjaXBlX2VuZ2luZS9ydW50aW1lEigqJgoNCgdpc19sdWNpEgIgAQoVCg9pc19leHBlcmltZW50YWwSAiAACi4KCCRraXRjaGVuEiIqIAoOCghkZXZzaGVsbBICIAAKDgoIZ2l0X2F1dGgSAiAAChgKC2J1aWxkbnVtYmVyEgkRAAAAAAAAcEAKHQoLYnVpbGRlcm5hbWUSDhoMcmVlZi1wYWxhZGluGigKIGNocm9taXVtLXJldmlldy5nb29nbGVzb3VyY2UuY29tGLDCVCABGi8KJ2Nocm9tZS1pbnRlcm5hbC1yZXZpZXcuZ29vZ2xlc291cmNlLmNvbRjat0EgAYIBfwp9CiIKDGJ1aWxkX3RhcmdldBISKhAKDgoEbmFtZRIGGgRyZWVmClcKCWFydGlmYWN0cxJKKkgKJAoJZ3NfYnVja2V0EhcaFWdzOi8vc29tZS1idWNrZXQtbmFtZQogCgdnc19wYXRoEhUaE3JlZWYtZHVtbXkvUjEtMC4wLjA="
- }
- ]
-}
diff --git a/cmd/test_plan_generator/test_plan_generator.go b/cmd/test_plan_generator/test_plan_generator.go
deleted file mode 100644
index 7edda62..0000000
--- a/cmd/test_plan_generator/test_plan_generator.go
+++ /dev/null
@@ -1,380 +0,0 @@
-// Copyright 2019 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-package main
-
-import (
- "bytes"
- "context"
- "flag"
- "fmt"
- "io/ioutil"
- "log"
- "os"
- "path"
-
- "github.com/golang/protobuf/jsonpb"
- "github.com/golang/protobuf/proto"
- "github.com/maruel/subcommands"
- "go.chromium.org/chromiumos/infra/go/internal/generator"
- igerrit "go.chromium.org/chromiumos/infra/go/internal/gerrit"
- "go.chromium.org/chromiumos/infra/go/internal/repo"
- "go.chromium.org/chromiumos/infra/proto/go/testplans"
- "go.chromium.org/luci/auth"
- "go.chromium.org/luci/auth/client/authcli"
- bbproto "go.chromium.org/luci/buildbucket/proto"
- "go.chromium.org/luci/common/api/gerrit"
- "go.chromium.org/luci/common/cli"
- "go.chromium.org/luci/hardcoded/chromeinfra"
-)
-
-const (
- boardPriorityConfigPath = "testingconfig/generated/board_priority.binaryproto"
- sourceTreeTestConfigPath = "testingconfig/generated/source_tree_test_config.binaryproto"
- targetTestRequirementsPath = "testingconfig/generated/target_test_requirements.binaryproto"
-)
-
-var (
- unmarshaler = jsonpb.Unmarshaler{AllowUnknownFields: true}
-)
-
-func cmdGenTestPlan(authOpts auth.Options) *subcommands.Command {
- return &subcommands.Command{
- UsageLine: "gen-test-plan --input_json=/path/to/input.json --output_json=/path/to/output.json",
- ShortDesc: "Generates a test plan",
- LongDesc: "Generates a test plan",
- CommandRun: func() subcommands.CommandRun {
- c := &getTestPlanRun{}
- c.authFlags = authcli.Flags{}
- c.authFlags.Register(c.GetFlags(), authOpts)
- c.Flags.StringVar(&c.inputJson, "input_json", "", "Path to JSON proto representing a GenerateTestPlanRequest")
- c.Flags.StringVar(&c.outputJson, "output_json", "", "Path to file to write output GenerateTestPlanResponse JSON proto")
- c.Flags.StringVar(&c.inputBinaryPb, "input_binary_pb", "", "Path to binaryproto file representing a GenerateTestPlanRequest")
- c.Flags.StringVar(&c.outputBinaryPb, "output_binary_pb", "", "Path to file to write output GenerateTestPlanResponse binaryproto")
- c.Flags.StringVar(&c.localConfigDir, "local_config_dir", "", "Path to an infra/config checkout, to be used rather than origin HEAD")
- c.Flags.StringVar(&c.manifestFile, "manifest_file", "", "Path to local manifest file. If given, will be used instead of default snapshot.xml")
- return c
- }}
-}
-
-func (c *getTestPlanRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
- flag.Parse()
-
- req, err := c.readInput()
- if err != nil {
- log.Print(err)
- return 1
- }
-
- var boardPriorityList *testplans.BoardPriorityList
- var sourceTreeConfig *testplans.SourceTreeTestCfg
- var testReqsConfig *testplans.TargetTestRequirementsCfg
- if c.localConfigDir == "" {
- boardPriorityList, sourceTreeConfig, testReqsConfig, err = c.fetchConfigFromGitiles()
- } else {
- boardPriorityList, sourceTreeConfig, testReqsConfig, err = c.readLocalConfigFiles()
- }
- if err != nil {
- log.Print(err)
- return 2
- }
-
- bbBuilds, err := readBuildbucketBuilds(req.BuildbucketProtos)
- if err != nil {
- log.Print(err)
- return 3
- }
-
- gerritChanges, err := readGerritChanges(req.GerritChanges)
- if err != nil {
- log.Print(err)
- return 8
- }
-
- changeRevs, err := c.fetchGerritData(gerritChanges)
- if err != nil {
- log.Print(err)
- return 4
- }
-
- var repoToSrcRoot *map[string]map[string]string
- // If we have a local manifest file provided, use that. Else get it from Gerrit.
- if c.manifestFile == "" {
- gitilesCommit, err := readGitilesCommit(req.GitilesCommit)
- if err != nil {
- log.Print(err)
- return 5
- }
-
- repoToSrcRoot, err = c.getRepoToSourceRoot(gitilesCommit)
- if err != nil {
- log.Print(err)
- return 6
- }
- } else {
- log.Printf("Reading local manifest from %s", c.manifestFile)
- repoToSrcRootMap, err := repo.GetRepoToRemoteBranchToSourceRootFromManifestFile(c.manifestFile)
- if err != nil {
- log.Print(err)
- return 9
- }
- repoToSrcRoot = &repoToSrcRootMap
- }
-
- testPlan, err := generator.CreateTestPlan(testReqsConfig, sourceTreeConfig, boardPriorityList, bbBuilds, gerritChanges, changeRevs, *repoToSrcRoot)
- if err != nil {
- log.Printf("Error creating test plan:\n%v", err)
- return 7
- }
-
- if err = c.writeOutput(testPlan); err != nil {
- log.Print(err)
- return 8
- }
- return 0
-}
-
-type getTestPlanRun struct {
- subcommands.CommandRunBase
- authFlags authcli.Flags
- inputJson string
- outputJson string
- inputBinaryPb string
- outputBinaryPb string
- localConfigDir string
- manifestFile string
-}
-
-func (c *getTestPlanRun) readInput() (*testplans.GenerateTestPlanRequest, error) {
- // use input_binary_pb if it's specified
- if len(c.inputBinaryPb) > 0 {
- inputPb, err := ioutil.ReadFile(c.inputBinaryPb)
- if err != nil {
- return nil, fmt.Errorf("Failed reason input_binary_pb\n%v", err)
- }
- req := &testplans.GenerateTestPlanRequest{}
- if err := proto.Unmarshal(inputPb, req); err != nil {
- return nil, fmt.Errorf("Failed parsing input_binary_pb as proto\n%v", err)
- }
- return req, nil
- // otherwise use input_json
- } else {
- inputBytes, err := ioutil.ReadFile(c.inputJson)
- if err != nil {
- return nil, fmt.Errorf("Failed reading input_json\n%v", err)
- }
- req := &testplans.GenerateTestPlanRequest{}
- if err := unmarshaler.Unmarshal(bytes.NewReader(inputBytes), req); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a GenerateTestPlanRequest\n%v", c.inputJson, err)
- }
- return req, nil
- }
-}
-
-func (c *getTestPlanRun) fetchConfigFromGitiles() (*testplans.BoardPriorityList, *testplans.SourceTreeTestCfg, *testplans.TargetTestRequirementsCfg, error) {
- // Create an authenticated client for Gerrit RPCs, then fetch all required CL data from Gerrit.
- ctx := context.Background()
- authOpts, err := c.authFlags.Options()
- if err != nil {
- return nil, nil, nil, err
- }
- authedClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, authOpts).Client()
- if err != nil {
- return nil, nil, nil, err
- }
-
- m, err := igerrit.FetchFilesFromGitiles(authedClient, ctx,
- "chrome-internal.googlesource.com",
- "chromeos/infra/config",
- "main",
- []string{boardPriorityConfigPath, sourceTreeTestConfigPath, targetTestRequirementsPath})
- if err != nil {
- return nil, nil, nil, err
- }
-
- boardPriorityList := &testplans.BoardPriorityList{}
- if err := proto.Unmarshal([]byte((*m)[boardPriorityConfigPath]), boardPriorityList); err != nil {
- return nil, nil, nil, fmt.Errorf("Couldn't decode %s as a BoardPriorityList\n%v", (*m)[boardPriorityConfigPath], err)
- }
- sourceTreeConfig := &testplans.SourceTreeTestCfg{}
- if err := proto.Unmarshal([]byte((*m)[sourceTreeTestConfigPath]), sourceTreeConfig); err != nil {
- return nil, nil, nil, fmt.Errorf("Couldn't decode %s as a SourceTreeTestCfg\n%v", (*m)[sourceTreeTestConfigPath], err)
- }
- testReqsConfig := &testplans.TargetTestRequirementsCfg{}
- if err := proto.Unmarshal([]byte((*m)[targetTestRequirementsPath]), testReqsConfig); err != nil {
- return nil, nil, nil, fmt.Errorf("Couldn't decode %s as a TargetTestRequirementsCfg\n%v", (*m)[targetTestRequirementsPath], err)
- }
- log.Printf("Fetched config from Gitiles:\n%s\n\n%s\n\n%s", proto.MarshalTextString(boardPriorityList),
- proto.MarshalTextString(sourceTreeConfig), proto.MarshalTextString(testReqsConfig))
- return boardPriorityList, sourceTreeConfig, testReqsConfig, nil
-}
-
-func (c *getTestPlanRun) readLocalConfigFiles() (*testplans.BoardPriorityList, *testplans.SourceTreeTestCfg, *testplans.TargetTestRequirementsCfg, error) {
- log.Print("--------------------------------------------")
- log.Print("WARNING: Reading config from local dir.")
- log.Print("Be sure that you've run `./regenerate_configs.sh -b` first to generate binaryproto files")
- log.Print("--------------------------------------------")
-
- bplBytes, err := ioutil.ReadFile(path.Join(c.localConfigDir, boardPriorityConfigPath))
- if err != nil {
- return nil, nil, nil, fmt.Errorf("couldn't read BoardPriorityList file: %v", err)
- }
- boardPriorityList := &testplans.BoardPriorityList{}
- if err := proto.Unmarshal(bplBytes, boardPriorityList); err != nil {
- return nil, nil, nil, fmt.Errorf("couldn't decode file as BoardPriorityList: %v", err)
- }
-
- stcBytes, err := ioutil.ReadFile(path.Join(c.localConfigDir, sourceTreeTestConfigPath))
- if err != nil {
- return nil, nil, nil, fmt.Errorf("couldn't read SourceTreeTestCfg file: %v", err)
- }
- sourceTreeConfig := &testplans.SourceTreeTestCfg{}
- if err := proto.Unmarshal(stcBytes, sourceTreeConfig); err != nil {
- return nil, nil, nil, fmt.Errorf("couldn't decode file as SourceTreeTestCfg: %v", err)
- }
-
- ttrBytes, err := ioutil.ReadFile(path.Join(c.localConfigDir, targetTestRequirementsPath))
- if err != nil {
- return nil, nil, nil, fmt.Errorf("couldn't read TargetTestRequirementsCfg file: %v", err)
- }
- testReqsConfig := &testplans.TargetTestRequirementsCfg{}
- if err := proto.Unmarshal(ttrBytes, testReqsConfig); err != nil {
- return nil, nil, nil, fmt.Errorf("couldn't decode file as TargetTestRequirementsCfg: %v", err)
- }
- log.Printf("Read local config:\n%s\n\n%s\n\n%s", proto.MarshalTextString(boardPriorityList),
- proto.MarshalTextString(sourceTreeConfig), proto.MarshalTextString(testReqsConfig))
- return boardPriorityList, sourceTreeConfig, testReqsConfig, nil
-}
-
-func readBuildbucketBuilds(bbBuildsBytes []*testplans.ProtoBytes) ([]*bbproto.Build, error) {
- bbBuilds := make([]*bbproto.Build, 0)
- for _, bbBuildBytes := range bbBuildsBytes {
- bbBuild := &bbproto.Build{}
- if err := proto.Unmarshal(bbBuildBytes.SerializedProto, bbBuild); err != nil {
- return bbBuilds, fmt.Errorf("Couldn't decode %s as a Buildbucket Build\n%v", bbBuildBytes.String(), err)
- }
- bbBuilds = append(bbBuilds, bbBuild)
- }
- if len(bbBuilds) > 0 {
- log.Printf("Sample buildbucket proto:\n%s", proto.MarshalTextString(bbBuilds[0]))
- }
- return bbBuilds, nil
-}
-
-func readGerritChanges(changesBytes []*testplans.ProtoBytes) ([]*bbproto.GerritChange, error) {
- changes := make([]*bbproto.GerritChange, 0)
- for _, changeBytes := range changesBytes {
- change := &bbproto.GerritChange{}
- if err := proto.Unmarshal(changeBytes.SerializedProto, change); err != nil {
- return changes, fmt.Errorf("Couldn't decode %s as a GerritChange\n%v", changeBytes.String(), err)
- }
- changes = append(changes, change)
- }
- if len(changes) > 0 {
- log.Printf("Sample GerritChange proto:\n%s", proto.MarshalTextString(changes[0]))
- }
- return changes, nil
-}
-
-func (c *getTestPlanRun) fetchGerritData(changes []*bbproto.GerritChange) (*igerrit.ChangeRevData, error) {
- // Create an authenticated client for Gerrit RPCs, then fetch all required CL data from Gerrit.
- ctx := context.Background()
- authOpts, err := c.authFlags.Options()
- if err != nil {
- return nil, err
- }
- authedClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, authOpts).Client()
- if err != nil {
- return nil, err
- }
- changeIds := make([]igerrit.ChangeRevKey, 0)
- for _, c := range changes {
- changeIds = append(changeIds, igerrit.ChangeRevKey{Host: c.Host, ChangeNum: c.Change, Revision: int32(c.Patchset)})
- }
- chRevData, err := igerrit.GetChangeRevData(authedClient, ctx, changeIds)
- if err != nil {
- return nil, fmt.Errorf("Failed to fetch CL data from Gerrit. "+
- "Note that a NotFound error may indicate authorization issues.\n%v", err)
- }
- return chRevData, nil
-}
-
-func readGitilesCommit(gitilesBytes *testplans.ProtoBytes) (*bbproto.GitilesCommit, error) {
- gc := &bbproto.GitilesCommit{}
- if err := proto.Unmarshal(gitilesBytes.SerializedProto, gc); err != nil {
- return nil, fmt.Errorf("Couldn't decode %s as a GitilesCommit\n%v", gitilesBytes.String(), err)
- }
- log.Printf("Got GitilesCommit proto:\n%s", proto.MarshalTextString(gc))
- return gc, nil
-}
-
-func (c *getTestPlanRun) getRepoToSourceRoot(gc *bbproto.GitilesCommit) (*map[string]map[string]string, error) {
- ctx := context.Background()
- authOpts, err := c.authFlags.Options()
- if err != nil {
- return nil, err
- }
- authedClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, authOpts).Client()
- if err != nil {
- return nil, err
- }
- if gc.Id == "" {
- log.Print("No manifest commit provided. Using 'snapshot' instead.")
- gc.Id = "snapshot"
- }
- repoToRemoteBranchToSrcRoot, err := repo.GetRepoToRemoteBranchToSourceRootFromManifests(authedClient, ctx, gc)
- if err != nil {
- return nil, fmt.Errorf("Error with repo tool call\n%v", err)
- }
- return &repoToRemoteBranchToSrcRoot, nil
-}
-func (c *getTestPlanRun) writeOutput(tp *testplans.GenerateTestPlanResponse) error {
- if len(c.outputJson) > 0 {
- marshal := &jsonpb.Marshaler{EmitDefaults: true, Indent: " "}
- jsonOutput, err := marshal.MarshalToString(tp)
- if err != nil {
- return fmt.Errorf("Failed to marshal JSON %v\n%v", tp, err)
- }
- if err = ioutil.WriteFile(c.outputJson, []byte(jsonOutput), 0644); err != nil {
- return fmt.Errorf("Failed to write output JSON!\n%v", err)
- }
- log.Printf("Wrote output JSON to %s", c.outputJson)
- }
-
- if len(c.outputBinaryPb) > 0 {
- binaryOutput, err := proto.Marshal(tp)
- if err != nil {
- return fmt.Errorf("Failed to marshal binaryproto %v\n%v", tp, err)
- }
- if err = ioutil.WriteFile(c.outputBinaryPb, binaryOutput, 0644); err != nil {
- return fmt.Errorf("Failed to write output binary proto!\n%v", err)
- }
- log.Printf("Wrote output binary proto to %s", c.outputBinaryPb)
- }
-
- return nil
-}
-
-func GetApplication(authOpts auth.Options) *cli.Application {
- return &cli.Application{
- Name: "test_planner",
-
- Context: func(ctx context.Context) context.Context {
- return ctx
- },
-
- Commands: []*subcommands.Command{
- authcli.SubcommandInfo(authOpts, "auth-info", false),
- authcli.SubcommandLogin(authOpts, "auth-login", false),
- authcli.SubcommandLogout(authOpts, "auth-logout", false),
- cmdGenTestPlan(authOpts),
- },
- }
-}
-
-func main() {
- opts := chromeinfra.DefaultAuthOptions()
- opts.Scopes = []string{gerrit.OAuthScope, auth.OAuthScopeEmail}
- app := GetApplication(opts)
- os.Exit(subcommands.Run(app, nil))
-}