feat(internal/postprocessor): only regen snippets for changed modules (#7300)

This change identifies the changed modules in the PR using the commit URLs in the open OwlBot PR description. The scopes are used to limit snippet regeneration to only those modules.

Author: adrianajg

internal/postprocessor/main.go

@@ -72,14 +72,12 @@ func main() {
 	log.Println("googleapis-dir set to", *googleapisDir)
 	log.Println("branch set to", *branchOverride)
 	log.Println("prFilepath is", *prFilepath)
+	log.Println("directories are", *directories)
 
-	var modules []string
+	dirSlice := []string{}
 	if *directories != "" {
 		dirSlice := strings.Split(*directories, ",")
-		for _, dir := range dirSlice {
-			modules = append(modules, filepath.Join(*clientRoot, dir))
-		}
-		log.Println("Postprocessor running on", modules)
+		log.Println("Postprocessor running on", dirSlice)
 	} else {
 		log.Println("Postprocessor running on all modules.")
 	}
@@ -103,11 +101,13 @@ func main() {
 	c := &config{
 		googleapisDir:  *googleapisDir,
 		googleCloudDir: *clientRoot,
-		modules:        modules,
+		modules:        dirSlice,
 		branchOverride: *branchOverride,
 		githubUsername: *githubUsername,
 		prFilepath:     *prFilepath,
 		runAll:         runAll,
+		prTitle:        "",
+		prBody:         "",
 	}
 
 	if err := c.run(ctx); err != nil {
@@ -120,11 +120,19 @@ func main() {
 type config struct {
 	googleapisDir  string
 	googleCloudDir string
-	modules        []string
+
+	// At this time modules are either provided at the time of invocation locally
+	// and extracted from the open OwlBot PR description. If we would like
+	// the postprocessor to be able to be run on non-OwlBot PRs, we would
+	// need to change the method of populating this field.
+	modules []string
+
 	branchOverride string
 	githubUsername string
 	prFilepath     string
 	runAll         bool
+	prTitle        string
+	prBody         string
 }
 
 // runAll uses git to tell if the PR being updated should run all post
@@ -150,7 +158,16 @@ func (c *config) run(ctx context.Context) error {
 		log.Println("exiting post processing early")
 		return nil
 	}
-	if err := gocmd.ModTidyAll(c.googleCloudDir); err != nil {
+	// TODO(guadriana): Once the PR for initializing new modules is merged, we
+	// will need to add logic here that sets c.modules to modConfigs (a slice of
+	// all modules) if branchOverride != ""
+
+	// TODO(codyoss): In the future we may want to make it possible to be able
+	// to run this locally with a user defined remote branch.
+	if err := c.SetScopesAndPRInfo(ctx); err != nil {
+		return err
+	}
+	if err := c.TidyAffectedMods(); err != nil {
 		return err
 	}
 	if err := gocmd.Vet(c.googleCloudDir); err != nil {
@@ -162,28 +179,44 @@ func (c *config) run(ctx context.Context) error {
 	if _, err := c.Manifest(generator.MicrogenGapicConfigs); err != nil {
 		return err
 	}
-	// TODO(codyoss): In the future we may want to make it possible to be able
-	// to run this locally with a user defined remote branch.
-	if err := c.AmendPRDescription(ctx); err != nil {
+	if err := c.WritePRInfoToFile(); err != nil {
 		return err
 	}
 	return nil
 }
 
+func (c *config) getDirs() []string {
+	dirs := []string{}
+	for _, module := range c.modules {
+		dirs = append(dirs, filepath.Join(c.googleCloudDir, module))
+	}
+	return dirs
+}
+
+func (c *config) TidyAffectedMods() error {
+	dirs := c.getDirs()
+	for _, dir := range dirs {
+		if err := gocmd.ModTidy(dir); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // RegenSnippets regenerates the snippets for all GAPICs configured to be generated.
 func (c *config) RegenSnippets() error {
 	log.Println("regenerating snippets")
-
 	snippetDir := filepath.Join(c.googleCloudDir, "internal", "generated", "snippets")
-	apiShortnames, err := generator.ParseAPIShortnames(c.googleapisDir, generator.MicrogenGapicConfigs, generator.ManualEntries)
-
+	confs := c.getChangedClientConfs()
+	apiShortnames, err := generator.ParseAPIShortnames(c.googleapisDir, confs, generator.ManualEntries)
 	if err != nil {
 		return err
 	}
-	if err := gensnippets.GenerateSnippetsDirs(c.googleCloudDir, snippetDir, apiShortnames, c.modules); err != nil {
+	dirs := c.getDirs()
+	if err := gensnippets.GenerateSnippetsDirs(c.googleCloudDir, snippetDir, apiShortnames, dirs); err != nil {
 		log.Printf("warning: got the following non-fatal errors generating snippets: %v", err)
 	}
-	if err := c.replaceAllForSnippets(c.googleCloudDir, snippetDir); err != nil {
+	if err := c.replaceAllForSnippets(snippetDir); err != nil {
 		return err
 	}
 	if err := gocmd.ModTidy(snippetDir); err != nil {
@@ -193,15 +226,44 @@ func (c *config) RegenSnippets() error {
 	return nil
 }
 
-func (c *config) replaceAllForSnippets(googleCloudDir, snippetDir string) error {
-	return execv.ForEachMod(googleCloudDir, func(dir string) error {
+// getChangedClientConfs iterates through the MicrogenGapicConfigs and returns
+// a slice of the entries corresponding to modified modules and clients
+func (c *config) getChangedClientConfs() []*generator.MicrogenConfig {
+	if len(c.modules) != 0 {
+		runConfs := []*generator.MicrogenConfig{}
+		for _, conf := range generator.MicrogenGapicConfigs {
+			for _, scope := range c.modules {
+				scopePathElement := "/" + scope + "/"
+				if strings.Contains(conf.InputDirectoryPath, scopePathElement) {
+					runConfs = append(runConfs, conf)
+				}
+			}
+		}
+		return runConfs
+	}
+	return generator.MicrogenGapicConfigs
+}
+
+func (c *config) replaceAllForSnippets(snippetDir string) error {
+	return execv.ForEachMod(c.googleCloudDir, func(dir string) error {
+		processMod := false
 		if c.modules != nil {
+			// Checking each path component in its entirety prevents mistaken addition of modules whose names
+			// contain the scope as a substring. For example if the scope is "video" we do not want to regenerate
+			// snippets for "videointelligence"
+			dirSlice := strings.Split(dir, "/")
 			for _, mod := range c.modules {
-				if !strings.Contains(dir, mod) {
-					return nil
+				for _, dirElem := range dirSlice {
+					if mod == dirElem {
+						processMod = true
+						break
+					}
 				}
 			}
 		}
+		if !processMod {
+			return nil
+		}
 		if dir == snippetDir {
 			return nil
 		}
@@ -280,7 +342,7 @@ func docURL(cloudDir, importPath string) (string, error) {
 	return "https://cloud.google.com/go/docs/reference/" + mod + "/latest/" + pkgPath, nil
 }
 
-func (c *config) AmendPRDescription(ctx context.Context) error {
+func (c *config) SetScopesAndPRInfo(ctx context.Context) error {
 	log.Println("Amending PR title and body")
 	pr, err := c.getPR(ctx)
 	if err != nil {
@@ -290,13 +352,16 @@ func (c *config) AmendPRDescription(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	return c.writePRCommitToFile(newPRTitle, newPRBody)
+	c.prTitle = newPRTitle
+	c.prBody = newPRBody
+	return nil
 }
 
 func (c *config) processCommit(title, body string) (string, string, error) {
 	var newPRTitle string
 	var commitTitle string
 	var commitTitleIndex int
+	var modules []string
 
 	bodySlice := strings.Split(body, "\n")
 	for index, line := range bodySlice {
@@ -312,7 +377,12 @@ func (c *config) processCommit(title, body string) (string, string, error) {
 			continue
 		}
 		hash := extractHashFromLine(line)
-		scope, err := c.getScopeFromGoogleapisCommitHash(hash)
+		scopes, err := c.getScopesFromGoogleapisCommitHash(hash)
+		modules = append(modules, scopes...)
+		var scope string
+		if len(scopes) == 1 {
+			scope = scopes[0]
+		}
 		if err != nil {
 			return "", "", err
 		}
@@ -324,6 +394,7 @@ func (c *config) processCommit(title, body string) (string, string, error) {
 		bodySlice[commitTitleIndex] = newCommitTitle
 	}
 	body = strings.Join(bodySlice, "\n")
+	c.modules = append(c.modules, modules...)
 	return newPRTitle, body, nil
 }
 
@@ -349,14 +420,14 @@ func (c *config) getPR(ctx context.Context) (*github.PullRequest, error) {
 	return owlbotPR, nil
 }
 
-func (c *config) getScopeFromGoogleapisCommitHash(commitHash string) (string, error) {
+func (c *config) getScopesFromGoogleapisCommitHash(commitHash string) ([]string, error) {
 	files, err := c.filesChanged(commitHash)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	// if no files changed, return empty string
 	if len(files) == 0 {
-		return "", nil
+		return nil, nil
 	}
 	scopesMap := make(map[string]bool)
 	scopes := []string{}
@@ -375,12 +446,7 @@ func (c *config) getScopeFromGoogleapisCommitHash(commitHash string) (string, er
 			}
 		}
 	}
-	// if no in-scope packages are found or if many packages found, return empty string
-	if len(scopes) != 1 {
-		return "", nil
-	}
-	// if single scope found, return
-	return scopes[0], nil
+	return scopes, nil
 }
 
 // filesChanged returns a list of files changed in a commit for the provdied
@@ -421,8 +487,9 @@ func updateCommitTitle(title, titlePkg string) string {
 	return newTitle
 }
 
-// writePRCommitToFile uses OwlBot env variable specified path to write updated PR title and body at that location
-func (c *config) writePRCommitToFile(title, body string) error {
+// WritePRInfoToFile uses OwlBot env variable specified path to write updated
+// PR title and body at that location
+func (c *config) WritePRInfoToFile() error {
 	// if file exists at location, delete
 	if err := os.Remove(c.prFilepath); err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
@@ -436,7 +503,12 @@ func (c *config) writePRCommitToFile(title, body string) error {
 		return err
 	}
 	defer f.Close()
-	if _, err := f.WriteString(fmt.Sprintf("%s\n\n%s", title, body)); err != nil {
+	if c.prTitle == "" && c.prBody == "" {
+		log.Println("No updated PR info found, will not write PR title and description to file.")
+		return nil
+	}
+	log.Println("Writing PR title and description to file.")
+	if _, err := f.WriteString(fmt.Sprintf("%s\n\n%s", c.prTitle, c.prBody)); err != nil {
 		return err
 	}
 	return nil