feat: DependsOn files (#305)

- Add ability to depend on files - Convert file names to terraform naming convention (snake_case). - Validate file name collisions in config implements #304
vincenthsh · May 24, 2024 · 76fd9e3 · 76fd9e3
1 parent 3320560
commit 76fd9e3
Show file tree

Hide file tree

Showing 21 changed files with 313 additions and 68 deletions.
diff --git a/apply/apply.go b/apply/apply.go
@@ -29,8 +29,6 @@ import (
 	"github.com/spf13/afero"
 )
 
-const rootPath = "terraform"
-
 // Apply will run a plan and apply all the changes to the current repo.
 func Apply(fs afero.Fs, conf *v2.Config, tmpl *templates.T, upgrade bool) error {
 	if !upgrade {
@@ -241,7 +239,7 @@ func applyTFE(fs afero.Fs, plan *plan.Plan, tmpl *templates.T) error {
 	}
 
 	logrus.Debug("applying tfe")
-	path := fmt.Sprintf("%s/tfe", rootPath)
+	path := fmt.Sprintf("%s/tfe", util.RootPath)
 	err := fs.MkdirAll(path, 0755)
 	if err != nil {
 		return errors.Wrapf(err, "unable to make directory %s", path)
@@ -323,7 +321,7 @@ func applyRepo(fs afero.Fs, p *plan.Plan, repoTemplates, commonTemplates fs.FS)
 
 func applyGlobal(fs afero.Fs, p plan.Component, repoBox, commonBox fs.FS) error {
 	logrus.Debug("applying global")
-	path := fmt.Sprintf("%s/global", rootPath)
+	path := fmt.Sprintf("%s/global", util.RootPath)
 	e := fs.MkdirAll(path, 0755)
 	if e != nil {
 		return errs.WrapUserf(e, "unable to make directory %s", path)
@@ -333,7 +331,7 @@ func applyGlobal(fs afero.Fs, p plan.Component, repoBox, commonBox fs.FS) error
 
 func applyAccounts(fs afero.Fs, p *plan.Plan, accountBox, commonBox fs.FS) (e error) {
 	for account, accountPlan := range p.Accounts {
-		path := fmt.Sprintf("%s/accounts/%s", rootPath, account)
+		path := fmt.Sprintf("%s/accounts/%s", util.RootPath, account)
 		e = fs.MkdirAll(path, 0755)
 		if e != nil {
 			return errs.WrapUser(e, "unable to make directories for accounts")
@@ -348,7 +346,7 @@ func applyAccounts(fs afero.Fs, p *plan.Plan, accountBox, commonBox fs.FS) (e er
 
 func applyModules(fs afero.Fs, p map[string]plan.Module, moduleBox, commonBox fs.FS) (e error) {
 	for module, modulePlan := range p {
-		path := fmt.Sprintf("%s/modules/%s", rootPath, module)
+		path := fmt.Sprintf("%s/modules/%s", util.RootPath, module)
 		e = fs.MkdirAll(path, 0755)
 		if e != nil {
 			return errs.WrapUserf(e, "unable to make path %s", path)
@@ -373,7 +371,7 @@ func applyEnvs(
 	pathModuleConfigs = make(PathModuleConfigs)
 	for env, envPlan := range p.Envs {
 		logrus.Debugf("applying %s", env)
-		path := fmt.Sprintf("%s/envs/%s", rootPath, env)
+		path := fmt.Sprintf("%s/envs/%s", util.RootPath, env)
 		err = fs.MkdirAll(path, 0755)
 		if err != nil {
 			return nil, errs.WrapUserf(err, "unable to make directory %s", path)
@@ -384,7 +382,7 @@ func applyEnvs(
 		}
 		reg := registry.NewClient(nil, nil)
 		for component, componentPlan := range envPlan.Components {
-			path = fmt.Sprintf("%s/envs/%s/%s", rootPath, env, component)
+			path = fmt.Sprintf("%s/envs/%s/%s", util.RootPath, env, component)
 			err = fs.MkdirAll(path, 0755)
 			if err != nil {
 				return nil, errs.WrapUser(err, "unable to make directories for component")
@@ -398,6 +396,25 @@ func applyEnvs(
 				return nil, errs.NewUserf("component of kind '%s' not supported, must be 'terraform'", kind)
 			}
 
+			if componentPlan.AutoplanFiles != nil {
+				for _, file := range componentPlan.AutoplanFiles {
+					relPath, _ := filepath.Rel(path, file)
+					ext := filepath.Ext(file)
+					filename := filepath.Base(file)
+					key := strings.TrimSuffix(filename, ext)
+					key = util.ConvertToSnake(key)
+
+					switch ext {
+					case ".yaml", ".yml":
+						componentPlan.LocalsBlock[key] = fmt.Sprintf("yamldecode(file(%q))", relPath)
+					case ".json":
+						componentPlan.LocalsBlock[key] = fmt.Sprintf("jsondecode(file(%q))", relPath)
+					default:
+						componentPlan.LocalsBlock[key] = fmt.Sprintf("file(%q)", relPath)
+					}
+				}
+			}
+
 			err := applyTree(fs, componentBox, commonBox, path, componentPlan)
 			if err != nil {
 				return nil, errs.WrapUser(err, "unable to apply templates for component")

diff --git a/apply/apply_test.go b/apply/apply_test.go
@@ -388,7 +388,7 @@ version: 2
 	c, e := v2.ReadConfig(fs, []byte(yml), "fogg.yml")
 	r.NoError(e)
 
-	w, e := c.Validate()
+	w, e := c.Validate(fs)
 	r.NoError(e)
 	r.Len(w, 0)
 

diff --git a/apply/golden_file_test.go b/apply/golden_file_test.go
@@ -55,12 +55,42 @@ func TestIntegration(t *testing.T) {
 		t.Run(tt.fileName, func(t *testing.T) {
 			r := require.New(t)
 
-			testdataFs := afero.NewBasePathFs(afero.NewOsFs(), filepath.Join(util.ProjectRoot(), "testdata", tt.fileName))
 			configFile := "fogg.yml"
+
+			// delete all files except fogg.yml, foo.yaml, bar.json
+			var fixtureFiles = []string{
+				configFile,
+				"terraform/foo-fooFoo.yaml", // sample file for decode tests
+				"terraform/bar.json",
+			}
+			// delete all dirs except conf.d and foo_modules directories
+			var fixtureDirs = []string{
+				"fogg.d",
+				"foo_modules",
+			}
+
+			isFixtureFile := func(path string) bool {
+				for _, file := range fixtureFiles {
+					if path == file {
+						return true
+					}
+				}
+				return false
+			}
+
+			isInFixtureDir := func(path string) bool {
+				for _, dir := range fixtureDirs {
+					if strings.Contains(path, dir) {
+						return true
+					}
+				}
+				return false
+			}
+
+			testdataFs := afero.NewBasePathFs(afero.NewOsFs(), filepath.Join(util.ProjectRoot(), "testdata", tt.fileName))
 			if *updateGoldenFiles {
-				// delete all files except fogg.yml and conf.d, foo_modules directories
 				e := afero.Walk(testdataFs, ".", func(path string, info os.FileInfo, err error) error {
-					if !info.IsDir() && !(path == configFile) && !(strings.Contains(path, "fogg.d")) && !(strings.Contains(path, "foo_modules")) {
+					if !info.IsDir() && !isFixtureFile(path) && !isInFixtureDir(path) {
 						return testdataFs.Remove(path)
 					}
 					return nil
@@ -72,7 +102,7 @@ func TestIntegration(t *testing.T) {
 				fmt.Printf("conf %#v\n", conf)
 				fmt.Println("READ CONFIG")
 
-				w, e := conf.Validate()
+				w, e := conf.Validate(testdataFs)
 				r.NoError(e)
 				r.Len(w, 0)
 
@@ -82,48 +112,41 @@ func TestIntegration(t *testing.T) {
 				fs, _, e := util.TestFs()
 				r.NoError(e)
 
-				// copy fogg.yml into the tmp test dir (so that it doesn't show up as a diff)
-				configContents, e := afero.ReadFile(testdataFs, configFile)
-				r.NoError(e)
-				configMode, e := testdataFs.Stat(configFile)
-				r.NoError(e)
-				r.NoError(afero.WriteFile(fs, configFile, configContents, configMode.Mode()))
-				// if fogg.d exists, copy all partial configs too
-				confDir, e := testdataFs.Stat("fogg.d")
-				fs.Mkdir("fogg.d", 0700)
-				if e == nil && confDir.IsDir() {
-					afero.Walk(testdataFs, "fogg.d", func(path string, info os.FileInfo, err error) error {
-						if !info.IsDir() {
-							partialConfigContents, e := afero.ReadFile(testdataFs, path)
-							r.NoError(e)
-							r.NoError(afero.WriteFile(fs, path, partialConfigContents, info.Mode()))
-							return nil
-						}
-						return nil
-					})
+				// Copy all fixtures into the tmp test fs (so that they don't show up as a diff)
+				for _, file := range fixtureFiles {
+					if _, err := testdataFs.Stat(file); err == nil {
+						contents, e := afero.ReadFile(testdataFs, file)
+						r.NoError(e)
+						mode, e := testdataFs.Stat(file)
+						r.NoError(e)
+						directory := filepath.Dir(file)
+						e = fs.MkdirAll(directory, 0755)
+						r.NoError(e)
+						r.NoError(afero.WriteFile(fs, file, contents, mode.Mode()))
+					}
 				}
-				// if foo_modules exists, copy these too...
-				fooModulesDir, e := testdataFs.Stat("foo_modules")
-				fs.Mkdir("foo_modules", 0700)
-				if e == nil && fooModulesDir.IsDir() {
-					afero.Walk(testdataFs, "foo_modules", func(path string, info os.FileInfo, err error) error {
-						if !info.IsDir() {
-							moduleFileContents, e := afero.ReadFile(testdataFs, path)
-							r.NoError(e)
-							r.NoError(afero.WriteFile(fs, path, moduleFileContents, info.Mode()))
+				// Copy all fixtures directories into the tmp test fs
+				for _, dir := range fixtureDirs {
+					if dirInfo, err := testdataFs.Stat(dir); err == nil && dirInfo.IsDir() {
+						fs.Mkdir(dir, 0700)
+						afero.Walk(testdataFs, dir, func(path string, info os.FileInfo, err error) error {
+							if !info.IsDir() {
+								contents, e := afero.ReadFile(testdataFs, path)
+								r.NoError(e)
+								r.NoError(afero.WriteFile(fs, path, contents, info.Mode()))
+							} else {
+								fs.Mkdir(path, 0700)
+							}
 							return nil
-						} else {
-							fs.Mkdir(path, 0700)
-						}
-						return nil
-					})
+						})
+					}
 				}
 
 				conf, e := config.FindAndReadConfig(fs, configFile)
 				r.NoError(e)
 				fmt.Printf("conf %#v\n", conf)
 
-				w, e := conf.Validate()
+				w, e := conf.Validate(fs)
 				r.NoError(e)
 				r.Len(w, 0)
 

diff --git a/cmd/util.go b/cmd/util.go
@@ -32,7 +32,7 @@ func readAndValidateConfig(fs afero.Fs, configFile string) (*v2.Config, []string
 	logrus.Debug("CONFIG")
 	logrus.Debugf("%s\n=====", pretty.Sprint(conf))
 
-	warnings, e := conf.Validate()
+	warnings, e := conf.Validate(fs)
 	return conf, warnings, e
 }
 

diff --git a/config/v2/config.go b/config/v2/config.go
@@ -549,6 +549,10 @@ type DependsOn struct {
 	Components []string `yaml:"components"`
 	//RelativeGlobs to the component
 	RelativeGlobs []string `yaml:"relative_globs"`
+	//Absolute file paths,
+	//fogg validates their existence and
+	//generates locals block for their content
+	Files []string `yaml:"files"`
 }
 
 type CIProviderConfig struct {

diff --git a/config/v2/config_test.go b/config/v2/config_test.go
@@ -21,7 +21,7 @@ func TestReadConfig(t *testing.T) {
 	c, e := ReadConfig(fs, b, "fogg.yml")
 	r.NoError(e)
 
-	w, e := c.Validate()
+	w, e := c.Validate(fs)
 	r.Error(e)
 	r.Len(w, 0)
 }
@@ -39,7 +39,7 @@ func TestReadConfigYaml(t *testing.T) {
 	c, e := ReadConfig(fs, b2, "fogg.yml")
 	r.NoError(e)
 
-	w, e := c.Validate()
+	w, e := c.Validate(fs)
 	r.NoError(e)
 	r.Len(w, 0)
 }
@@ -59,7 +59,7 @@ func TestReadSnowflakeProviderYaml(t *testing.T) {
 	r.NoError(e)
 	r.NotNil(c)
 
-	w, e := c.Validate()
+	w, e := c.Validate(fs)
 	r.NoError(e)
 	r.Len(w, 0)
 
@@ -85,7 +85,7 @@ func TestReadOktaProvider(t *testing.T) {
 	r.NoError(e)
 	r.NotNil(c)
 
-	w, e := c.Validate()
+	w, e := c.Validate(fs)
 	r.NoError(e)
 	r.Len(w, 0)
 
@@ -110,7 +110,7 @@ func TestReadBlessProviderYaml(t *testing.T) {
 	r.NoError(e)
 	r.NotNil(c)
 
-	w, e := c.Validate()
+	w, e := c.Validate(fs)
 	r.NoError(e)
 	r.Len(w, 0)
 

diff --git a/config/v2/resolvers.go b/config/v2/resolvers.go
@@ -1382,3 +1382,10 @@ func DependsOnRelativeGlobsGetter(comm Common) []string {
 	}
 	return comm.DependsOn.RelativeGlobs
 }
+
+func DependsOnFilesGetter(comm Common) []string {
+	if comm.DependsOn == nil {
+		return nil
+	}
+	return comm.DependsOn.Files
+}
diff --git a/config/v2/validation.go b/config/v2/validation.go
@@ -2,23 +2,29 @@ package v2
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
 
 	"github.com/chanzuckerberg/fogg/errs"
+	"github.com/chanzuckerberg/fogg/util"
 	multierror "github.com/hashicorp/go-multierror"
 	goVersion "github.com/hashicorp/go-version"
 	"github.com/pkg/errors"
+	"github.com/spf13/afero"
 	validator "gopkg.in/go-playground/validator.v9"
 )
 
+const rootPath = "terraform"
+
 var validCICommands = map[string]struct{}{
 	"check": {},
 	"lint":  {},
 }
 
 // Validate validates the config
-func (c *Config) Validate() ([]string, error) {
+func (c *Config) Validate(fs afero.Fs) ([]string, error) {
 	if c == nil {
 		return nil, errs.NewInternal("config is nil")
 	}
@@ -55,6 +61,7 @@ func (c *Config) Validate() ([]string, error) {
 	errs = multierror.Append(errs, c.ValidateTravis())
 	errs = multierror.Append(errs, c.ValidateGithubActionsCI())
 	errs = multierror.Append(errs, c.validateTFE())
+	errs = multierror.Append(errs, c.ValidateFileDependencies(fs))
 
 	// refactor to make it easier to manage these
 	w, e := c.ValidateToolsTfLint()
@@ -502,3 +509,27 @@ func (c *Config) validateModules() error {
 func nonEmptyString(s *string) bool {
 	return s != nil && len(*s) > 0
 }
+
+func (c *Config) ValidateFileDependencies(fs afero.Fs) error {
+	var errs *multierror.Error
+	c.WalkComponents(func(component string, comms ...Common) {
+		files := ResolveOptionalStringSlice(DependsOnFilesGetter, comms...)
+		keys := make(map[string]bool)
+		for _, file := range files {
+			ext := filepath.Ext(file)
+			filename := filepath.Base(file)
+			key := strings.TrimSuffix(filename, ext)
+			key = util.ConvertToSnake(key)
+			if keys[key] {
+				errs = multierror.Append(errs, fmt.Errorf("component: %s - local.%s, file dependency naming collision. %v\n", component, key, files))
+			} else {
+				keys[key] = true
+			}
+			if _, err := fs.Stat(file); os.IsNotExist(err) {
+				errs = multierror.Append(errs, fmt.Errorf("component: %s - File does not exist: %s\n", component, file))
+			}
+		}
+	})
+
+	return errs.ErrorOrNil()
+}