feat: Multi config and multi ignore file functionality (#10)

* feat: Ability to specify multiple config and ignore files where applicable

* Fix typo

Makefile

@@ -2,7 +2,7 @@ SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 GO := GOPROXY=https://proxy.golang.org go
 
-VERSION:="0.0.11"
+VERSION:="0.0.12"
 EXE:="git-get"
 BUILD:=`git rev-parse --short HEAD`
 TIME:=`date`

README.md

@@ -61,6 +61,7 @@ for use in CI/CD.
 
 ```bash
 % git-get --help
+
 'git-get' - all your project repositories
 
 git-get clone/refresh all your local project repositories in
@@ -78,6 +79,10 @@ Usage:
 Examples:
 
 git get -c 12 -f Gitfile
+git get -c 12 -f Gitfile.1 -f Gitfile.2 -f Gitfile.3,Gitfile.4
+git get -c 12 -f Gitfile -i Gitfile.ignore.1 -i Gitfile.ignore.2
+git get -c 8 -f Gitfile --status -i Gitfile.ignore -l panic \
+  | awk '$0 ~ /REPOSITORY/ || $3 ~ /true/ { print $0 }'
 
 Available Commands:
   completion  Generate the autocompletion script for the specified shell
@@ -88,10 +93,10 @@ Available Commands:
 
 Flags:
   -c, --concurrency-level int        Git get concurrency level (default 1)
-  -f, --config-file string           Configuration file (default "~/Gitfile")
+  -f, --config-file strings          Configuration file or comma separated list of files (default [~/Gitfile])
   -b, --default-main-branch string   Default main branch (default "master")
   -h, --help                         help for git-get
-  -i, --ignore-file string           Ignore file (default "~/Gitfile.ignore")
+  -i, --ignore-file strings          Ignore file or comma separated list of files (default [~/Gitfile.ignore])
   -l, --log-level string             Logging level [debug|info|warn|error|fatal|panic] (default "info")
   -s, --shallow                      Shallow clone, can be used in CI to fetch dependencies by ref
       --status                       Print extra status information after clone is performed
@@ -138,7 +143,7 @@ Flags:
       --gitlab-owned                                Gitlab: only traverse groups and repositories owned by user
       --gitlab-project-visibility string            Gitlab: project visibility [public|internal|private]
   -h, --help                                        help for config-gen
-  -i, --ignore-file string                          Ignore file (default "~/Gitfile.ignore")
+  -i, --ignore-file strings                         Ignore file or comma separated list of files (default [~/Gitfile.ignore])
   -l, --log-level string                            Logging level [debug|info|warn|error|fatal|panic] (default "info")
   -t, --target-clone-path string                    Target clone path used to set 'path' for each repository in Gitfile
 ```
@@ -176,10 +181,10 @@ git-get mirror -c 2 -f Gitfile -l debug -u "[email protected]:acmeorg" -p "bitbu
 Flags:
   -b, --bitbucket-mirror-project-name string   Bitbucket mirror project name (only effective for Bitbucket and is optional)
   -c, --concurrency-level int                  Git get concurrency level (default 1)
-  -f, --config-file string                     Configuration file (default "~/Gitfile")
+  -f, --config-file strings                    Configuration file or comma separated list of files (default [~/Gitfile])
   -d, --dry-run                                Dry-run - do not push to remote mirror repositories
   -h, --help                                   help for mirror
-  -i, --ignore-file string                     Ignore file (default "~/Gitfile.ignore")
+  -i, --ignore-file strings                    Ignore file or comma separated list of files (default [~/Gitfile.ignore])
   -l, --log-level string                       Logging level [debug|info|warn|error|fatal|panic] (default "info")
   -p, --mirror-provider string                 Git mirror provider name [gitlab|github|bitbucket] (default "gitlab")
   -u, --mirror-url string                      Private Mirror URL prefix to push repositories to (example: [email protected]:acmeorg)

TODO.md

@@ -1,8 +1,10 @@
-* use goreleaser to release new versions
-* add `status` subcommand to see which local repositories have unstaged/uncommited changes or are seating on a feature branch
+* maybe: add ability to use multiple Gitfiles and multiple Gitfile.ignore
+  * https://github.com/spf13/cobra/issues/661
 * maybe: add ability to exclude/include repositories by regex in config-gen
+* add support for gitea
+* use goreleaser to release new versions
 * improve test coverage
 * improve Auth mechanism in all provider specific code - maybe use singleton
-* aggregate log entries for each item - to print like these would be run sequentially
-* potentially add more providers gitea, AWS CodeCommit, Azure DevOps Git, Google Cloud Source Repositories, Launchpad, 
+* maybe: aggregate log entries for each item - to print like these would be run sequentially
+* potentially add more providers: AWS CodeCommit, Azure DevOps Git, Google Cloud Source Repositories, Launchpad, other?
 * Evaluate: https://github.com/fluxcd/go-git-providers to simplify code

cmd/config_gen.go

@@ -57,7 +57,7 @@ git-get config-gen -f Gitfile -p "github" -u "[email protected]:AcmeOrg" -t AcmeOrg
 		initLogging(logLevel)
 		log.Debug("Generate Gitfile configuration file")
 		gitget.GenerateGitfileConfig(
-			cfgFile, ignoreFile, gitCloudProviderRootURL, gitCloudProvider, targetClonePath, configGenParams)
+			cfgFile, ignoreFiles, gitCloudProviderRootURL, gitCloudProvider, targetClonePath, configGenParams)
 	},
 }
 
@@ -77,11 +77,11 @@ func init() {
 		"f",
 		defaultValue,
 		"Configuration file")
-	configGenCmd.Flags().StringVarP(
-		&ignoreFile, "ignore-file",
+	configGenCmd.Flags().StringSliceVarP(
+		&ignoreFiles, "ignore-file",
 		"i",
-		defaultIgnoreValue,
-		"Ignore file")
+		[]string{defaultIgnoreValue},
+		"Ignore file or comma separated list of files")
 	configGenCmd.Flags().StringVarP(
 		&logLevel,
 		"log-level",

cmd/mirror.go

@@ -59,16 +59,18 @@ git get mirror -f Gitfile -u "[email protected]:acmeorg" -p "github"
 git-get mirror -c 2 -f Gitfile -l debug -u "[email protected]:acmeorg/mirrors"
 git-get mirror -c 2 -f Gitfile -l debug -u "[email protected]:acmeorg" -p "bitbucket" -b "mirrors"`,
 	Run: func(cmd *cobra.Command, args []string) {
-		if _, err := os.Stat(cfgFile); os.IsNotExist(err) {
-			log.Fatalln(err)
-			os.Exit(1)
+		for _, cfgFile := range cfgFiles {
+			if _, err := os.Stat(cfgFile); os.IsNotExist(err) {
+				log.Fatalln(err)
+				os.Exit(1)
+			}
 		}
 		initLogging(logLevel)
 		log.Debugf("%t - push to mirror", pushMirror)
 		pushMirror = !dryRun
 		gitget.MirrorRepositories(
-			cfgFile,
-			ignoreFile,
+			cfgFiles,
+			ignoreFiles,
 			concurrencyLevel,
 			pushMirror,
 			gitCloudProviderRootURL,
@@ -90,16 +92,16 @@ func init() {
 
 	defaultValue := filepath.Join(wdir, "Gitfile")
 	defaultIgnoreValue := fmt.Sprintf("%s.ignore", defaultValue)
-	mirrorCmd.Flags().StringVarP(
-		&cfgFile, "config-file",
+	mirrorCmd.Flags().StringSliceVarP(
+		&cfgFiles, "config-file",
 		"f",
-		defaultValue,
-		"Configuration file")
-	mirrorCmd.Flags().StringVarP(
-		&ignoreFile, "ignore-file",
+		[]string{defaultValue},
+		"Configuration file or comma separated list of files")
+	mirrorCmd.Flags().StringSliceVarP(
+		&ignoreFiles, "ignore-file",
 		"i",
-		defaultIgnoreValue,
-		"Ignore file")
+		[]string{defaultIgnoreValue},
+		"Ignore file or comma separated list of files")
 	mirrorCmd.Flags().StringVarP(
 		&logLevel, "log-level",
 		"l",

cmd/root.go

@@ -37,7 +37,8 @@ import (
 var configGenParams gitget.ConfigGenParamsStruct
 
 var cfgFile string
-var ignoreFile string
+var cfgFiles []string
+var ignoreFiles []string
 var logLevel string
 var stayOnRef bool
 var shallow bool
@@ -67,7 +68,8 @@ var levels = map[string]log.Level{
 var rootCmd = &cobra.Command{
 	Use:   "git-get",
 	Short: "'git-get' - all your project repositories",
-	Long: `'git-get' - all your project repositories
+	Long: `
+'git-get' - all your project repositories
 
 git-get clone/refresh all your local project repositories in
 one go.
@@ -77,16 +79,22 @@ structure of the project. git-get allows to create symlinks
 to cloned repositories, clone one repository multiple time
 having different directory name.`,
 	Example: `
-git get -c 12 -f Gitfile`,
+git get -c 12 -f Gitfile
+git get -c 12 -f Gitfile.1 -f Gitfile.2 -f Gitfile.3,Gitfile.4
+git get -c 12 -f Gitfile -i Gitfile.ignore.1 -i Gitfile.ignore.2
+git get -c 8 -f Gitfile --status -i Gitfile.ignore -l panic \
+  | awk '$0 ~ /REPOSITORY/ || $3 ~ /true/ { print $0 }'`,
 	Run: func(cmd *cobra.Command, args []string) {
-		if _, err := os.Stat(cfgFile); os.IsNotExist(err) {
-			log.Fatalln(err)
-			os.Exit(1)
+		for _, cfgFile := range cfgFiles {
+			if _, err := os.Stat(cfgFile); os.IsNotExist(err) {
+				log.Fatalln(err)
+				os.Exit(1)
+			}
 		}
 		initLogging(logLevel)
 		gitget.GetRepositories(
-			cfgFile,
-			ignoreFile,
+			cfgFiles,
+			ignoreFiles,
 			concurrencyLevel,
 			stayOnRef,
 			shallow,
@@ -118,16 +126,16 @@ func init() {
 
 	defaultValue := filepath.Join(wdir, "Gitfile")
 	defaultIgnoreValue := fmt.Sprintf("%s.ignore", defaultValue)
-	rootCmd.Flags().StringVarP(
-		&cfgFile, "config-file",
+	rootCmd.Flags().StringSliceVarP(
+		&cfgFiles, "config-file",
 		"f",
-		defaultValue,
-		"Configuration file")
-	rootCmd.Flags().StringVarP(
-		&ignoreFile, "ignore-file",
+		[]string{defaultValue},
+		"Configuration file or comma separated list of files")
+	rootCmd.Flags().StringSliceVarP(
+		&ignoreFiles, "ignore-file",
 		"i",
-		defaultIgnoreValue,
-		"Ignore file")
+		[]string{defaultIgnoreValue},
+		"Ignore file or comma separated list of files")
 	rootCmd.Flags().StringVarP(
 		&logLevel, "log-level",
 		"l",