feat: add ignore file functionality (exclude list) (#6)

* feat: add ignore file functionality (exclude list)

.github/workflows/go.yaml

@@ -1,3 +1,4 @@
+# UPDATE_HERE
 name: build-git-get
 on: [push]
 jobs:
@@ -8,10 +9,10 @@ jobs:
     if: github.ref != 'refs/heads/master'
     steps:
 
-    - name: Set up Go 1.17.5
+    - name: Set up Go 1.17.6
       uses: actions/setup-go@v1
       with:
-        go-version: 1.17.5
+        go-version: 1.17.6
       id: go
 
     - name: Check out code into the Go module directory
@@ -27,10 +28,10 @@ jobs:
     if: github.ref == 'refs/heads/master'
     steps:
 
-    - name: Set up Go 1.17.5
+    - name: Set up Go 1.17.6
       uses: actions/setup-go@v1
       with:
-        go-version: 1.17.5
+        go-version: 1.17.6
       id: go
 
     - name: Check out code into the Go module directory

.gitignore

@@ -1,6 +1,7 @@
 # Custom
 env
 Gitfile
+Gitfile.ignore
 bin/
 chglog.tmp
 gitlab_api_postman.json

.goreleaser.yaml

@@ -0,0 +1,32 @@
+# This is an example .goreleaser.yml file with some sensible defaults.
+# Make sure to check the documentation at https://goreleaser.com
+before:
+  hooks:
+    # You may remove this if you don't use go modules.
+    - go mod tidy
+    # you may remove this if you don't need go generate
+    - go generate ./...
+builds:
+  - env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+archives:
+  - replacements:
+      darwin: Darwin
+      linux: Linux
+      windows: Windows
+      386: i386
+      amd64: x86_64
+checksum:
+  name_template: 'checksums.txt'
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'

.tool-versions

@@ -1,2 +1,3 @@
+# UPDATE_HERE
 # https://golang.org/dl/
-golang 1.17.5
+golang 1.17.6

Makefile

@@ -2,7 +2,7 @@ SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 GO := GOPROXY=https://proxy.golang.org go
 
-VERSION:="0.0.8"
+VERSION:="0.0.9"
 EXE:="git-get"
 BUILD:=`git rev-parse --short HEAD`
 TIME:=`date`
@@ -101,6 +101,11 @@ tidy: ## Fetches dependencies
 echo: ## Prints image name and version of the tool
 	@echo "git-get ${VERSION} ${BUILD}"
 
+.PHONY: update-here
+update-here: ## Helper target to start editing all occurances with UPDATE_HERE.
+	@echo "Update following files for release:"
+	@grep --color -nHR UPDATE_HERE .
+
 .PHONY: release
 release: ## Release application
 	@{ \

README.md

@@ -14,8 +14,12 @@ $ brew install git-get
 
 ## Configuration file `Gitfile`
 
-`git-get` (which can be executed as `git get` if is in the `$PATH`) fetches git repositories using
+`git-get` (which can be executed as `git get`) fetches git repositories using
 configuration file (by default `Gitfile` in current directory of invocation).
+`Gitfile.ignore` file can control which repositories to skip from the operations
+and it has same format, where in `Gitfile.ignore` the only significant field is
+`url`, which value is compared with the target one. If `Gitfile.ignore` is missing
+this functionality is ignored.
 
 `Gitfile` format is:
 
@@ -42,6 +46,15 @@ configuration file (by default `Gitfile` in current directory of invocation).
 * `symlinks` is an optional list of paths to create symlinks to this clones repository. If such a file
   already exists (symlink, directory or regular file) - nothing will be done
 
+## Other `git-get` operations
+
+`git-get` can also generate configuration file from repositories in git provider or mirror
+repositories specified by `Gitfile` to a chosen git provider. For the clone/fetch operations
+on git repositories ssh keys must be used. For creating target repositories in git provider
+(during mirror operation) or for fetching the list of available repositories in git provider -
+user API keys are used. `git-get` allows shallow clone of the repositories, which is suitable
+for use in CI/CD.
+
 # Command line options
 
 ## Fetching/Refreshing repositories specified by Gitfile
@@ -79,6 +92,7 @@ Flags:
   -f, --config-file string           Configuration file (default "~/Gitfile")
   -b, --default-main-branch string   Default main branch (default "master")
   -h, --help                         help for git-get
+  -i, --ignore-file string           Ignore file (default "~/Gitfile.ignore")
   -l, --log-level string             Logging level [debug|info|warn|error|fatal|panic] (default "info")
   -s, --shallow                      Shallow clone, can be used in CI to fetch dependencies by ref
   -t, --stay-on-ref                  After refreshing repository from remote stay on ref branch
@@ -124,6 +138,7 @@ Flags:
       --gitlab-owned                                Gitlab: only traverse groups and repositories owned by user
       --gitlab-project-visibility string            Gitlab: project visibility [public|internal|private]
   -h, --help                                        help for config-gen
+  -i, --ignore-file string                          Ignore file (default "~/Gitfile.ignore")
   -l, --log-level string                            Logging level [debug|info|warn|error|fatal|panic] (default "info")
   -t, --target-clone-path string                    Target clone path used to set 'path' for each repository in Gitfile
 ```
@@ -164,6 +179,7 @@ Flags:
   -f, --config-file string                     Configuration file (default "~/Gitfile")
   -d, --dry-run                                Dry-run - do not push to remote mirror repositories
   -h, --help                                   help for mirror
+  -i, --ignore-file string                     Ignore file (default "~/Gitfile.ignore")
   -l, --log-level string                       Logging level [debug|info|warn|error|fatal|panic] (default "info")
   -p, --mirror-provider string                 Git mirror provider name [gitlab|github|bitbucket] (default "gitlab")
   -u, --mirror-url string                      Private Mirror URL prefix to push repositories to (example: [email protected]:acmeorg)

TODO.md

@@ -1,7 +1,8 @@
-* WIP: Add construction of the Gitfile subcommand `generate-config`
+* use goreleaser to release new versions
 * add `status` subcommand to see which local repositories have unstaged/uncommited changes or are seating on a feature branch
-* improve Auth mechanism in all provider specific code - maybe use singleton
+* maybe: add ability to exclude/include repositories by regex in config-gen
 * improve test coverage
+* improve Auth mechanism in all provider specific code - maybe use singleton
 * aggregate log entries for each item - to print like these would be run sequentially
 * potentially add more providers gitea, AWS CodeCommit, Azure DevOps Git, Google Cloud Source Repositories, Launchpad, 
 * Evaluate: https://github.com/fluxcd/go-git-providers to simplify code

bitbucket/bitbucket.go

@@ -1,5 +1,5 @@
 /*
-Copyright © 2021 Eriks Zelenka <[email protected]>
+Copyright © 2021-2022 Eriks Zelenka <[email protected]>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -51,11 +51,13 @@ func bitbucketAuth(repoSha string) *bitbucket.Client {
 	return git
 }
 
+// GenerateProjectKey - convert project name to project key
 func GenerateProjectKey(projectName string) string {
 	re := regexp.MustCompile(`[^a-zA-Z0-9_]`)
 	return strings.ToUpper(re.ReplaceAllString(projectName, ""))
 }
 
+// RepositoryExists - checks if bitbucket repository exists
 func RepositoryExists(repoSha string, owner string, repository string) bool {
 	git := bitbucketAuth(repoSha)
 
@@ -68,12 +70,13 @@ func RepositoryExists(repoSha string, owner string, repository string) bool {
 	if err != nil {
 		log.Debugf("%s: Error fetching repository '%s/%s': %+v", repoSha, owner, repository, err)
 		return false
-	} else {
-		log.Debugf("%s: Fetched repository '%+v'", repoSha, repo)
-		return true
 	}
+
+	log.Debugf("%s: Fetched repository '%+v'", repoSha, repo)
+	return true
 }
 
+// ProjectExists - checks if bitbucket project exists
 func ProjectExists(git *bitbucket.Client, repoSha string, workspace string, project string) bool {
 	opt := &bitbucket.ProjectOptions{
 		Owner: workspace,
@@ -86,12 +89,13 @@ func ProjectExists(git *bitbucket.Client, repoSha string, workspace string, proj
 	if err != nil {
 		log.Debugf("%s: Error fetching project '%s' in workspace '%s': %+v\n", repoSha, project, workspace, err)
 		return false
-	} else {
-		log.Debugf("%s: Fetched project '%+v'\n", repoSha, prj)
-		return true
 	}
+
+	log.Debugf("%s: Fetched project '%+v'\n", repoSha, prj)
+	return true
 }
 
+// CreateRepository - create bitbucket repository
 func CreateRepository(repoSha, repository, mirrorVisibilityMode, sourceURL, projectName string) *bitbucket.Repository {
 	git := bitbucketAuth(repoSha)
 

cmd/config_gen.go

@@ -1,5 +1,5 @@
 /*
-Copyright © 2021 Eriks Zelenka <[email protected]>
+Copyright © 2021-2022 Eriks Zelenka <[email protected]>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -23,6 +23,7 @@ THE SOFTWARE.
 package cmd
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 
@@ -56,7 +57,7 @@ git-get config-gen -f Gitfile -p "github" -u "[email protected]:AcmeOrg" -t AcmeOrg
 		initLogging(logLevel)
 		log.Debug("Generate Gitfile configuration file")
 		gitget.GenerateGitfileConfig(
-			cfgFile, gitCloudProviderRootURL, gitCloudProvider, targetClonePath, configGenParams)
+			cfgFile, ignoreFile, gitCloudProviderRootURL, gitCloudProvider, targetClonePath, configGenParams)
 	},
 }
 
@@ -70,11 +71,17 @@ func init() {
 	}
 
 	defaultValue := filepath.Join(wdir, "Gitfile")
+	defaultIgnoreValue := fmt.Sprintf("%s.ignore", defaultValue)
 	configGenCmd.Flags().StringVarP(
 		&cfgFile, "config-file",
 		"f",
 		defaultValue,
 		"Configuration file")
+	configGenCmd.Flags().StringVarP(
+		&ignoreFile, "ignore-file",
+		"i",
+		defaultIgnoreValue,
+		"Ignore file")
 	configGenCmd.Flags().StringVarP(
 		&logLevel,
 		"log-level",

cmd/mirror.go

@@ -1,5 +1,5 @@
 /*
-Copyright © 2021 Eriks Zelenka <[email protected]>
+Copyright © 2021-2022 Eriks Zelenka <[email protected]>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -24,6 +24,7 @@ THE SOFTWARE.
 package cmd
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 
@@ -65,7 +66,16 @@ git-get mirror -c 2 -f Gitfile -l debug -u "[email protected]:acmeorg" -p "bitbu
 		initLogging(logLevel)
 		log.Debugf("%t - push to mirror", pushMirror)
 		pushMirror = !dryRun
-		gitget.MirrorRepositories(cfgFile, concurrencyLevel, pushMirror, gitCloudProviderRootURL, gitCloudProvider, mirrorVisibilityMode, mirrorBitbucketProjectName)
+		gitget.MirrorRepositories(
+			cfgFile,
+			ignoreFile,
+			concurrencyLevel,
+			pushMirror,
+			gitCloudProviderRootURL,
+			gitCloudProvider,
+			mirrorVisibilityMode,
+			mirrorBitbucketProjectName,
+		)
 	},
 }
 
@@ -79,12 +89,57 @@ func init() {
 	}
 
 	defaultValue := filepath.Join(wdir, "Gitfile")
-	mirrorCmd.Flags().StringVarP(&cfgFile, "config-file", "f", defaultValue, "Configuration file")
-	mirrorCmd.Flags().StringVarP(&logLevel, "log-level", "l", "info", "Logging level [debug|info|warn|error|fatal|panic]")
-	mirrorCmd.Flags().IntVarP(&concurrencyLevel, "concurrency-level", "c", 1, "Git get concurrency level")
-	mirrorCmd.Flags().BoolVarP(&dryRun, "dry-run", "d", false, "Dry-run - do not push to remote mirror repositories")
-	mirrorCmd.Flags().StringVarP(&gitCloudProviderRootURL, "mirror-url", "u", "", "Private Mirror URL prefix to push repositories to (example: [email protected]:acmeorg)")
-	mirrorCmd.Flags().StringVarP(&gitCloudProvider, "mirror-provider", "p", "gitlab", "Git mirror provider name [gitlab|github|bitbucket]")
-	mirrorCmd.Flags().StringVarP(&mirrorVisibilityMode, "mirror-visibility-mode", "v", "private", "Mirror visibility mode [private|internal|public]")
-	mirrorCmd.Flags().StringVarP(&mirrorBitbucketProjectName, "bitbucket-mirror-project-name", "b", "", "Bitbucket mirror project name (only effective for Bitbucket and is optional)")
+	defaultIgnoreValue := fmt.Sprintf("%s.ignore", defaultValue)
+	mirrorCmd.Flags().StringVarP(
+		&cfgFile, "config-file",
+		"f",
+		defaultValue,
+		"Configuration file")
+	mirrorCmd.Flags().StringVarP(
+		&ignoreFile, "ignore-file",
+		"i",
+		defaultIgnoreValue,
+		"Ignore file")
+	mirrorCmd.Flags().StringVarP(
+		&logLevel, "log-level",
+		"l",
+		"info",
+		"Logging level [debug|info|warn|error|fatal|panic]",
+	)
+	mirrorCmd.Flags().IntVarP(
+		&concurrencyLevel, "concurrency-level",
+		"c",
+		1,
+		"Git get concurrency level",
+	)
+	mirrorCmd.Flags().BoolVarP(
+		&dryRun, "dry-run",
+		"d",
+		false,
+		"Dry-run - do not push to remote mirror repositories",
+	)
+	mirrorCmd.Flags().StringVarP(
+		&gitCloudProviderRootURL, "mirror-url",
+		"u",
+		"",
+		"Private Mirror URL prefix to push repositories to (example: [email protected]:acmeorg)",
+	)
+	mirrorCmd.Flags().StringVarP(
+		&gitCloudProvider, "mirror-provider",
+		"p",
+		"gitlab",
+		"Git mirror provider name [gitlab|github|bitbucket]",
+	)
+	mirrorCmd.Flags().StringVarP(
+		&mirrorVisibilityMode, "mirror-visibility-mode",
+		"v",
+		"private",
+		"Mirror visibility mode [private|internal|public]",
+	)
+	mirrorCmd.Flags().StringVarP(
+		&mirrorBitbucketProjectName, "bitbucket-mirror-project-name",
+		"b",
+		"",
+		"Bitbucket mirror project name (only effective for Bitbucket and is optional)",
+	)
 }