fix: issue with git tracked file not ignoring ge_ignore patterns

.ge_ignore

@@ -15,7 +15,7 @@
 **/*.tar
 **/*.7z
 **/*.pyc
-**/*.git*
+**/.git/*
 **/*.svn*
 **/*vendor*
 **/bin/*

go.mod

@@ -1,6 +1,6 @@
 module github.com/americanexpress/earlybird/v4
 
-go 1.20
+go 1.23
 
 require (
 	code.sajari.com/docconv v1.3.8
@@ -11,7 +11,7 @@ require (
 	github.com/gorilla/mux v1.8.1
 	github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c
 	golang.org/x/net v0.28.0
-	golang.org/x/text v0.17.0
+	golang.org/x/text v0.21.0
 	gopkg.in/src-d/go-git.v4 v4.13.1
 )
 

go.sum

@@ -47,6 +47,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
@@ -148,8 +149,8 @@ golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
 golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
-golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=

pkg/file/fileUtil.go

@@ -43,7 +43,7 @@ var (
 	ignorePatterns []string
 )
 
-//MultipartToScanFiles converts the multipart file upload into Earlybird files
+// MultipartToScanFiles converts the multipart file upload into Earlybird files
 func MultipartToScanFiles(files []*multipart.FileHeader, cfg cfgreader.EarlybirdConfig) (fileList []scan.File, err error) {
 	ignorePatterns = getIgnorePatterns(cfg.SearchDir, cfg.IgnoreFile, cfg.VerboseEnabled)
 
@@ -119,7 +119,7 @@ func MultipartToScanFiles(files []*multipart.FileHeader, cfg cfgreader.Earlybird
 	return
 }
 
-//GetGitFiles Builds the list of staged or tracked files
+// GetGitFiles Builds the list of staged or tracked files
 func GetGitFiles(fileType string, cfg *cfgreader.EarlybirdConfig) (fileContext Context, err error) {
 	ignorePatterns = getIgnorePatterns(cfg.SearchDir, cfg.IgnoreFile, cfg.VerboseEnabled)
 
@@ -128,6 +128,7 @@ func GetGitFiles(fileType string, cfg *cfgreader.EarlybirdConfig) (fileContext C
 		compressList []scan.File
 		convertList  []scan.File
 		fileList     []scan.File
+		skipList     []string
 	)
 
 	if fileType == utils.Tracked {
@@ -141,7 +142,7 @@ func GetGitFiles(fileType string, cfg *cfgreader.EarlybirdConfig) (fileContext C
 		return fileContext, err
 	}
 
-	fileList = parseGitFiles(output, cfg.VerboseEnabled, cfg.MaxFileSize)
+	fileList, skipList = parseGitFiles(output, cfg.VerboseEnabled, cfg.MaxFileSize, cfg.SearchDir)
 	compressList, fileList = separateCompressedAndUncompressed(fileList)
 	compressList, fileContext.CompressPaths, err = GetCompressedFiles(compressList, cfg.SearchDir) //Get the files within our compressed list
 	if err != nil {
@@ -151,10 +152,11 @@ func GetGitFiles(fileType string, cfg *cfgreader.EarlybirdConfig) (fileContext C
 	convertList, fileContext.ConvertPaths = GetConvertedFiles(fileContext.Files) //Get the files that need to be converted and convert them to plaintext
 	fileContext.Files = append(fileContext.Files, convertList...)
 	fileContext.IgnorePatterns = ignorePatterns
+	fileContext.SkippedFiles = skipList
 	return fileContext, nil
 }
 
-func parseGitFiles(out []byte, verbose bool, maxFileSize int64) (fileList []scan.File) {
+func parseGitFiles(out []byte, verbose bool, maxFileSize int64, searchDir string) (fileList []scan.File, skipList []string) {
 	var curFile scan.File
 	// Convert byteArray to string
 	gitFiles := string(out)
@@ -165,25 +167,42 @@ func parseGitFiles(out []byte, verbose bool, maxFileSize int64) (fileList []scan
 		scanner := bufio.NewScanner(strings.NewReader(gitFiles))
 		for scanner.Scan() {
 			curFile.Path = scanner.Text()
+			// add slash to make the path absolute this is used to match the ignore patterns
+			absolutePath := "/" + scanner.Text()
+			// if the path is absolute, set the absolute path to the current path
+			if strings.HasPrefix(scanner.Text(), "/") {
+				absolutePath = scanner.Text()
+			}
+
 			curFile.Name = filepath.Base(scanner.Text())
-			if fileExists := Exists(curFile.Path); fileExists {
-				pathIsDirectory, dirErr := isDirectory(curFile.Path)
-				if dirErr != nil {
-					log.Println(dirErr)
-				}
-				if !pathIsDirectory && getFileSizeOK(curFile.Path, maxFileSize) {
-					if verbose {
-						log.Println("Reading file ", curFile.Path)
+			//Skip file with extensions Earlybird ignores
+			if !isIgnoredFile(absolutePath, searchDir) {
+				if fileExists := Exists(curFile.Path); fileExists {
+					pathIsDirectory, dirErr := isDirectory(curFile.Path)
+					if dirErr != nil {
+						log.Println(dirErr)
 					}
-					fileList = append(fileList, curFile)
+
+					if !pathIsDirectory && getFileSizeOK(curFile.Path, maxFileSize) {
+						if verbose {
+							log.Println("Reading file ", curFile.Path)
+						}
+						fileList = append(fileList, curFile)
+					}
+				}
+			} else {
+				skipList = append(skipList, curFile.Path)
+				if verbose {
+					log.Println("Ignoring", curFile.Path, ". File blacklisted.")
 				}
 			}
 		}
 	}
-	return fileList
+
+	return fileList, skipList
 }
 
-//GetFiles Build the list of files
+// GetFiles Build the list of files
 func GetFiles(searchDir, ignoreFile string, verbose bool, maxFileSize int64) (fileContext Context, err error) {
 	ignorePatterns = getIgnorePatterns(searchDir, ignoreFile, verbose)
 	fileList := make([]scan.File, 0)
@@ -238,7 +257,7 @@ func GetFiles(searchDir, ignoreFile string, verbose bool, maxFileSize int64) (fi
 	return fileContext, nil
 }
 
-//GetFileFromStream Builds a file as a collection of lines from the input stream.
+// GetFileFromStream Builds a file as a collection of lines from the input stream.
 // This will be fed to the scan modules.
 func GetFileFromStream(cfg *cfgreader.EarlybirdConfig) []scan.File {
 	// Read Stdin
@@ -285,7 +304,7 @@ func GetFileFromStream(cfg *cfgreader.EarlybirdConfig) []scan.File {
 	return fileList
 }
 
-//GetFileSize returns the file size of target file
+// GetFileSize returns the file size of target file
 func GetFileSize(path string) (size int64, err error) {
 	stat, err := os.Stat(path)
 	if err != nil {
@@ -385,7 +404,7 @@ func isDirectory(path string) (bool, error) {
 	return fileInfo.IsDir(), nil
 }
 
-//GetWD Gets the current working directory
+// GetWD Gets the current working directory
 func GetWD() (string, error) {
 	dir, err := os.Getwd()
 	if err != nil {
@@ -394,7 +413,7 @@ func GetWD() (string, error) {
 	return dir, err
 }
 
-//IsEmpty Check to see if a directory is empty
+// IsEmpty Check to see if a directory is empty
 func IsEmpty(path string) (bool, error) {
 	fileHandle, err := os.Open(path)
 	if err != nil {
@@ -406,7 +425,7 @@ func IsEmpty(path string) (bool, error) {
 	return err == io.EOF, err
 }
 
-//Exists Check to see if a path exists
+// Exists Check to see if a path exists
 func Exists(path string) bool {
 	_, err := os.Stat(path)
 	return !os.IsNotExist(err)
@@ -423,7 +442,7 @@ func separateCompressedAndUncompressed(files []scan.File) (compressed, uncompres
 	return compressed, uncompressed
 }
 
-//GetCompressedFiles provides all the files contained within compressed files
+// GetCompressedFiles provides all the files contained within compressed files
 func GetCompressedFiles(files []scan.File, rootPath string) (newfiles []scan.File, compresspaths []string, err error) {
 	//check if file list contains compressed files, if so, scan their contents
 	for _, file := range files {
@@ -449,7 +468,7 @@ func GetCompressedFiles(files []scan.File, rootPath string) (newfiles []scan.Fil
 	return newfiles, compresspaths, nil
 }
 
-//Uncompress decompresses zip files safely
+// Uncompress decompresses zip files safely
 func Uncompress(src string, dest string) (filenames []string, err error) {
 	r, err := zip.OpenReader(src)
 	if err != nil {
@@ -507,7 +526,7 @@ func Uncompress(src string, dest string) (filenames []string, err error) {
 	return filenames, nil
 }
 
-//GetConvertedFiles converts files into plaintext
+// GetConvertedFiles converts files into plaintext
 func GetConvertedFiles(files []scan.File) (convertedFiles []scan.File, convertedPaths []string) {
 	var toBeConverted []scan.File
 	for _, f := range files {

pkg/file/fileUtil_test.go

@@ -18,6 +18,7 @@ package file
 
 import (
 	"github.com/americanexpress/earlybird/v4/pkg/scan"
+	"os/exec"
 	"reflect"
 
 	"os"
@@ -404,3 +405,18 @@ func TestGetConvertedFiles(t *testing.T) {
 		})
 	}
 }
+
+func Test_parseGitFiles(t *testing.T) {
+	output, err := exec.Command("git", "-C", projectRoot, "ls-tree", "--full-tree", "-r", "--name-only", "HEAD").Output()
+	if err != nil {
+		t.Errorf("parseGitFiles() error = %v", err)
+	}
+	if len(output) == 0 {
+		t.Errorf("parseGitFiles() output = %v, want multiple files", output)
+	}
+	_, skipFiles := parseGitFiles(output, true, int64(1000000), projectRoot)
+
+	if len(skipFiles) == 0 {
+		t.Errorf("parseGitFiles() skipFiles = %v, want multiple files", skipFiles)
+	}
+}