Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

no longer report full or partial raw fragment #2

Open
wants to merge 13 commits into
base: master
Choose a base branch
from
Open

no longer report full or partial raw fragment #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
3a46376
no longer report full or partial raw fragment
Kemosabert Feb 7, 2024
e1ac1b7
fix current tests
Kemosabert Feb 7, 2024
5fc38d3
add fullline repo with vulns
Kemosabert Feb 7, 2024
e7f7207
add test case for multilines
Kemosabert Feb 7, 2024
0397509
move tests to separate file
Kemosabert Feb 8, 2024
b89855b
use git repo as that is more relevant
Kemosabert Feb 8, 2024
59e0da0
moar tests
Kemosabert Feb 8, 2024
ad2455d
add finding
Kemosabert Feb 8, 2024
9d0acba
add case
Kemosabert Feb 8, 2024
26d4c43
add tests for full line report with partial string change
Kemosabert Feb 8, 2024
b5b3bab
add comments for test case explanation
Kemosabert Feb 8, 2024
b557f06
remove git hooks
Kemosabert Feb 8, 2024
7e42a75
remove test
Kemosabert Feb 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 13 additions & 8 deletions detect/detect.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,13 +270,6 @@ func (d *Detector) detectRule(fragment Fragment, rule config.Rule) []report.Find
if matchIndex[1] > loc.endLineIndex {
loc.endLineIndex = matchIndex[1]
}

full_fragment := ""
if( len(fragment.Raw) > 250 ){
Kemosabert marked this conversation as resolved.
Show resolved Hide resolved
full_fragment = strings.TrimSpace(fragment.Raw[0:250])
}else{
full_fragment = strings.TrimSpace(fragment.Raw[0:])
}

finding := report.Finding{
Description: rule.Description,
Expand All @@ -291,7 +284,7 @@ func (d *Detector) detectRule(fragment Fragment, rule config.Rule) []report.Find
Match: secret,
Tags: rule.Tags,
Line: fragment.Raw[loc.startLineIndex:loc.endLineIndex],
FullLine: full_fragment,
FullLine: getFullLine(fragment, loc, secret),
}

if strings.Contains(fragment.Raw[loc.startLineIndex:loc.endLineIndex],
Expand Down Expand Up @@ -413,3 +406,15 @@ func (d *Detector) addFinding(finding report.Finding) {
func (d *Detector) addCommit(commit string) {
d.commitMap[commit] = true
}

func getFullLine(frag Fragment, loc Location, secret string) string {
full_line := strings.TrimSpace(frag.Raw[loc.startLineIndex:loc.endLineIndex])

// if line is longer than 250 chars, its most likely found in a minified file
// just return secret in this case
if len(full_line) > 250 {
return secret
}

return full_line
}
191 changes: 191 additions & 0 deletions detect/detect_full_line_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
package detect

import (
"fmt"
"os"
"path/filepath"
"testing"

"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/zricethezav/gitleaks/v8/config"
"github.com/zricethezav/gitleaks/v8/report"
"github.com/zricethezav/gitleaks/v8/sources"
)

// var configPath = "../testdata/config/"
// const repoBasePath = "../testdata/repos/"

// const fixturesBasePath = "../testdata/full_line"

func TestDetectWithFullLine(t *testing.T) {
tests := []struct {
cfgName string
source string
logOpts string
expectedFindings []report.Finding
}{
{
source: filepath.Join(repoBasePath, "full_line"),
cfgName: "simple",
expectedFindings: []report.Finding{
{ // this finding is from a single partial line change in a big xml file
Description: "AWS Access Key",
StartLine: 8,
EndLine: 8,
StartColumn: 15,
EndColumn: 34,
Line: " <YEAR>AKIAJWY75QGOEOC2J5GA</YEAR>",
FullLine: "<YEAR>AKIAJWY75QGOEOC2J5GA</YEAR>",
Match: "AKIAJWY75QGOEOC2J5GA",
Secret: "AKIAJWY75QGOEOC2J5GA",
File: "test.xml",
SymlinkFile: "",
Commit: "a422992fa845ddabb3044fe94f9c7dc816fefd15",
Entropy: 3.6841838,
Author: "Kemosabert",
Email: "[email protected]",
Date: "2024-02-08T09:26:03Z",
Message: "inject secret",
RuleID: "aws-access-key",
Fingerprint: "a422992fa845ddabb3044fe94f9c7dc816fefd15:test.xml:aws-access-key:8",
Tags: []string{"key", "AWS"},
},
{
// this finding is from a file where all contents is on the same line
Description: "Github Personal Access Token",
StartLine: 1,
EndLine: 1,
StartColumn: 1019,
EndColumn: 1058,
Line: "<CATALOG><CD><TITLE>Empire Burlesque</TITLE><ARTIST>Bob Dylan</ARTIST><COUNTRY>USA</COUNTRY><COMPANY>Columbia</COMPANY>z<PRICE>10.90</PRICE><YEAR>2000</YEAR></CD><CD><TITLE>Hide your heart</TITLE><ARTIST>Bonnie Tyler</ARTIST><COUNTRY>UK</COUNTRY><COMPANY>CBS Records</COMPANY><PRICE>9.90</PRICE><YEAR>1988</YEAR></CD><CD><TITLE>Greatest Hits</TITLE><ARTIST> Dolly Parton</ARTIST><COUNTRY>USA</COUNTRY><COMPANY>RCA</COMPANY><PRICE>9.90</PRICE><YEAR>1982</YEAR></CD><CD><TITLE>Still got the blues</TITLE><ARTIST>Gary Moore</ARTIST><COUNTRY>UK</COUNTRY><COMPANY>Virgin records</COMPANY><PRICE>10.20</PRICE><YEAR>1990</YEAR></CD><CD><TITLE>Eros</TITLE><ARTIST>Eros Ramazzotti</ARTIST><COUNTRY>EU</COUNTRY><COMPANY>BMG</COMPANY><PRICE>9.90</PRICE><YEAR>1997</YEAR></CD><CD><TITLE>One night only</TITLE><ARTIST>Bee Gees</ARTIST><COUNTRY>UK</COUNTRY><COMPANY>Polydor</COMPANY><PRICE>10.90</PRICE><YEAR>1998</YEAR></CD><CD><TITLE>Sylvias Mother</TITLE><ARTIST>Dr.Hook</ARTIST><COUNTRY>UK</COUNTRY><COMPANY>CBS</COMPANY><PRICE>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</PRICE><YEAR>1973</YEAR></CD><CD><TITLE>Maggie May</TITLE><ARTIST>Rod Stewart</ARTIST><COUNTRY>UK</COUNTRY><COMPANY>Pickwick</COMPANY><PRICE>8.50</PRICE><YEAR>1990</YEAR></CD><CD><TITLE>Romanza</TITLE><ARTIST>Andrea Bocelli</ARTIST><COUNTRY>EU</COUNTRY><COMPANY>Polydor</COMPANY><PRICE>10.80</PRICE><YEAR>1996</YEAR></CD><CD><TITLE>When a man loves a woman</TITLE><ARTIST>Percy Sledge</ARTIST><COUNTRY>USA</COUNTRY><COMPANY>Atlantic</COMPANY><PRICE>8.70</PRICE><YEAR>1987</YEAR></CD><CD><TITLE>Black angel</TITLE><ARTIST>Savage Rose</ARTIST><COUNTRY>EU</COUNTRY><COMPANY>Mega</COMPANY><PRICE>10.90</PRICE><YEAR>1995</YEAR></CD></CATALOG>",
FullLine: "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",
Match: "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",
Secret: "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",
File: "longfile.txt",
SymlinkFile: "",
Commit: "f181f98031ded3e9fe1b01a057ee0b657152ff9e",
Entropy: 4.8341837,
Author: "Kemosabert",
Email: "[email protected]",
Date: "2024-02-08T09:37:27Z",
Message: "add le token",
RuleID: "github-pat",
Fingerprint: "f181f98031ded3e9fe1b01a057ee0b657152ff9e:longfile.txt:github-pat:1",
Tags: []string{"key", "Github"},
},
{
// this finding is from a partial line change in a yml file
Description: "Github Personal Access Token",
StartLine: 19,
EndLine: 19,
StartColumn: 79,
EndColumn: 118,
Line: " image: europe-docker.pkg.dev/qover-platform/cr/operators/authgoogle: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",
FullLine: "image: europe-docker.pkg.dev/qover-platform/cr/operators/authgoogle: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",
Match: "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",
Secret: "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",
File: "deployment.yaml",
SymlinkFile: "",
Commit: "b03f41c505380c4c70f3a8309b85c880271f4f1f",
Entropy: 4.8341837,
Author: "Kemosabert",
Email: "[email protected]",
Date: "2024-02-08T09:45:49Z",
Message: "increase build",
RuleID: "github-pat",
Fingerprint: "b03f41c505380c4c70f3a8309b85c880271f4f1f:deployment.yaml:github-pat:19",
Tags: []string{"key", "Github"},
},
{
// this finding is from a big file change in a big xml file
Description: "AWS Access Key",
Secret: "AKIALALEMEL33243OLIA",
Match: "AKIALALEMEL33243OLIA",
Line: "\n <TOKEN>AKIALALEMEL33243OLIA</TOKEN>",
FullLine: `<TOKEN>AKIALALEMEL33243OLIA</TOKEN>`,
File: "test.xml",
RuleID: "aws-access-key",
Tags: []string{"key", "AWS"},
StartLine: 185,
EndLine: 185,
StartColumn: 17,
EndColumn: 36,
Entropy: 3.0841837,
Commit: "e654f5bf0f10926b828ccf8f07b5b2f49fd0a179",
Author: "Kemosabert",
Email: "[email protected]",
Date: "2024-02-08T09:09:55Z",
Message: "add test file",
Fingerprint: "e654f5bf0f10926b828ccf8f07b5b2f49fd0a179:test.xml:aws-access-key:185",
},
{
// this finding is from a big file change in a big xml file
Description: "AWS Access Key",
Secret: "AKIAJWY75QGOEOC2J5GA",
Match: "AKIAJWY75QGOEOC2J5GA",
Line: "\n <TITLE>AKIAJWY75QGOEOC2J5GA</TITLE>",
FullLine: `<TITLE>AKIAJWY75QGOEOC2J5GA</TITLE>`,
File: "test.xml",
RuleID: "aws-access-key",
Tags: []string{"key", "AWS"},
StartLine: 204,
EndLine: 204,
StartColumn: 17,
EndColumn: 36,
Entropy: 3.6841838,
Commit: "e654f5bf0f10926b828ccf8f07b5b2f49fd0a179",
Author: "Kemosabert",
Email: "[email protected]",
Date: "2024-02-08T09:09:55Z",
Message: "add test file",
Fingerprint: "e654f5bf0f10926b828ccf8f07b5b2f49fd0a179:test.xml:aws-access-key:204",
},
},
},
}

moveDotGit(t, "dotGit", ".git")
defer moveDotGit(t, ".git", "dotGit")

for _, tt := range tests {

viper.AddConfigPath(configPath)
viper.SetConfigName("simple")
viper.SetConfigType("toml")
err := viper.ReadInConfig()
require.NoError(t, err)

var vc config.ViperConfig
err = viper.Unmarshal(&vc)
require.NoError(t, err)
cfg, err := vc.Translate()
require.NoError(t, err)
detector := NewDetector(cfg)

var ignorePath string
info, err := os.Stat(tt.source)
require.NoError(t, err)

if info.IsDir() {
ignorePath = filepath.Join(tt.source, ".gitleaksignore")
} else {
ignorePath = filepath.Join(filepath.Dir(tt.source), ".gitleaksignore")
}
err = detector.AddGitleaksIgnore(ignorePath)
require.NoError(t, err)

gitCmd, err := sources.NewGitLogCmd(tt.source, tt.logOpts)
require.NoError(t, err)
findings, err := detector.DetectGit(gitCmd)
fmt.Printf("%+v", err)
require.NoError(t, err)

for _, f := range findings {
f.Match = "" // remove lines cause copying and pasting them has some wack formatting
}
assert.ElementsMatch(t, tt.expectedFindings, findings)
}
}
38 changes: 15 additions & 23 deletions detect/detect_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ func TestDetect(t *testing.T) {
Match: "AKIALALEMEL33243OKIA",
File: "tmp.go",
Line: `awsToken := \"AKIALALEMEL33243OKIA\"`,
FullLine: `awsToken := \"AKIALALEMEL33243OKIA\"`,
RuleID: "aws-access-key",
Tags: []string{"key", "AWS"},
StartLine: 0,
Expand All @@ -90,6 +91,7 @@ func TestDetect(t *testing.T) {
Secret: "pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB",
Match: "pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB",
Line: `pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB`,
FullLine: `pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB`,
File: "tmp.go",
RuleID: "pypi-upload-token",
Tags: []string{"key", "pypi"},
Expand All @@ -101,29 +103,6 @@ func TestDetect(t *testing.T) {
},
},
},
{
cfgName: "simple",
fragment: Fragment{
Raw: `awsToken := \"AKIALALEMEL33243OLIA\"`,
FilePath: "tmp.go",
},
expectedFindings: []report.Finding{
{
Description: "AWS Access Key",
Secret: "AKIALALEMEL33243OLIA",
Match: "AKIALALEMEL33243OLIA",
Line: `awsToken := \"AKIALALEMEL33243OLIA\"`,
File: "tmp.go",
RuleID: "aws-access-key",
Tags: []string{"key", "AWS"},
StartLine: 0,
EndLine: 0,
StartColumn: 15,
EndColumn: 34,
Entropy: 3.0841837,
},
},
},
{
cfgName: "simple",
fragment: Fragment{
Expand All @@ -136,6 +115,7 @@ func TestDetect(t *testing.T) {
Match: "BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef;",
Secret: "cafebabe:deadbeef",
Line: `export BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef;`,
FullLine: `export BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef;`,
File: "tmp.sh",
RuleID: "sidekiq-secret",
Tags: []string{},
Expand All @@ -160,6 +140,7 @@ func TestDetect(t *testing.T) {
Secret: "cafebabe:deadbeef",
File: "tmp.sh",
Line: `echo hello1; export BUNDLE_ENTERPRISE__CONTRIBSYS__COM="cafebabe:deadbeef" && echo hello2`,
FullLine: `echo hello1; export BUNDLE_ENTERPRISE__CONTRIBSYS__COM="cafebabe:deadbeef" && echo hello2`,
RuleID: "sidekiq-secret",
Tags: []string{},
Entropy: 2.6098502,
Expand All @@ -183,6 +164,7 @@ func TestDetect(t *testing.T) {
Secret: "cafeb4b3:d3adb33f",
File: "tmp.sh",
Line: `url = "http://cafeb4b3:[email protected]:80/path?param1=true&param2=false#heading1"`,
FullLine: `url = "http://cafeb4b3:[email protected]:80/path?param1=true&param2=false#heading1"`,
RuleID: "sidekiq-sensitive-url",
Tags: []string{},
Entropy: 2.984234,
Expand Down Expand Up @@ -230,6 +212,7 @@ func TestDetect(t *testing.T) {
Match: "Discord_Public_Key = \"e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5\"",
Secret: "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5",
Line: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`,
FullLine: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`,
File: "tmp.go",
RuleID: "discord-api-key",
Tags: []string{},
Expand Down Expand Up @@ -261,6 +244,7 @@ func TestDetect(t *testing.T) {
Match: "Key = \"e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5\"",
Secret: "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5",
Line: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`,
FullLine: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`,
File: "tmp.py",
RuleID: "generic-api-key",
Tags: []string{},
Expand Down Expand Up @@ -373,6 +357,7 @@ func TestFromGit(t *testing.T) {
StartColumn: 19,
EndColumn: 38,
Line: "\n awsToken := \"AKIALALEMEL33243OLIA\"",
FullLine: `awsToken := "AKIALALEMEL33243OLIA"`,
Secret: "AKIALALEMEL33243OLIA",
Match: "AKIALALEMEL33243OLIA",
File: "main.go",
Expand All @@ -395,6 +380,7 @@ func TestFromGit(t *testing.T) {
Secret: "AKIALALEMEL33243OLIA",
Match: "AKIALALEMEL33243OLIA",
Line: "\n\taws_token := \"AKIALALEMEL33243OLIA\"",
FullLine: `aws_token := "AKIALALEMEL33243OLIA"`,
File: "foo/foo.go",
Date: "2021-11-02T23:48:06Z",
Commit: "491504d5a31946ce75e22554cc34203d8e5ff3ca",
Expand All @@ -421,6 +407,7 @@ func TestFromGit(t *testing.T) {
EndColumn: 36,
Secret: "AKIALALEMEL33243OLIA",
Line: "\n\taws_token := \"AKIALALEMEL33243OLIA\"",
FullLine: `aws_token := "AKIALALEMEL33243OLIA"`,
Match: "AKIALALEMEL33243OLIA",
Date: "2021-11-02T23:48:06Z",
File: "foo/foo.go",
Expand Down Expand Up @@ -470,6 +457,7 @@ func TestFromGit(t *testing.T) {
gitCmd, err := sources.NewGitLogCmd(tt.source, tt.logOpts)
require.NoError(t, err)
findings, err := detector.DetectGit(gitCmd)
fmt.Printf("%+v", err)
require.NoError(t, err)

for _, f := range findings {
Expand All @@ -496,6 +484,7 @@ func TestFromGitStaged(t *testing.T) {
StartColumn: 18,
EndColumn: 37,
Line: "\n\taws_token2 := \"AKIALALEMEL33243OLIA\" // this one is not",
FullLine: `aws_token2 := "AKIALALEMEL33243OLIA" // this one is not`,
Match: "AKIALALEMEL33243OLIA",
Secret: "AKIALALEMEL33243OLIA",
File: "api/api.go",
Expand Down Expand Up @@ -568,6 +557,7 @@ func TestFromFiles(t *testing.T) {
Match: "AKIALALEMEL33243OLIA",
Secret: "AKIALALEMEL33243OLIA",
Line: "\n\tawsToken := \"AKIALALEMEL33243OLIA\"",
FullLine: `awsToken := "AKIALALEMEL33243OLIA"`,
File: "../testdata/repos/nogit/main.go",
SymlinkFile: "",
RuleID: "aws-access-key",
Expand All @@ -590,6 +580,7 @@ func TestFromFiles(t *testing.T) {
Match: "AKIALALEMEL33243OLIA",
Secret: "AKIALALEMEL33243OLIA",
Line: "\n\tawsToken := \"AKIALALEMEL33243OLIA\"",
FullLine: `awsToken := "AKIALALEMEL33243OLIA"`,
File: "../testdata/repos/nogit/main.go",
RuleID: "aws-access-key",
Tags: []string{"key", "AWS"},
Expand Down Expand Up @@ -657,6 +648,7 @@ func TestDetectWithSymlinks(t *testing.T) {
Match: "-----BEGIN OPENSSH PRIVATE KEY-----",
Secret: "-----BEGIN OPENSSH PRIVATE KEY-----",
Line: "-----BEGIN OPENSSH PRIVATE KEY-----",
FullLine: "-----BEGIN OPENSSH PRIVATE KEY-----",
File: "../testdata/repos/symlinks/source_file/id_ed25519",
SymlinkFile: "../testdata/repos/symlinks/file_symlink/symlinked_id_ed25519",
RuleID: "apkey",
Expand Down
1 change: 1 addition & 0 deletions report/json_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ func TestWriteJSON(t *testing.T) {

Description: "",
RuleID: "test-rule",
FullLine: "",
Match: "line containing secret",
Secret: "a secret",
StartLine: 1,
Expand Down
Loading
Loading