diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go index 1be6d1c1f6e3..e971e09b9673 100644 --- a/pkg/detectors/detectors.go +++ b/pkg/detectors/detectors.go @@ -4,7 +4,9 @@ import ( "context" "crypto/rand" "errors" + "io" "math/big" + "net/http" "net/url" "strings" "unicode" @@ -13,6 +15,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" + "golang.org/x/sync/singleflight" ) // Detector defines an interface for scanning for and verifying secrets. @@ -316,3 +319,39 @@ func ParseURLAndStripPathAndParams(u string) (*url.URL, error) { parsedURL.RawQuery = "" return parsedURL, nil } + +type VerificationResult struct { + StatusCode int + Body []byte +} + +var verificationGroup = new(singleflight.Group) + +func VerificationRequest(identifier string, request *http.Request, client *http.Client) (*VerificationResult, error) { + result, err, _ := verificationGroup.Do(identifier, func() (any, error) { + resp, err := client.Do(request) + if err != nil { + return nil, err + } + + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return &VerificationResult{ + StatusCode: resp.StatusCode, + Body: bodyBytes, + }, nil + }) + if err != nil { + return nil, err + } + + return result.(*VerificationResult), nil +} diff --git a/pkg/detectors/detectors_test.go b/pkg/detectors/detectors_test.go index 67c6b053cde3..a4fc49a98b31 100644 --- a/pkg/detectors/detectors_test.go +++ b/pkg/detectors/detectors_test.go @@ -4,8 +4,12 @@ package detectors import ( + "net/http" + "sync" "testing" + "time" + "github.com/stretchr/testify/assert" regexp "github.com/wasilibs/go-re2" ) @@ -67,6 +71,66 @@ func TestPrefixRegexKeywords(t *testing.T) { } } +// The https://httpbin.org/uuid API returns a new UUID on each call. +// However, because we're using singleflight and issuing concurrent requests, +// all response bodies should be identical (only one actual HTTP request is made). +func TestVerificationRequest_Singleflight(t *testing.T) { + client := &http.Client{ + Timeout: 10 * time.Second, + } + + // Create two separate *http.Request instances pointing to same endpoint + request1, err := http.NewRequest(http.MethodGet, "https://httpbin.org/uuid", http.NoBody) + assert.NoError(t, err) + + request2, err := http.NewRequest(http.MethodGet, "https://httpbin.org/uuid", http.NoBody) + assert.NoError(t, err) + + const key = "uuid-test" + + var wg sync.WaitGroup + const goroutines = 5 + results := make([]*VerificationResult, goroutines) + errors := make([]error, goroutines) + + // launch several concurrent goroutines all requesting the same identifier + for i := range goroutines { + wg.Add(1) + go func(i int) { + defer wg.Done() + // alternate between two identical requests just to prove it doesn't matter + req := request1 + if i%2 == 0 { + req = request2 + } + + res, err := VerificationRequest(key, req, client) + results[i] = res + errors[i] = err + }(i) + } + + wg.Wait() + + for _, err := range errors { + assert.NoError(t, err) + } + + // all goroutines should get a non-nil result + for _, r := range results { + assert.NotNil(t, r) + } + + // since singleflight coalesces concurrent calls, all results should have identical bodies + firstBody := results[0].Body + for i := 1; i < goroutines; i++ { + assert.Equal(t, string(firstBody), string(results[i].Body), + "Expected all results to share the same response body (one HTTP call only)") + } + + t.Logf("All %d goroutines received the same UUID: %s", goroutines, string(firstBody)) +} + func BenchmarkPrefixRegex(b *testing.B) { kws := []string{"securitytrails"} for i := 0; i < b.N; i++ { diff --git a/pkg/detectors/meraki/meraki.go b/pkg/detectors/meraki/meraki.go index e326e84ced34..2e5c9099ca91 100644 --- a/pkg/detectors/meraki/meraki.go +++ b/pkg/detectors/meraki/meraki.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "io" "net/http" regexp "github.com/wasilibs/go-re2" @@ -58,13 +57,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result dataStr := string(data) // uniqueMatches will hold unique match values and ensure we only process unique matches found in the data string - var uniqueMatches = make(map[string]struct{}) + var matches = make([]string, 0) for _, match := range apiKey.FindAllStringSubmatch(dataStr, -1) { - uniqueMatches[match[1]] = struct{}{} + matches = append(matches, match[1]) } - for match := range uniqueMatches { + for _, match := range matches { s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_Meraki, Raw: []byte(match), @@ -110,20 +109,16 @@ func verifyMerakiApiKey(ctx context.Context, client *http.Client, match string) // set the required auth header req.Header.Set("X-Cisco-Meraki-API-Key", match) - resp, err := client.Do(req) + result, err := detectors.VerificationRequest(match, req, client) if err != nil { return nil, false, err } - defer func() { - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - }() - switch resp.StatusCode { + switch result.StatusCode { case http.StatusOK: // in case token is verified, capture the organization id's and name which are accessible via token. var organizations []merakiOrganizations - if err = json.NewDecoder(resp.Body).Decode(&organizations); err != nil { + if err = json.Unmarshal(result.Body, &organizations); err != nil { return nil, false, err } @@ -131,6 +126,6 @@ func verifyMerakiApiKey(ctx context.Context, client *http.Client, match string) case http.StatusUnauthorized: return nil, false, nil default: - return nil, false, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + return nil, false, fmt.Errorf("unexpected status code: %d", result.StatusCode) } } diff --git a/pkg/output/plain.go b/pkg/output/plain.go index 62c08c506fb5..2a5e8ceaa002 100644 --- a/pkg/output/plain.go +++ b/pkg/output/plain.go @@ -57,9 +57,11 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) yellowPrinter.Printf("Verification issue: %s\n", out.VerificationError) } } + if r.VerificationFromCache { - cyanPrinter.Print("(Verification info cached)\n") + cyanPrinter.Print("(🔍 Using cached verification)\n") } + printer.Printf("Detector Type: %s\n", out.DetectorType) printer.Printf("Decoder Type: %s\n", out.DecoderType) printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw))