[Testing] Avoid duplicate verification requests which happen concurrently

pkg/detectors/detectors.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"crypto/rand"
 	"errors"
+	"io"
 	"math/big"
+	"net/http"
 	"net/url"
 	"strings"
 	"unicode"
@@ -13,6 +15,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"golang.org/x/sync/singleflight"
 )
 
 // Detector defines an interface for scanning for and verifying secrets.
@@ -316,3 +319,39 @@ func ParseURLAndStripPathAndParams(u string) (*url.URL, error) {
 	parsedURL.RawQuery = ""
 	return parsedURL, nil
 }
+
+type VerificationResult struct {
+	StatusCode int
+	Body       []byte
+}
+
+var verificationGroup = new(singleflight.Group)
+
+func VerificationRequest(identifier string, request *http.Request, client *http.Client) (*VerificationResult, error) {
+	result, err, _ := verificationGroup.Do(identifier, func() (any, error) {
+		resp, err := client.Do(request)
+		if err != nil {
+			return nil, err
+		}
+
+		defer func() {
+			_, _ = io.Copy(io.Discard, resp.Body)
+			_ = resp.Body.Close()
+		}()
+
+		bodyBytes, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+
+		return &VerificationResult{
+			StatusCode: resp.StatusCode,
+			Body:       bodyBytes,
+		}, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return result.(*VerificationResult), nil
+}

pkg/detectors/detectors_test.go

@@ -4,8 +4,12 @@
 package detectors
 
 import (
+	"net/http"
+	"sync"
 	"testing"
+	"time"
 
+	"github.com/stretchr/testify/assert"
 	regexp "github.com/wasilibs/go-re2"
 )
 
@@ -67,6 +71,66 @@ func TestPrefixRegexKeywords(t *testing.T) {
 	}
 }
 
+// The https://httpbin.org/uuid API returns a new UUID on each call.
+// However, because we're using singleflight and issuing concurrent requests,
+// all response bodies should be identical (only one actual HTTP request is made).
+func TestVerificationRequest_Singleflight(t *testing.T) {
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	// Create two separate *http.Request instances pointing to same endpoint
+	request1, err := http.NewRequest(http.MethodGet, "https://httpbin.org/uuid", http.NoBody)
+	assert.NoError(t, err)
+
+	request2, err := http.NewRequest(http.MethodGet, "https://httpbin.org/uuid", http.NoBody)
+	assert.NoError(t, err)
+
+	const key = "uuid-test"
+
+	var wg sync.WaitGroup
+	const goroutines = 5
+	results := make([]*VerificationResult, goroutines)
+	errors := make([]error, goroutines)
+
+	// launch several concurrent goroutines all requesting the same identifier
+	for i := range goroutines {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			// alternate between two identical requests just to prove it doesn't matter
+			req := request1
+			if i%2 == 0 {
+				req = request2
+			}
+
+			res, err := VerificationRequest(key, req, client)
+			results[i] = res
+			errors[i] = err
+		}(i)
+	}
+
+	wg.Wait()
+
+	for _, err := range errors {
+		assert.NoError(t, err)
+	}
+
+	// all goroutines should get a non-nil result
+	for _, r := range results {
+		assert.NotNil(t, r)
+	}
+
+	// since singleflight coalesces concurrent calls, all results should have identical bodies
+	firstBody := results[0].Body
+	for i := 1; i < goroutines; i++ {
+		assert.Equal(t, string(firstBody), string(results[i].Body),
+			"Expected all results to share the same response body (one HTTP call only)")
+	}
+
+	t.Logf("All %d goroutines received the same UUID: %s", goroutines, string(firstBody))
+}
+
 func BenchmarkPrefixRegex(b *testing.B) {
 	kws := []string{"securitytrails"}
 	for i := 0; i < b.N; i++ {

pkg/detectors/meraki/meraki.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
 
 	regexp "github.com/wasilibs/go-re2"
@@ -58,13 +57,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	dataStr := string(data)
 
 	// uniqueMatches will hold unique match values and ensure we only process unique matches found in the data string
-	var uniqueMatches = make(map[string]struct{})
+	var matches = make([]string, 0)
 
 	for _, match := range apiKey.FindAllStringSubmatch(dataStr, -1) {
-		uniqueMatches[match[1]] = struct{}{}
+		matches = append(matches, match[1])
 	}
 
-	for match := range uniqueMatches {
+	for _, match := range matches {
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Meraki,
 			Raw:          []byte(match),
@@ -110,27 +109,23 @@ func verifyMerakiApiKey(ctx context.Context, client *http.Client, match string)
 	// set the required auth header
 	req.Header.Set("X-Cisco-Meraki-API-Key", match)
 
-	resp, err := client.Do(req)
+	result, err := detectors.VerificationRequest(match, req, client)
 	if err != nil {
 		return nil, false, err
 	}
-	defer func() {
-		_, _ = io.Copy(io.Discard, resp.Body)
-		_ = resp.Body.Close()
-	}()
 
-	switch resp.StatusCode {
+	switch result.StatusCode {
 	case http.StatusOK:
 		// in case token is verified, capture the organization id's and name which are accessible via token.
 		var organizations []merakiOrganizations
-		if err = json.NewDecoder(resp.Body).Decode(&organizations); err != nil {
+		if err = json.Unmarshal(result.Body, &organizations); err != nil {
 			return nil, false, err
 		}
 
 		return organizations, true, nil
 	case http.StatusUnauthorized:
 		return nil, false, nil
 	default:
-		return nil, false, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+		return nil, false, fmt.Errorf("unexpected status code: %d", result.StatusCode)
 	}
 }

pkg/output/plain.go

@@ -57,9 +57,11 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
 			yellowPrinter.Printf("Verification issue: %s\n", out.VerificationError)
 		}
 	}
+
 	if r.VerificationFromCache {
-		cyanPrinter.Print("(Verification info cached)\n")
+		cyanPrinter.Print("(🔍 Using cached verification)\n")
 	}
+
 	printer.Printf("Detector Type: %s\n", out.DetectorType)
 	printer.Printf("Decoder Type: %s\n", out.DecoderType)
 	printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw))