Added support for indeterminate verification for letter Z detectors (#4165)

* initial commit

* handle and set verification errors

* some fixes

Author: kashifkhan0771

pkg/detectors/zapierwebhook/zapierwebhook.go

@@ -2,6 +2,8 @@ package zapierwebhook
 
 import (
 	"context"
+	"fmt"
+	"io"
 	"net/http"
 	"strings"
 
@@ -44,17 +46,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "POST", "https://hooks.zapier.com/hooks/catch/"+resMatch, nil)
-			if err != nil {
-				continue
-			}
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				}
-			}
+			isVerified, verificationErr := verifyZapierWebhook(ctx, client, resMatch)
+			s1.Verified = isVerified
+			s1.SetVerificationError(verificationErr, resMatch)
 		}
 
 		results = append(results, s1)
@@ -70,3 +64,29 @@ func (s Scanner) Type() detectorspb.DetectorType {
 func (s Scanner) Description() string {
 	return "Zapier is an automation tool that connects your apps and services. Zapier webhooks can be used to automate workflows by sending HTTP requests to a unique URL."
 }
+
+func verifyZapierWebhook(ctx context.Context, client *http.Client, key string) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://hooks.zapier.com/hooks/catch/"+key, http.NoBody)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+
+	defer func() {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusUnauthorized:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+}

pkg/detectors/zendeskapi/zendeskapi.go

@@ -2,18 +2,17 @@ package zendeskapi
 
 import (
 	"context"
-	b64 "encoding/base64"
 	"fmt"
+	"io"
 	"net/http"
-	"strings"
 
 	regexp "github.com/wasilibs/go-re2"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{
+type Scanner struct {
 	detectors.DefaultMultiPartCredentialProvider
 }
 
@@ -25,7 +24,7 @@ var (
 
 	token  = regexp.MustCompile(detectors.PrefixRegex([]string{"zendesk"}) + `([A-Za-z0-9_-]{40})`)
 	email  = regexp.MustCompile(`\b([a-zA-Z-0-9-]{5,16}\@[a-zA-Z-0-9]{4,16}\.[a-zA-Z-0-9]{3,6})\b`)
-	domain = regexp.MustCompile(`\b([a-zA-Z-0-9]{3,16}\.zendesk\.com)\b`)
+	domain = regexp.MustCompile(`\b([a-zA-Z-0-9]{3,25}\.zendesk\.com)\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -38,40 +37,32 @@ func (s Scanner) Keywords() []string {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
-	tokens := token.FindAllStringSubmatch(dataStr, -1)
-	domains := domain.FindAllStringSubmatch(dataStr, -1)
-	emails := email.FindAllStringSubmatch(dataStr, -1)
+	var uniqueEmails, uniqueTokens, uniqueDomains = make(map[string]struct{}), make(map[string]struct{}), make(map[string]struct{})
 
-	for _, token := range tokens {
-		resMatch := strings.TrimSpace(token[1])
+	for _, match := range email.FindAllStringSubmatch(dataStr, -1) {
+		uniqueEmails[match[1]] = struct{}{}
+	}
 
-		var resDomain string
-		for _, domain := range domains {
-			resDomain = strings.TrimSpace(domain[1])
+	for _, match := range token.FindAllStringSubmatch(dataStr, -1) {
+		uniqueTokens[match[1]] = struct{}{}
+	}
 
-			for _, email := range emails {
-				resEmail := strings.TrimSpace(email[1])
+	for _, match := range domain.FindAllStringSubmatch(dataStr, -1) {
+		uniqueDomains[match[1]] = struct{}{}
+	}
 
+	for token := range uniqueTokens {
+		for email := range uniqueEmails {
+			for domain := range uniqueDomains {
 				s1 := detectors.Result{
 					DetectorType: detectorspb.DetectorType_ZendeskApi,
-					Raw:          []byte(resMatch),
+					Raw:          []byte(token),
 				}
 
 				if verify {
-					data := fmt.Sprintf("%s/token:%s", resEmail, resMatch)
-					sEnc := b64.StdEncoding.EncodeToString([]byte(data))
-					req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/api/v2/users.json", nil)
-					if err != nil {
-						continue
-					}
-					req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
-					res, err := client.Do(req)
-					if err == nil {
-						defer res.Body.Close()
-						if res.StatusCode >= 200 && res.StatusCode < 300 {
-							s1.Verified = true
-						}
-					}
+					isVerified, verificationErr := verifyZendesk(ctx, client, email, token, domain)
+					s1.Verified = isVerified
+					s1.SetVerificationError(verificationErr, token)
 				}
 
 				results = append(results, s1)
@@ -91,3 +82,31 @@ func (s Scanner) Type() detectorspb.DetectorType {
 func (s Scanner) Description() string {
 	return "Zendesk is a customer service platform. Zendesk API tokens can be used to access and modify customer service data."
 }
+
+func verifyZendesk(ctx context.Context, client *http.Client, email, token, domain string) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+domain+"/api/v2/users.json", http.NoBody)
+	if err != nil {
+		return false, err
+	}
+
+	// docs: https://developer.zendesk.com/api-reference/introduction/security-and-auth/
+	req.SetBasicAuth(email+"/token", token)
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+
+	defer func() {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusUnauthorized, http.StatusNotFound:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+}

pkg/detectors/zenkitapi/zenkitapi.go

@@ -2,10 +2,13 @@ package zenkitapi
 
 import (
 	"context"
-	regexp "github.com/wasilibs/go-re2"
+	"fmt"
+	"io"
 	"net/http"
 	"strings"
 
+	regexp "github.com/wasilibs/go-re2"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -44,18 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://base.zenkit.com/api/v1/users/me", nil)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Zenkit-API-Key", resMatch)
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				}
-			}
+			isVerified, verificationErr := verifyZenkitKey(ctx, client, resMatch)
+			s1.Verified = isVerified
+			s1.SetVerificationError(verificationErr, resMatch)
 		}
 
 		results = append(results, s1)
@@ -71,3 +65,31 @@ func (s Scanner) Type() detectorspb.DetectorType {
 func (s Scanner) Description() string {
 	return "Zenkit is a collaborative SaaS platform for project management, database building, and more. Zenkit API keys can be used to access and interact with Zenkit's services programmatically."
 }
+
+func verifyZenkitKey(ctx context.Context, client *http.Client, key string) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://base.zenkit.com/api/v1/users/me", http.NoBody)
+	if err != nil {
+		return false, err
+	}
+
+	req.Header.Add("Zenkit-API-Key", key)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+
+	defer func() {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusUnauthorized:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+}

pkg/detectors/zenrows/zenrows.go

@@ -3,10 +3,12 @@ package zenrows
 import (
 	"context"
 	"fmt"
-	regexp "github.com/wasilibs/go-re2"
+	"io"
 	"net/http"
 	"strings"
 
+	regexp "github.com/wasilibs/go-re2"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -45,17 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.zenrows.com/v1/?apikey=%s&url=https://httpbin.org/anything", resMatch), nil)
-			if err != nil {
-				continue
-			}
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				}
-			}
+			isVerified, verificationErr := verifyZenrowsKey(ctx, client, resMatch)
+			s1.Verified = isVerified
+			s1.SetVerificationError(verificationErr, resMatch)
 		}
 
 		results = append(results, s1)
@@ -71,3 +65,29 @@ func (s Scanner) Type() detectorspb.DetectorType {
 func (s Scanner) Description() string {
 	return "ZenRows is a web scraping API service that allows users to extract data from websites. ZenRows API keys can be used to access and scrape web data."
 }
+
+func verifyZenrowsKey(ctx context.Context, client *http.Client, key string) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("https://api.zenrows.com/v1/?apikey=%s&url=https://httpbin.org/anything", key), http.NoBody)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+
+	defer func() {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusUnauthorized, http.StatusPaymentRequired: // docs: https://docs.zenrows.com/api-error-codes#402-payment-required
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+}

pkg/detectors/zenscrape/zenscrape.go

@@ -2,6 +2,8 @@ package zenscrape
 
 import (
 	"context"
+	"fmt"
+	"io"
 	"net/http"
 	"strings"
 
@@ -45,18 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://app.zenscrape.com/api/v1/status", nil)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("apikey", resMatch)
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if (res.StatusCode >= 200 && res.StatusCode < 300) || res.StatusCode == 429 {
-					s1.Verified = true
-				}
-			}
+			isVerified, verificationErr := verifyZenScrapeKey(ctx, client, resMatch)
+			s1.Verified = isVerified
+			s1.SetVerificationError(verificationErr, resMatch)
 		}
 
 		results = append(results, s1)
@@ -72,3 +65,31 @@ func (s Scanner) Type() detectorspb.DetectorType {
 func (s Scanner) Description() string {
 	return "Zenscrape is a web scraping service that provides an API to extract data from websites. Zenscrape API keys can be used to access and scrape data from web pages."
 }
+
+func verifyZenScrapeKey(ctx context.Context, client *http.Client, key string) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://app.zenscrape.com/api/v1/status", http.NoBody)
+	if err != nil {
+		return false, err
+	}
+
+	req.Header.Add("apikey", key)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+
+	defer func() {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+
+	switch resp.StatusCode {
+	case http.StatusOK, http.StatusTooManyRequests:
+		return true, nil
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+}