-
Notifications
You must be signed in to change notification settings - Fork 2.1k
V2 detector for confluent #4489
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Draft
ap00rv
wants to merge
4
commits into
trufflesecurity:main
Choose a base branch
from
ap00rv:ap00rv/update-cflt-secrets-pattern
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Draft
Changes from all commits
Commits
Show all changes
4 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| package confluent | ||
|
|
||
| import ( | ||
| "context" | ||
| b64 "encoding/base64" | ||
| "fmt" | ||
| "hash/crc32" | ||
| "strings" | ||
|
|
||
| regexp "github.com/wasilibs/go-re2" | ||
|
|
||
| "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" | ||
| "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" | ||
| ) | ||
|
|
||
| type Scanner struct { | ||
| detectors.DefaultMultiPartCredentialProvider | ||
| } | ||
|
|
||
| // Ensure the Scanner satisfies the interface at compile time. | ||
| var _ detectors.Detector = (*Scanner)(nil) | ||
|
|
||
| var ( | ||
| keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"confluent"}) + `\b([A-Z0-9]{16})\b`) | ||
| // Match cflt prefix followed by 60 characters consisting of A-Z, a-z, 0-9, + or / | ||
| //See https://docs.confluent.io/cloud/current/security/authenticate/workload-identities/service-accounts/api-keys/overview.html#api-secret-format | ||
| secretPat = regexp.MustCompile(`\b(cflt[A-Za-z0-9+/]{60})\b`) | ||
| ) | ||
|
|
||
| // Keywords are used for efficiently pre-filtering chunks. | ||
| // Use identifiers in the secret preferably, or the provider name. | ||
| func (s Scanner) Keywords() []string { | ||
| return []string{"cflt"} | ||
| } | ||
|
|
||
| func (s Scanner) Version() int { return 2 } | ||
|
|
||
| // FromData will find and optionally verify Confluent secrets in a given set of bytes. | ||
| func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { | ||
| dataStr := string(data) | ||
|
|
||
| matches := keyPat.FindAllStringSubmatch(dataStr, -1) | ||
| secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1) | ||
|
|
||
| for _, match := range matches { | ||
| resMatch := strings.TrimSpace(match[1]) | ||
|
|
||
| for _, match := range secretMatches { | ||
| resSecret := strings.TrimSpace(match[1]) // Use index 1 for the captured group | ||
|
|
||
| s1 := detectors.Result{ | ||
| DetectorType: detectorspb.DetectorType_Confluent, | ||
| Raw: []byte(resMatch), | ||
| RawV2: []byte(resMatch + resSecret), | ||
| ExtraData: map[string]string{ | ||
| "rotation_guide": "https://docs.confluent.io/cloud/current/security/authenticate/workload-identities/service-accounts/api-keys/best-practices-api-keys.html#rotate-api-keys-regularly", | ||
| "version": fmt.Sprintf("%d", s.Version()), | ||
| }, | ||
| } | ||
|
|
||
| if verify { | ||
| s1.Verified = verifyConfluentSecret(resSecret) | ||
| } | ||
|
|
||
| results = append(results, s1) | ||
| } | ||
| } | ||
|
|
||
| return results, nil | ||
| } | ||
|
|
||
| // verifyConfluentSecret verifies the Confluent secret by checking the CRC32 checksum | ||
| func verifyConfluentSecret(secret string) bool { | ||
| if len(secret) != 64 { // cflt + 60 characters | ||
| return false | ||
| } | ||
|
|
||
| if !strings.HasPrefix(secret, "cflt") { | ||
| return false | ||
| } | ||
|
|
||
| // Extract the first 54 characters after 'cflt' prefix (58 total - 4 for cflt) | ||
| payload := secret[4:58] // Characters 5-58 (54 characters) | ||
|
|
||
| // Extract the last 6 characters as the checksum | ||
| checksumEncoded := secret[58:64] | ||
|
|
||
| // Decode the checksum from base64 | ||
| checksumBytes, err := b64.StdEncoding.DecodeString(checksumEncoded + "==") // Add padding if needed | ||
| if err != nil { | ||
| // Try without padding | ||
| checksumBytes, err = b64.StdEncoding.DecodeString(checksumEncoded) | ||
| if err != nil { | ||
| return false | ||
| } | ||
| } | ||
|
|
||
| if len(checksumBytes) < 4 { | ||
| return false | ||
| } | ||
|
|
||
| // Calculate CRC32 checksum of the payload | ||
| expectedChecksum := crc32.ChecksumIEEE([]byte(payload)) | ||
|
|
||
| // Convert received checksum bytes to uint32 (little endian to match the encoding) | ||
| receivedChecksum := uint32(checksumBytes[3])<<24 | uint32(checksumBytes[2])<<16 | | ||
| uint32(checksumBytes[1])<<8 | uint32(checksumBytes[0]) | ||
|
|
||
| return expectedChecksum == receivedChecksum | ||
| } | ||
|
|
||
| func (s Scanner) Type() detectorspb.DetectorType { | ||
| return detectorspb.DetectorType_Confluent | ||
| } | ||
|
|
||
| func (s Scanner) Description() string { | ||
| return "Confluent provides a streaming platform based on Apache Kafka to help companies harness their data in real-time. Confluent Cloud API keys can be used to access and manage Confluent Cloud control plane APIs and resources." | ||
| } |
123 changes: 123 additions & 0 deletions
123
pkg/detectors/confluent/v2/confluent_integration_test.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| //go:build detectors | ||
| // +build detectors | ||
|
|
||
| package confluent | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "testing" | ||
|
|
||
| "github.com/kylelemons/godebug/pretty" | ||
| "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" | ||
|
|
||
| "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" | ||
| ) | ||
|
|
||
| func TestConfluent_FromChunk(t *testing.T) { | ||
| // Valid secret with proper CRC32 checksum | ||
| validSecret := "cfltT8d8RzkNseTMEDKcjNM1BZTFPHqRn/dQm9q7w6SjzZ12wZfwjaJdipHZtDjw" | ||
| // Invalid secret with wrong checksum | ||
| invalidSecret := "cfltT8d8RzkNseTMEDKcjNM1BZTFPHqRn/dQm9q7w6SjzZ12wZfwjaJdipHZtDje" | ||
| key := "JSAOOCIC74SGECCP" | ||
|
|
||
| type args struct { | ||
| ctx context.Context | ||
| data []byte | ||
| verify bool | ||
| } | ||
| tests := []struct { | ||
| name string | ||
| s Scanner | ||
| args args | ||
| want []detectors.Result | ||
| wantErr bool | ||
| }{ | ||
| { | ||
| name: "found, verified", | ||
| s: Scanner{}, | ||
| args: args{ | ||
| ctx: context.Background(), | ||
| data: []byte(fmt.Sprintf("You can find a confluent secret %s with key %s", validSecret, key)), | ||
| verify: true, | ||
| }, | ||
| want: []detectors.Result{ | ||
| { | ||
| DetectorType: detectorspb.DetectorType_Confluent, | ||
| Verified: true, | ||
| ExtraData: map[string]string{ | ||
| "rotation_guide": "https://docs.confluent.io/cloud/current/security/authenticate/workload-identities/service-accounts/api-keys/best-practices-api-keys.html#rotate-api-keys-regularly", | ||
| "version": "2", | ||
| }, | ||
| }, | ||
| }, | ||
| wantErr: false, | ||
| }, | ||
| { | ||
| name: "found, unverified", | ||
| s: Scanner{}, | ||
| args: args{ | ||
| ctx: context.Background(), | ||
| data: []byte(fmt.Sprintf("You can find a confluent secret %s with %s key but not valid", invalidSecret, key)), // the secret would satisfy the regex but not pass validation | ||
| verify: true, | ||
| }, | ||
| want: []detectors.Result{ | ||
| { | ||
| DetectorType: detectorspb.DetectorType_Confluent, | ||
| Verified: false, | ||
| ExtraData: map[string]string{ | ||
| "rotation_guide": "https://docs.confluent.io/cloud/current/security/authenticate/workload-identities/service-accounts/api-keys/best-practices-api-keys.html#rotate-api-keys-regularly", | ||
| "version": "2", | ||
| }, | ||
| }, | ||
| }, | ||
| wantErr: false, | ||
| }, | ||
| { | ||
| name: "not found", | ||
| s: Scanner{}, | ||
| args: args{ | ||
| ctx: context.Background(), | ||
| data: []byte("You cannot find the secret within"), | ||
| verify: true, | ||
| }, | ||
| want: nil, | ||
| wantErr: false, | ||
| }, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| s := Scanner{} | ||
| got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) | ||
| if (err != nil) != tt.wantErr { | ||
| t.Errorf("Confluent.FromData() error = %v, wantErr %v", err, tt.wantErr) | ||
| return | ||
| } | ||
| for i := range got { | ||
| if len(got[i].Raw) == 0 { | ||
| t.Fatalf("no raw secret present: \n %+v", got[i]) | ||
| } | ||
| got[i].Raw = nil | ||
| } | ||
| if diff := pretty.Compare(got, tt.want); diff != "" { | ||
| t.Errorf("Confluent.FromData() %s diff: (-got +want)\n%s", tt.name, diff) | ||
| } | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func BenchmarkFromData(benchmark *testing.B) { | ||
| ctx := context.Background() | ||
| s := Scanner{} | ||
| for name, data := range detectors.MustGetBenchmarkData() { | ||
| benchmark.Run(name, func(b *testing.B) { | ||
| b.ResetTimer() | ||
| for n := 0; n < b.N; n++ { | ||
| _, err := s.FromData(ctx, false, data) | ||
| if err != nil { | ||
| b.Fatal(err) | ||
| } | ||
| } | ||
| }) | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| package confluent | ||
|
|
||
| import ( | ||
| "context" | ||
| "testing" | ||
|
|
||
| "github.com/google/go-cmp/cmp" | ||
|
|
||
| "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" | ||
| "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" | ||
| ) | ||
|
|
||
| var ( | ||
| validPattern = ` | ||
| === Confluent Cloud API key === | ||
|
|
||
| API key: | ||
| JSAOOCIC74SGECCP | ||
|
|
||
| API secret: | ||
| cfltT8d8RzkNseTMEDKcjNM1BZTFPHqRn/dQm9q7w6SjzZ12wZfwjaJdipHZtDjw | ||
|
|
||
| Resource scope: | ||
| Cloud resource management | ||
|
|
||
| ` | ||
| secret = "JSAOOCIC74SGECCPcfltT8d8RzkNseTMEDKcjNM1BZTFPHqRn/dQm9q7w6SjzZ12wZfwjaJdipHZtDjw" | ||
| ) | ||
|
|
||
| func TestConfluent_Pattern(t *testing.T) { | ||
| d := Scanner{} | ||
| ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) | ||
|
|
||
| tests := []struct { | ||
| name string | ||
| input string | ||
| want []string | ||
| }{ | ||
| { | ||
| name: "valid pattern", | ||
| input: validPattern, | ||
| want: []string{secret}, | ||
| }, | ||
| } | ||
|
|
||
| for _, test := range tests { | ||
| t.Run(test.name, func(t *testing.T) { | ||
| matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) | ||
| if len(matchedDetectors) == 0 { | ||
| t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) | ||
| return | ||
| } | ||
|
|
||
| results, err := d.FromData(context.Background(), false, []byte(test.input)) | ||
| if err != nil { | ||
| t.Errorf("error = %v", err) | ||
| return | ||
| } | ||
|
|
||
| if len(results) != len(test.want) { | ||
| if len(results) == 0 { | ||
| t.Errorf("did not receive result") | ||
| } else { | ||
| t.Errorf("expected %d results, only received %d", len(test.want), len(results)) | ||
| } | ||
| return | ||
| } | ||
|
|
||
| actual := make(map[string]struct{}, len(results)) | ||
| for _, r := range results { | ||
| if len(r.RawV2) > 0 { | ||
| actual[string(r.RawV2)] = struct{}{} | ||
| } else { | ||
| actual[string(r.Raw)] = struct{}{} | ||
| } | ||
| } | ||
| expected := make(map[string]struct{}, len(test.want)) | ||
| for _, v := range test.want { | ||
| expected[v] = struct{}{} | ||
| } | ||
|
|
||
| if diff := cmp.Diff(expected, actual); diff != "" { | ||
| t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) | ||
| } | ||
| }) | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
According to this, we must send the API key in an Authorization: Basic {credentials} header, which requires us to provide the credentials as the API key ID and associated API secret separated by a colon and encoded using Base64 format.
So my question is, how useful is this API secret alone without an API key ID? I get that we can detect and potentially verify the API secret using your new implementation, but can an API secret alone be exploited if leaked?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
good point ! API secret cannot be exploited without the corresponding API key.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
added API key detection in V2, similar to V1. Sorry about that.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks for the update. I am not saying this is wrong but I do have some concerns regarding the way we are verifying the secrets. Right now, we are detecting keys and secrets, but we are verifying only secrets, instead of verifying a pair of key and secret.
For example, I copied the secret you have added in the test file and pasted in a file, and added a fake api key "0123456789ABCDEF", and ran a scan and the detector is showing this finding as a verified secret. However, this is a fake pair of secret and if this pair of API key and secret is leaked, they are worthless(correct me if I am wrong here).
So maybe we should switch to the API-based verification. I am open to suggestions here but just giving you the idea of how we detect and verify secrets in the detectors.
I am attaching the screenshot of the scan result where the incorrect API key is shown as verified.
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There is no way to accurately verify confluent secrets via API (for both V1 and V2) . V2 secrets don't need API verification since they have CRC32 checksum appended. But, I have added API key + secret detection for V2 (instead of just secrets detection). More context in the issue : #4490
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks for adding the API Key ID detection, however, I do not see the API Key ID actually being used in the verification part. The reason I asked for the addition of API Key ID here was because in order to verify a confluent API Key, we need both the API key ID and the associated API Secret as mentioned in the docs.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Here is another example. I have created the following secret using a Python script:
Both of these are fake, yet they were able to bypass the checksum-based validation, and the detector marked them as a verified result.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hi @shahzadhaider1 , The test secret you shared in the last comment has a valid checksum. That is why it's passing the check. However, I think we can rely on this checksum based mechanism because of the following reasons :
\bwhich will ensure that the key pattern only matches complete words and not random characters.I think the above checks are sufficient to eliminate FPs in real world. Please let me know your thoughts. Thanks !
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@shahzadhaider1 please let me know if you agree my last comment in this thread. Thanks !
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
after some internal discussion, I have decided to convert this to a draft. we are thinking about how we can accurately perform validation on different types of API keys. Thanks a lot for your inputs.