diff --git a/pkg/detectors/azure_entra/serviceprincipal/v1/spv1.go b/pkg/detectors/azure_entra/serviceprincipal/v1/spv1.go index 8cced3f236a3..ae7b794ff8eb 100644 --- a/pkg/detectors/azure_entra/serviceprincipal/v1/spv1.go +++ b/pkg/detectors/azure_entra/serviceprincipal/v1/spv1.go @@ -25,15 +25,13 @@ var _ interface { detectors.Versioner } = (*Scanner)(nil) -var ( - defaultClient = common.SaneHttpClient() - // TODO: Azure storage access keys and investigate other types of creds. - // https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#second-case-access-token-request-with-a-certificate - // https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#third-case-access-token-request-with-a-federated-credential - //clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}?([\w~@[\]:.?*/+=-]{31,34}`) - // TODO: Tighten this regex and replace it with above. - secretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]([A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]{31,34})[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]`) -) +func (s Scanner) Type() detectorspb.DetectorType { + return detectorspb.DetectorType_Azure +} + +func (s Scanner) Description() string { + return serviceprincipal.Description +} func (s Scanner) Version() int { return 1 @@ -45,6 +43,19 @@ func (s Scanner) Keywords() []string { return []string{"azure", "az", "entra", "msal", "login.microsoftonline.com", ".onmicrosoft.com"} } +var ( + defaultClient = common.SaneHttpClient() + // TODO: Azure storage access keys and investigate other types of creds. + // https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#second-case-access-token-request-with-a-certificate + // https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#third-case-access-token-request-with-a-federated-credential + // clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}?([\w~@[\]:.?*/+=-]{31,34}`) + // TODO: Tighten this regex and replace it with above. + secretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]([A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]{31,34})(?:[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]|\z)`) + + invalidMatchPat = regexp.MustCompile(`^passwordCredentials":`) + invalidSecretPat = regexp.MustCompile(`^[a-zA-Z]+$`) +) + // FromData will find and optionally verify Azure secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -68,20 +79,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } -func (s Scanner) Type() detectorspb.DetectorType { - return detectorspb.DetectorType_Azure -} - -func (s Scanner) Description() string { - return serviceprincipal.Description -} - func findSecretMatches(data string) map[string]struct{} { uniqueMatches := make(map[string]struct{}) for _, match := range secretPat.FindAllStringSubmatch(data, -1) { m := match[1] - // Ignore secrets that are handled by the V2 detector. if v2.SecretPat.MatchString(m) { + // Ignore secrets that are handled by the V2 detector. + continue + } else if detectors.StringShannonEntropy(m) < 3 { + // Ignore low-entropy results. + continue + } else if invalidSecretPat.MatchString(m) || invalidMatchPat.MatchString(match[0]) { + // Ignore patterns that are known to be false. continue } uniqueMatches[m] = struct{}{} diff --git a/pkg/detectors/azure_entra/serviceprincipal/v1/spv1_test.go b/pkg/detectors/azure_entra/serviceprincipal/v1/spv1_test.go index 56861fc9b8c7..3231b0778e35 100644 --- a/pkg/detectors/azure_entra/serviceprincipal/v1/spv1_test.go +++ b/pkg/detectors/azure_entra/serviceprincipal/v1/spv1_test.go @@ -1,94 +1,11 @@ package v1 import ( - "context" "testing" "github.com/google/go-cmp/cmp" - - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" -) - -var ( - validPattern = ` - azure credentials: - azureClientID: clientid9304d5df4-aac1-6117-552c-7f70c89a40d9 - azureTenant: tenant_idid9304d5df4-aac1-6117-552c-7f70c89a40d9 - azureClientSecret: clientsecretY_0w|[cGpan41k6ng.ol414sp4ccw2v_rkfmbs537i - ` - invalidPattern = ` - azure credentials: - azureClientID: 9304d5df4-aac1-6117-552c-7f70c89a - azureTenant: id9304d5df4-aac1-6117-55-7f70c89a40d9 - azureClientSecret: Y_0w|[cGpan41k6ng. - ` ) -func TestAzure_Pattern(t *testing.T) { - d := Scanner{} - ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - - tests := []struct { - name string - input string - want []string - }{ - { - name: "valid pattern", - input: validPattern, - want: []string{"304d5df4-aac1-6117-552c-7f70c89a40d9cGpan41k6ng.ol414sp4ccw2v_rkfmbs53304d5df4-aac1-6117-552c-7f70c89a40d9"}, - }, - { - name: "invalid pattern", - input: invalidPattern, - want: nil, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) - if len(matchedDetectors) == 0 { - t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) - return - } - - results, err := d.FromData(context.Background(), false, []byte(test.input)) - if err != nil { - t.Errorf("error = %v", err) - return - } - - if len(results) != len(test.want) { - if len(results) == 0 { - t.Errorf("did not receive result") - } else { - t.Errorf("expected %d results, only received %d", len(test.want), len(results)) - } - return - } - - actual := make(map[string]struct{}, len(results)) - for _, r := range results { - if len(r.RawV2) > 0 { - actual[string(r.RawV2)] = struct{}{} - } else { - actual[string(r.Raw)] = struct{}{} - } - } - expected := make(map[string]struct{}, len(test.want)) - for _, v := range test.want { - expected[v] = struct{}{} - } - - if diff := cmp.Diff(expected, actual); diff != "" { - t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) - } - }) - } -} - type testCase struct { Input string Expected map[string]struct{} @@ -96,11 +13,24 @@ type testCase struct { func Test_FindClientSecretMatches(t *testing.T) { cases := map[string]testCase{ + // secret + `secret`: { + Input: `"secret": "ljjK-62Q5bJbm43xU5At-NdeWDrhIO_28~",`, + Expected: map[string]struct{}{"ljjK-62Q5bJbm43xU5At-NdeWDrhIO_28~": {}}, + }, + + // client secret "client_secret": { Input: ` "TenantId": "3d7e0652-b03d-4ed2-bf86-f1299cecde17", "ClientSecret": "gHduiL_j6t4b6DG?Qr-G6M@IOS?mX3B9",`, Expected: map[string]struct{}{"gHduiL_j6t4b6DG?Qr-G6M@IOS?mX3B9": {}}, }, + "client secret at end": { + Input: `secret: UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==`, + Expected: map[string]struct{}{ + "UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==": {}, + }, + }, "client_secret1": { Input: ` public static string clientId = "413ff05b-6d54-41a7-9271-9f964bc10624"; public static string clientSecret = "k72~odcN_6TbVh5D~19_1Qkj~87trteArL"; @@ -155,10 +85,19 @@ configs = {"fs.azure.account.auth.type": "OAuth"`, Input: ` "AZUREAD-AKS-APPID-SECRET": "8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-",`, Expected: map[string]struct{}{"8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-": {}}, }, - //"client_secret6": { + "client_secret9": { + Input: ` client-id: 49abd816-45d1-479a-b49a-80bcf6d7213a + client-secret: 7.18gt1b2wO-t.~Cf.mlZCyHC7r_micnuO`, + Expected: map[string]struct{}{"7.18gt1b2wO-t.~Cf.mlZCyHC7r_micnuO": {}}, + }, + "client_secret10": { + Input: ` "aadClientSecret": "6p3t93TJzPgsNtQISqWc.-@?GCz9-ZWo",`, + Expected: map[string]struct{}{"6p3t93TJzPgsNtQISqWc.-@?GCz9-ZWo": {}}, + }, + // "client_secret6": { // Input: ``, // Expected: map[string]struct{}{"": {}}, - //}, + // }, "password": { Input: `# Login using Service Principal @@ -169,18 +108,29 @@ $Credential = New-Object -TypeName System.Management.Automation.PSCredential -Ar }, // False positives - "placeholder_secret": { + "invalid - placeholder_secret": { Input: `- Log in with a service principal using a client secret: az login --service-principal --username {{http://azure-cli-service-principal}} --password {{secret}} --tenant {{someone.onmicrosoft.com}}`, - Expected: nil, - }, - //"client_secret3": { - // Input: ``, - // Expected: map[string]struct{}{ - // "": {}, - // }, - //}, + }, + "invalid - only alpha characters": { + Input: `"passwordCredentials":[],"preferredTokenSigningKeyThumbprint":null,"publisherName":"Microsoft"`, + }, + "invalid - low entropy": { + Input: `clientSecret: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`, + }, + "invalid - passwordCredentials1": { + Input: `"passwordCredentials":[{"customKeyIdentifier":"UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==","endDate":"2019-07-16T23:01:19.028Z","keyId":`, + }, + "invalid - passwordCredentials2": { + Input: `"passwordCredentials":[{"customKeyIdentifier":"TQB5ACAARgBpAHIAcwB0ACAASwBlAHkA"`, + }, + "invalid - passwordCredentials3": { + Input: `,"passwordCredentials":[{"customKeyIdentifier":"awBlAHkAZgBvAHIAaQBtAHAAYQBsAGEA",`, + }, + "invalid - azure vault path": { + Input: ` public const string MsalArlingtonOBOKeyVaultUri = "https://msidlabs.vault.azure.net:443/secrets/ARLMSIDLAB1-IDLASBS-App-CC-Secret";`, + }, } for name, test := range cases { diff --git a/pkg/detectors/azure_entra/serviceprincipal/v2/spv2.go b/pkg/detectors/azure_entra/serviceprincipal/v2/spv2.go index 3b09b0fca0f4..3fa2da4cc2bd 100644 --- a/pkg/detectors/azure_entra/serviceprincipal/v2/spv2.go +++ b/pkg/detectors/azure_entra/serviceprincipal/v2/spv2.go @@ -26,11 +26,13 @@ var _ interface { detectors.Versioner } = (*Scanner)(nil) -var ( - defaultClient = common.SaneHttpClient() +func (s Scanner) Type() detectorspb.DetectorType { + return detectorspb.DetectorType_Azure +} - SecretPat = regexp.MustCompile(`(?:[^a-zA-Z0-9_~.-]|\A)([a-zA-Z0-9_~.-]{3}\dQ~[a-zA-Z0-9_~.-]{31,34})(?:[^a-zA-Z0-9_~.-]|\z)`) -) +func (s Scanner) Description() string { + return serviceprincipal.Description +} func (s Scanner) Version() int { return 2 @@ -42,13 +44,11 @@ func (s Scanner) Keywords() []string { return []string{"q~"} } -func (s Scanner) Type() detectorspb.DetectorType { - return detectorspb.DetectorType_Azure -} +var ( + defaultClient = common.SaneHttpClient() -func (s Scanner) Description() string { - return serviceprincipal.Description -} + SecretPat = regexp.MustCompile(`(?:[^a-zA-Z0-9_~.-]|\A)([a-zA-Z0-9_~.-]{3}\dQ~[a-zA-Z0-9_~.-]{31,34})(?:[^a-zA-Z0-9_~.-]|\z)`) +) // FromData will find and optionally verify Azure secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { @@ -181,7 +181,11 @@ func createResult(tenantId string, clientId string, clientSecret string, verifie func findSecretMatches(data string) map[string]struct{} { uniqueMatches := make(map[string]struct{}) for _, match := range SecretPat.FindAllStringSubmatch(data, -1) { - uniqueMatches[match[1]] = struct{}{} + m := match[1] + if detectors.StringShannonEntropy(m) < 3 { + continue + } + uniqueMatches[m] = struct{}{} } return uniqueMatches } diff --git a/pkg/detectors/azure_entra/serviceprincipal/v2/spv2_test.go b/pkg/detectors/azure_entra/serviceprincipal/v2/spv2_test.go index 68eb128368b1..fc7ec3f644b1 100644 --- a/pkg/detectors/azure_entra/serviceprincipal/v2/spv2_test.go +++ b/pkg/detectors/azure_entra/serviceprincipal/v2/spv2_test.go @@ -1,11 +1,99 @@ package v2 import ( + "context" "testing" "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) +func TestAzure_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + // Valid + { + name: "valid - single secret, client, tenant", + input: `ClientID - 9794fe8b-1ff6-4cf6-b28c-72c8fb124942 +Client Secret- nfu7Q~XRIzdfTQS4QN_ABnmQKg4dPA10~5lbocIl +Tenant ID - d4a48591-844d-44a2-8a84-9c94028bdfab`, + want: []string{`{"clientSecret":"nfu7Q~XRIzdfTQS4QN_ABnmQKg4dPA10~5lbocIl","clientId":"9794fe8b-1ff6-4cf6-b28c-72c8fb124942","tenantId":"d4a48591-844d-44a2-8a84-9c94028bdfab"}`}, + }, + { + name: "valid - single secret, multiple client/tenant", + input: ` +cas.authn.azure-active-directory.client-id=5b82d177-f2ee-461b-a1f6-0624fff3caf0, +#cas.authn.azure-active-directory.client-id=51b65b04-5658-49e0-9955-f1705935bf0a, +cas.authn.azure-active-directory.login-url=https://login.microsoftonline.com/common/, +cas.authn.azure-active-directory.tenant=19653e91-7a9a-4bd6-8752-3070fc17e9e7, +#cas.authn.azure-active-directory.tenant=9b5eb0ce-7b2c-4f8d-8542-6248ee2c6525, +cas.authn.azure-active-directory.client-secret=pe48Q~~WtAjXI8HronCfgvzgHPfMGWjn4Hy4vcgC, +`, + want: []string{"pe48Q~~WtAjXI8HronCfgvzgHPfMGWjn4Hy4vcgC"}, + }, + + // Invalid + { + name: "invalid - low entropy", + input: ` +tenant_id = "1821c750-3a5f-4255-88ad-e24b7a1564c1" +client_id = "4e14d6ff-c99b-4d10-8491-00f731747898" +client_secret = "bP88Q~xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"`, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} + type testCase struct { Input string Expected map[string]struct{} @@ -13,6 +101,7 @@ type testCase struct { func Test_FindClientSecretMatches(t *testing.T) { cases := map[string]testCase{ + // Valid "secret": { Input: `servicePrincipal: tenantId: "608e4ac4-2ca8-40dd-a046-4064540a1cde" @@ -50,6 +139,12 @@ OPENID_GRANT_TYPE=client_credentials`, "-6s8Q~.Q9CKMOXHGs_BA3ig2wUzyDRyulhWEOc3u": {}, }, }, + + // Invalid + "invalid - low entropy": { + Input: `CLIENT_SECRET = 'USe8Q~xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'`, + Expected: nil, + }, } for name, test := range cases {