Merge branch 'main' into feature/ins-58

Author: kashifkhan0771

pkg/engine/github.go

@@ -20,7 +20,6 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) (source
 		Repositories:               c.Repos,
 		ScanUsers:                  c.IncludeMembers,
 		IgnoreRepos:                c.ExcludeRepos,
-		IncludeRepos:               c.IncludeRepos,
 		IncludeForks:               c.IncludeForks,
 		IncludeIssueComments:       c.IncludeIssueComments,
 		IncludePullRequestComments: c.IncludePullRequestComments,

pkg/pb/sourcespb/sources.pb.go

@@ -253,7 +253,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
 
 	s.filteredRepoCache = s.newFilteredRepoCache(aCtx,
 		simple.NewCache[string](),
-		append(s.conn.GetRepositories(), s.conn.GetIncludeRepos()...),
+		s.conn.GetRepositories(),
 		s.conn.GetIgnoreRepos(),
 	)
 	s.repos = s.conn.Repositories
@@ -433,33 +433,42 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e
 	case *unauthenticatedConnector:
 		s.enumerateUnauthenticated(ctx, dedupeReporter)
 	}
-	s.repos = make([]string, 0, s.filteredRepoCache.Count())
-
-	// Double make sure that all enumerated repositories in the
-	// filteredRepoCache have an entry in the repoInfoCache.
-	for _, repo := range s.filteredRepoCache.Values() {
-		// Extract the repository name from the URL for filtering
-		repoName := repo
-		if strings.Contains(repo, "/") {
-			// Try to extract org/repo name from URL
-			if strings.Contains(repo, "github.com") {
-				parts := strings.Split(repo, "/")
-				if len(parts) >= 2 {
-					repoName = parts[len(parts)-2] + "/" + strings.TrimSuffix(parts[len(parts)-1], ".git")
+	// If explicit repositories were provided, use them directly without filtering
+	// Otherwise, rebuild s.repos from the filteredRepoCache
+	if len(s.conn.Repositories) > 0 {
+		// Explicit repositories bypass filtering - use them as-is
+		s.repos = s.conn.Repositories
+		ctx.Logger().V(1).Info("Using explicit repositories", "count", len(s.repos))
+	} else {
+		// No explicit repositories - rebuild from enumerated cache with filtering
+		s.repos = make([]string, 0, s.filteredRepoCache.Count())
+
+		// Double make sure that all enumerated repositories in the
+		// filteredRepoCache have an entry in the repoInfoCache.
+		for _, repo := range s.filteredRepoCache.Values() {
+			// Extract the repository name from the URL for filtering
+			repoName := repo
+			if strings.Contains(repo, "/") {
+				// Try to extract org/repo name from URL
+				if strings.Contains(repo, "github.com") {
+					parts := strings.Split(repo, "/")
+					if len(parts) >= 2 {
+						repoName = parts[len(parts)-2] + "/" + strings.TrimSuffix(parts[len(parts)-1], ".git")
+					}
 				}
 			}
-		}
 
-		// Final filter check - only include repositories that pass the filter
-		if s.filteredRepoCache.wantRepo(repoName) {
-			ctx = context.WithValue(ctx, "repo", repo)
+			// Final filter check - only include repositories that pass the filter
+			if s.filteredRepoCache.wantRepo(repoName) {
+				ctx = context.WithValue(ctx, "repo", repo)
 
-			repo, err := s.ensureRepoInfoCache(ctx, repo, &unitErrorReporter{reporter})
-			if err != nil {
-				ctx.Logger().Error(err, "error caching repo info")
-				_ = dedupeReporter.UnitErr(ctx, fmt.Errorf("error caching repo info: %w", err))
+				repo, err := s.ensureRepoInfoCache(ctx, repo, &unitErrorReporter{reporter})
+				if err != nil {
+					ctx.Logger().Error(err, "error caching repo info")
+					_ = dedupeReporter.UnitErr(ctx, fmt.Errorf("error caching repo info: %w", err))
+				}
+				s.repos = append(s.repos, repo)
 			}
-			s.repos = append(s.repos, repo)
 		}
 	}
 	githubReposEnumerated.WithLabelValues(s.name).Set(float64(len(s.repos)))

pkg/sources/github/github.go

@@ -957,7 +957,7 @@ func TestSource_Validate(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			connector, err := newConnector(tt.sourceConfig)
+			connector, err := newConnector(ctx, tt.sourceConfig)
 			require.NoError(t, err)
 			tt.sourceConfig.connector = connector
 

pkg/sources/github/github_integration_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-github/v67/github"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/types/known/anypb"
 	"gopkg.in/h2non/gock.v1"
@@ -111,7 +112,7 @@ func TestAddReposByOrg(t *testing.T) {
 	assert.True(t, gock.IsDone())
 }
 
-func TestAddReposByOrg_IncludeRepos(t *testing.T) {
+func TestAddReposByOrg_Repositories(t *testing.T) {
 	defer gock.Off()
 
 	gock.New("https://api.github.com").
@@ -127,7 +128,7 @@ func TestAddReposByOrg_IncludeRepos(t *testing.T) {
 		Credential: &sourcespb.GitHub_Token{
 			Token: "super secret token",
 		},
-		IncludeRepos:  []string{"super-secret-org/super*"},
+		Repositories:  []string{"super-secret-org/super-secret-repo", "super-secret-org/super-secret-repo2"},
 		Organizations: []string{"super-secret-org"},
 	})
 	err := s.getReposByOrg(context.Background(), "super-secret-org", noopReporter())
@@ -748,7 +749,7 @@ func BenchmarkEnumerate(b *testing.B) {
 	}
 }
 
-func TestEnumerateWithToken_IncludeRepos(t *testing.T) {
+func TestEnumerateWithToken_Repositories(t *testing.T) {
 	defer gock.Off()
 
 	gock.New("https://api.github.com").
@@ -1046,6 +1047,55 @@ func TestRepositoryFiltering(t *testing.T) {
 	assert.False(t, cache5.wantRepo("other/repo1"))
 }
 
+func TestExplicitRepositoryBypass(t *testing.T) {
+	// Test that explicit repositories are included in enumeration
+	ctx := context.Background()
+
+	// Set up mocks for the API calls
+	gock.New("https://api.github.com").
+		Get("/user").
+		Reply(200).
+		JSON(map[string]string{"login": "test-user"})
+
+	gock.New("https://api.github.com").
+		Get("/repos/org/explicit-repo").
+		Reply(200).
+		JSON(map[string]any{
+			"full_name": "org/explicit-repo",
+			"clone_url": "https://github.com/org/explicit-repo.git",
+			"size":      1,
+		})
+
+	gock.New("https://api.github.com").
+		Get("/repos/org/another-explicit").
+		Reply(200).
+		JSON(map[string]any{
+			"full_name": "org/another-explicit",
+			"clone_url": "https://github.com/org/another-explicit.git",
+			"size":      1,
+		})
+
+	// Create a source with explicit repositories
+	source := initTestSource(&sourcespb.GitHub{
+		Credential: &sourcespb.GitHub_Token{
+			Token: "super secret token",
+		},
+		Repositories: []string{
+			"https://github.com/org/explicit-repo.git",
+			"https://github.com/org/another-explicit.git",
+		},
+	})
+
+	// Test the Enumerate method
+	err := source.Enumerate(ctx, noopReporter())
+	require.NoError(t, err)
+
+	// Verify that explicit repositories are included in the enumeration
+	assert.Len(t, source.repos, 2, "Should have 2 explicit repositories")
+	assert.Contains(t, source.repos, "https://github.com/org/explicit-repo.git")
+	assert.Contains(t, source.repos, "https://github.com/org/another-explicit.git")
+}
+
 func noopReporter() sources.UnitReporter {
 	return sources.VisitorReporter{
 		VisitUnit: func(context.Context, sources.SourceUnit) error {

pkg/sources/github/github_test.go

@@ -266,7 +266,10 @@ message GitHub {
   string head = 9;
   string base = 10;
   repeated string ignore_repos = 11;
-  repeated string include_repos = 12;
+  // DEPRECATED: include_repos is deprecated in favor of repositories. 
+  // It was used for filtering when the repositories field wasn't populating the UI-input results correctly
+  // It can be removed when we no longer depend on it
+  repeated string include_repos = 12 [deprecated = true];
   bool include_pull_request_comments = 14;
   bool include_issue_comments = 15;
   bool include_gist_comments = 16;