Skip to content

Commit c8262b8

Browse files
KeinkilKeinkil
authored
Add support for filtering out private repositories (#991)
1 parent 46f7ba3 commit c8262b8

File tree

4 files changed

+22
-13
lines changed

4 files changed

+22
-13
lines changed

‎cmd/zoekt-indexserver/config.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ type ConfigEntry struct {
5858
GerritRepoNameFormat string
5959
ExcludeUserRepos bool
6060
Forks bool
61+
Visibility []string
6162
}
6263

6364
func randomize(entries []ConfigEntry) []ConfigEntry {
@@ -207,6 +208,9 @@ func executeMirror(cfg []ConfigEntry, repoDir string, pendingRepos chan<- string
207208
if c.Forks {
208209
cmd.Args = append(cmd.Args, "-forks")
209210
}
211+
for _, v := range c.Visibility {
212+
cmd.Args = append(cmd.Args, "-visibility", v)
213+
}
210214
} else if c.GitilesURL != "" {
211215
cmd = exec.Command("zoekt-mirror-gitiles",
212216
"-dest", repoDir, "-name", c.Name)

‎cmd/zoekt-mirror-github/main.go‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import (
3030
"strconv"
3131
"strings"
3232

33-
"github.com/google/go-github/v27/github"
33+
"github.com/google/go-github/v78/github"
3434
"golang.org/x/oauth2"
3535

3636
"github.com/sourcegraph/zoekt/gitindex"
@@ -51,6 +51,7 @@ type reposFilters struct {
5151
topics []string
5252
excludeTopics []string
5353
noArchived *bool
54+
visibility []string
5455
}
5556

5657
func main() {
@@ -68,6 +69,8 @@ func main() {
6869
excludeTopics := topicsFlag{}
6970
flag.Var(&excludeTopics, "exclude_topic", "don't clone repos whose have one of given topics. You can add multiple topics by setting this more than once.")
7071
noArchived := flag.Bool("no_archived", false, "mirror only projects that are not archived")
72+
visibility := topicsFlag{}
73+
flag.Var(&visibility, "visibility", "filter repos by visibility (public, private, internal). You can add multiple values by setting this more than once.")
7174

7275
flag.Parse()
7376

@@ -109,6 +112,7 @@ func main() {
109112
topics: topics,
110113
excludeTopics: excludeTopics,
111114
noArchived: noArchived,
115+
visibility: visibility,
112116
}
113117
var repos []*github.Repository
114118
var err error
@@ -228,11 +232,14 @@ func hasIntersection(s1, s2 []string) bool {
228232
return false
229233
}
230234

231-
func filterRepositories(repos []*github.Repository, include []string, exclude []string, noArchived bool) (filteredRepos []*github.Repository) {
235+
func filterRepositories(repos []*github.Repository, include []string, exclude []string, noArchived bool, visibility []string) (filteredRepos []*github.Repository) {
232236
for _, repo := range repos {
233237
if noArchived && *repo.Archived {
234238
continue
235239
}
240+
if len(visibility) > 0 && !hasIntersection(visibility, []string{repo.GetVisibility()}) {
241+
continue
242+
}
236243
if (len(include) == 0 || hasIntersection(include, repo.Topics)) &&
237244
!hasIntersection(exclude, repo.Topics) {
238245
filteredRepos = append(filteredRepos, repo)
@@ -254,7 +261,7 @@ func getOrgRepos(client *github.Client, org string, reposFilters reposFilters) (
254261
}
255262

256263
opt.Page = resp.NextPage
257-
repos = filterRepositories(repos, reposFilters.topics, reposFilters.excludeTopics, *reposFilters.noArchived)
264+
repos = filterRepositories(repos, reposFilters.topics, reposFilters.excludeTopics, *reposFilters.noArchived, reposFilters.visibility)
258265
allRepos = append(allRepos, repos...)
259266
if resp.NextPage == 0 {
260267
break
@@ -276,7 +283,7 @@ func getUserRepos(client *github.Client, user string, reposFilters reposFilters)
276283
}
277284

278285
opt.Page = resp.NextPage
279-
repos = filterRepositories(repos, reposFilters.topics, reposFilters.excludeTopics, *reposFilters.noArchived)
286+
repos = filterRepositories(repos, reposFilters.topics, reposFilters.excludeTopics, *reposFilters.noArchived, reposFilters.visibility)
280287
allRepos = append(allRepos, repos...)
281288
if resp.NextPage == 0 {
282289
break

‎go.mod‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ require (
1313
github.com/go-enry/go-enry/v2 v2.9.1
1414
github.com/go-git/go-git/v5 v5.13.1
1515
github.com/gobwas/glob v0.2.3
16-
github.com/google/go-cmp v0.6.0
17-
github.com/google/go-github/v27 v27.0.6
16+
github.com/google/go-cmp v0.7.0
17+
github.com/google/go-github/v78 v78.0.0
1818
github.com/google/slothfs v0.0.0-20190717100203-59c1163fd173
1919
github.com/grafana/regexp v0.0.0-20240607082908-2cb410fa05da
2020
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
@@ -145,4 +145,4 @@ require (
145145
gopkg.in/yaml.v3 v3.0.1 // indirect
146146
)
147147

148-
go 1.23.4
148+
go 1.24.0

‎go.sum‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -160,11 +160,10 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
160160
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
161161
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
162162
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
163-
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
164-
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
165-
github.com/google/go-github/v27 v27.0.6 h1:oiOZuBmGHvrGM1X9uNUAUlLgp5r1UUO/M/KnbHnLRlQ=
166-
github.com/google/go-github/v27 v27.0.6/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0=
167-
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
163+
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
164+
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
165+
github.com/google/go-github/v78 v78.0.0 h1:b1tytzFE8i//lRVDx5Qh/EdJbtTPtSVD3nF7hraEs9w=
166+
github.com/google/go-github/v78 v78.0.0/go.mod h1:Uxvdzy82AkNlC6JQ57se9TqvmgBT7RF0ouHDNg2jd6g=
168167
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
169168
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
170169
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
@@ -447,7 +446,6 @@ golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKl
447446
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
448447
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
449448
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
450-
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
451449
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
452450
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
453451
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

0 commit comments

Comments
 (0)