Add submodule indexing without repo cache

When not using repo cache, index git submodules recursively into

subrepos using the go-git API

Author: H3mul

gitindex/index.go

@@ -831,10 +831,13 @@ func prepareDeltaBuild(options Options, repository *git.Repository) (repos map[f
 
 func prepareNormalBuild(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, err error) {
 	var repoCache *RepoCache
-	if options.Submodules {
+	if options.Submodules && options.RepoCacheDir != "" {
 		repoCache = NewRepoCache(options.RepoCacheDir)
 	}
+	return prepareNormalBuildRecurse(options, repository, repoCache, false)
+}
 
+func prepareNormalBuildRecurse(options Options, repository *git.Repository, repoCache *RepoCache, isSubrepo bool) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, err error) {
 	// Branch => Repo => SHA1
 	branchVersions = map[string]map[string]plumbing.Hash{}
 
@@ -843,7 +846,22 @@ func prepareNormalBuild(options Options, repository *git.Repository) (repos map[
 		return nil, nil, fmt.Errorf("expandBranches: %w", err)
 	}
 
-	rw := NewRepoWalker(repository, options.BuildOptions.RepositoryDescription.URL, repoCache)
+	repoURL := options.BuildOptions.RepositoryDescription.URL
+
+	if isSubrepo {
+		cfg, err := repository.Config()
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to get repository config: %w", err)
+		}
+
+		u, err := normalizeSubmoduleRemoteURL(cfg)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to identify subrepository URL: %w", err)
+		}
+		repoURL = u.String()
+	}
+
+	rw := NewRepoWalker(repository, repoURL, repoCache)
 	for _, b := range branches {
 		commit, err := getCommit(repository, options.BranchPrefix, b)
 		if err != nil {
@@ -872,6 +890,47 @@ func prepareNormalBuild(options Options, repository *git.Repository) (repos map[
 		branchVersions[b] = subVersions
 	}
 
+	// Index submodules using go-git if we didn't do so using the repo cache
+	if options.Submodules && options.RepoCacheDir == "" {
+		worktree, err := repository.Worktree()
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to get repository worktree: %w", err)
+		}
+
+		submodules, err := worktree.Submodules()
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to get submodules: %w", err)
+		}
+
+		for _, submodule := range submodules {
+			subRepository, err := submodule.Repository()
+			if err != nil {
+				log.Printf("failed to open submodule repository: %s, %s", submodule.Config().Name, err)
+				continue
+			}
+
+			sw, subVersions, err := prepareNormalBuildRecurse(options, subRepository, repoCache, true)
+			if err != nil {
+				log.Printf("failed to index submodule repository: %s, %s", submodule.Config().Name, err)
+				continue
+			}
+
+			log.Printf("adding subrepository files from: %s", submodule.Config().Name)
+
+			for k, repo := range sw {
+				rw.Files[fileKey{
+					SubRepoPath: filepath.Join(submodule.Config().Path, k.SubRepoPath),
+					Path:        k.Path,
+					ID:          k.ID,
+				}] = repo
+			}
+
+			for k, v := range subVersions {
+				branchVersions[filepath.Join(submodule.Config().Path, k)] = v
+			}
+		}
+	}
+
 	return rw.Files, branchVersions, nil
 }
 

gitindex/tree.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/filemode"
 	"github.com/go-git/go-git/v5/plumbing/object"
@@ -93,6 +94,39 @@ func (rw *RepoWalker) parseModuleMap(t *object.Tree) error {
 	return nil
 }
 
+// This attempts to get a repo URL similar to the main repository template processing as in setTemplatesFromConfig()
+func normalizeSubmoduleRemoteURL(cfg *config.Config) (*url.URL, error) {
+	sec := cfg.Raw.Section("zoekt")
+	remoteURL := sec.Options.Get("web-url")
+	if remoteURL == "" {
+		// fall back to "origin" remote
+		remoteURL = configLookupRemoteURL(cfg, "origin")
+		if remoteURL == "" {
+			return nil, fmt.Errorf("no remote URL found in git config")
+		}
+	}
+
+	if sm := sshRelativeURLRegexp.FindStringSubmatch(remoteURL); sm != nil {
+		user := sm[1]
+		host := sm[2]
+		path := sm[3]
+
+		remoteURL = fmt.Sprintf("ssh+git://%s@%s/%s", user, host, path)
+	}
+
+	u, err := url.Parse(remoteURL)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse remote URL %q: %w", remoteURL, err)
+	}
+
+	if u.Scheme == "ssh+git" {
+		u.Scheme = "https"
+		u.User = nil
+	}
+
+	return u, nil
+}
+
 // CollectFiles fetches the blob SHA1s for the tree. If repoCache is
 // non-nil, recurse into submodules. In addition, it returns a mapping
 // that indicates in which repo each SHA1 can be found.
@@ -159,6 +193,14 @@ func (rw *RepoWalker) handleSubmodule(p string, id *plumbing.Hash, branch string
 
 	subRepoVersions[p] = *id
 
+	cfg, err := subRepo.Config()
+	if err == nil {
+		subRemoteURL, err := normalizeSubmoduleRemoteURL(cfg)
+		if err == nil {
+			subURL = subRemoteURL
+		}
+	}
+
 	sw := NewRepoWalker(subRepo, subURL.String(), rw.repoCache)
 	subVersions, err := sw.CollectFiles(tree, branch, ig)
 	if err != nil {
@@ -178,9 +220,16 @@ func (rw *RepoWalker) handleSubmodule(p string, id *plumbing.Hash, branch string
 }
 
 func (rw *RepoWalker) handleEntry(p string, e *object.TreeEntry, branch string, subRepoVersions map[string]plumbing.Hash, ig *ignore.Matcher) error {
-	if e.Mode == filemode.Submodule && rw.repoCache != nil {
-		if err := rw.tryHandleSubmodule(p, &e.Hash, branch, subRepoVersions, ig); err != nil {
-			return fmt.Errorf("submodule %s: %v", p, err)
+	if e.Mode == filemode.Submodule {
+		if rw.repoCache != nil {
+			// Index the submodule using repo cache
+			if err := rw.tryHandleSubmodule(p, &e.Hash, branch, subRepoVersions, ig); err != nil {
+				return fmt.Errorf("submodule %s: %v", p, err)
+			}
+		} else {
+			// Record the commit ID for the submodule path
+			// This will be the submodule's commit hash, not the parent's
+			subRepoVersions[p] = e.Hash
 		}
 	}