copy languages package from Sourcegraph to Zoekt (#979)

We want Zoekt and Sourcegraph to use the same language package. In this PR we move the languages package from Sourcegraph to Zoekt, so that Zoekt can use it and Sourcegraph can import it.

Notes: 
- Zoekt doesn't need to fetch content async which is why I added a little helper func `GetLanguagesFromContent` to make the call sites in Zoekt less awkward.
- Sourcegraph's languages package always classified .cls files as Apex, while Zoekt did a content based check. With this PR we follow Zoekt's approach. Specifically, I removed .cls from `unsupportedByEnryExtensionToNameMap`. I added an additional unit test to cover this case.

Test plan:
I appended the test cases from the old Zoekt languages packages to the tests I copied over from Sourcegraph

Author: stefanhengl

go.mod

@@ -54,6 +54,7 @@ require (
 	golang.org/x/sys v0.30.0
 	google.golang.org/grpc v1.69.4
 	google.golang.org/protobuf v1.36.3
+	pgregory.net/rapid v1.2.0
 )
 
 require (

go.sum

@@ -576,4 +576,6 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM=
 mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ=
+pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
+pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

index/shard_builder.go

@@ -23,16 +23,15 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strings"
 	"text/template"
 	"time"
 	"unicode/utf8"
 
-	"slices"
-
 	"github.com/sourcegraph/zoekt"
-	"github.com/sourcegraph/zoekt/internal/languages"
+	"github.com/sourcegraph/zoekt/languages"
 )
 
 var _ = log.Println
@@ -404,13 +403,18 @@ func DetermineLanguageIfUnknown(doc *Document) {
 		return
 	}
 
-	if doc.SkipReason != SkipReasonNone {
-		// If this document has been skipped, it's likely very large, or it's a non-code file like binary.
-		// In this case, we just guess the language based on file name to avoid examining the contents.
-		// Note: passing nil content is allowed by the go-enry contract (the underlying library we use here).
-		doc.Language = languages.GetLanguage(doc.Name, nil)
-	} else {
-		doc.Language = languages.GetLanguage(doc.Name, doc.Content)
+	// If this document has been skipped (doc.SkipReason != SkipReasonNone), it's
+	// likely very large, or it's a non-code file like binary. In this case, we just
+	// guess the language based on the file name to avoid examining the contents.
+	// Note: passing nil content is allowed by the go-enry contract (the underlying
+	// library we use here).
+	var content []byte
+	if doc.SkipReason == SkipReasonNone {
+		content = doc.Content
+	}
+	langs := languages.GetLanguagesFromContent(doc.Name, content)
+	if len(langs) > 0 {
+		doc.Language = langs[0]
 	}
 }
 

internal/languages/language.go

@@ -0,0 +1,15 @@
+package languages
+
+import "strings"
+
+// This file contains private functions
+// vendored from the go-enry codebase.
+
+// convertToAliasKey is vendored from go-enry to make sure
+// we're normalizing strings the same way.
+func convertToAliasKey(langName string) string {
+	ak := strings.SplitN(langName, `,`, 2)[0]
+	ak = strings.Replace(ak, ` `, `_`, -1)
+	ak = strings.ToLower(ak)
+	return ak
+}