refactor: Migrate from MongoDB to sqlite

.gitignore

@@ -1,2 +1,4 @@
 # generate job temporary files
 .tmp
+
+db/vimcolorschemes.db

cli/generate.go

@@ -1,21 +1,35 @@
 package cli
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"log"
 	"os"
 	"os/exec"
 	"strings"
-	"time"
 
 	"github.com/vimcolorschemes/worker/internal/database"
 	file "github.com/vimcolorschemes/worker/internal/file"
-	repoHelper "github.com/vimcolorschemes/worker/internal/repository"
-
-	"go.mongodb.org/mongo-driver/bson"
+	"github.com/vimcolorschemes/worker/internal/store"
 )
 
+type ColorschemeData struct {
+	Light []store.ColorschemeGroupDefinition
+	Dark  []store.ColorschemeGroupDefinition
+}
+
+type ColorschemeGroupDefinition struct {
+	Name    string
+	HexCode string
+}
+
+var colorschemeVariantStore *store.ColorschemeVariantStore
+
+func init() {
+	colorschemeVariantStore = store.NewColorschemeVariantStore(database.Connect())
+}
+
 var tmpDirectoryPath string
 var packDirectoryPath string
 var vimrcPath string
@@ -24,7 +38,7 @@ var colorDataFilePath string
 var debugMode bool
 
 // Generate vim color scheme data for all valid repositories
-func Generate(force bool, debug bool, repoKey string) bson.M {
+func Generate(force bool, debug bool, repoKey string) database.JSONB {
 	debugMode = debug
 
 	initVimFiles()
@@ -33,83 +47,78 @@ func Generate(force bool, debug bool, repoKey string) bson.M {
 
 	fmt.Println()
 
-	var repositories []repoHelper.Repository
+	var repositories []store.Repository
 	if repoKey != "" {
-		repository, err := database.GetRepository(repoKey)
+		repository, err := repositoryStore.GetByKey(context.TODO(), repoKey)
 		if err != nil {
 			log.Panic(err)
 		}
-		repositories = []repoHelper.Repository{repository}
-	} else if force || debug {
-		repositories = database.GetRepositories()
+		repositories = []store.Repository{*repository}
 	} else {
-		repositories = database.GetRepositoriesToGenerate()
+		repositories = repositoryStore.GetAll()
 	}
 
 	log.Printf("Generating vim preview for %d repositories", len(repositories))
 
 	for index, repository := range repositories {
-		log.Print("\nGenerating vim previews for ", repository.Owner.Name, "/", repository.Name, " (", index+1, "/", len(repositories), ")")
+		log.Print("\nGenerating vim previews for ", repository.Owner, "/", repository.Name, " (", index+1, "/", len(repositories), ")")
 
-		key := fmt.Sprintf("%s__%s", repository.Owner.Name, repository.Name)
+		key := fmt.Sprintf("%s__%s", repository.Owner, repository.Name)
 		err := installPlugin(repository.GithubURL, key)
 		if err != nil {
 			log.Printf("Error installing plugin: %s", err)
-			repository.GenerateValid = false
-			updateRepositoryAfterGenerate(repository)
+			// repository.GenerateValid = false
+			// updateRepositoryAfterGenerate(repository)
 			continue
 		}
 
-		var data, dataError = getVimColorSchemeColorData()
+		var dataMap, dataError = getColorschemeDataMap()
 		err = deletePlugin(key)
 		if err != nil {
 			log.Printf("Error deleting plugin: %s", err)
 		}
 		if dataError != nil {
 			log.Printf("Error getting color data: %s", dataError)
-			repository.GenerateValid = false
-			updateRepositoryAfterGenerate(repository)
+			// repository.GenerateValid = false
+			// updateRepositoryAfterGenerate(repository)
 			continue
 		}
 
-		var vimColorSchemes []repoHelper.VimColorScheme
-
-		for name := range data {
+		for name := range dataMap {
 			if name == "default" || name == "module-injection" || name == "tick_tock" {
 				continue
 			}
 
-			var backgrounds []repoHelper.VimBackgroundValue
-			if data[name].Light != nil {
-				backgrounds = append(backgrounds, repoHelper.LightBackground)
-			}
-			if data[name].Dark != nil {
-				backgrounds = append(backgrounds, repoHelper.DarkBackground)
+			if dataMap[name].Light != nil {
+				err = colorschemeVariantStore.UpsertColorschemeVariant(context.TODO(), &store.ColorschemeVariant{
+					Name:         name,
+					RepositoryID: repository.ID,
+					Background:   store.BackgroundLight,
+					ColorData:    dataMap[name].Light,
+				})
+				if err != nil {
+					log.Printf("Error upserting colorscheme variant: %s", err)
+				}
 			}
 
-			vimColorSchemes = append(
-				vimColorSchemes,
-				repoHelper.VimColorScheme{
-					Name:        name,
-					Data:        data[name],
-					Backgrounds: backgrounds,
+			if dataMap[name].Dark != nil {
+				err = colorschemeVariantStore.UpsertColorschemeVariant(context.TODO(), &store.ColorschemeVariant{
+					Name:         name,
+					RepositoryID: repository.ID,
+					Background:   store.BackgroundDark,
+					ColorData:    dataMap[name].Dark,
 				})
+				if err != nil {
+					log.Printf("Error upserting colorscheme variant: %s", err)
+				}
+			}
 		}
-
-		repository.VimColorSchemes = vimColorSchemes
-		repository.GenerateValid = len(repository.VimColorSchemes) > 0
-		updateRepositoryAfterGenerate(repository)
 	}
 
 	cleanUp()
 
-	return bson.M{"repositoryCount": len(repositories)}
-}
-
-func updateRepositoryAfterGenerate(repository repoHelper.Repository) {
-	log.Printf("Generate valid: %v", repository.GenerateValid)
-	generateObject := getGenerateRepositoryObject(repository)
-	database.UpsertRepository(repository.ID, generateObject)
+	log.Printf("Generated %d repositories", len(repositories))
+	return database.JSONB{"count": len(repositories)}
 }
 
 // Initializes a temporary directory for vim configuration files
@@ -259,22 +268,23 @@ func deletePlugin(key string) error {
 	return err
 }
 
-// Gathers the colorscheme data from vimcolorschemes/extractor.nvim
-func getVimColorSchemeColorData() (map[string]repoHelper.VimColorSchemeData, error) {
+// Gathers the colorscheme data from vimcolorschemes/extractor.nvim mapped by
+// colorscheme name
+func getColorschemeDataMap() (map[string]ColorschemeData, error) {
 	err := executePreviewGenerator()
 	if err != nil {
 		log.Printf("Error executing nvim: %s", err)
 		return nil, err
 	}
 
-	vimColorSchemeOutput, err := file.GetLocalFileContent(colorDataFilePath)
+	colorschemeOutput, err := file.GetLocalFileContent(colorDataFilePath)
 	if err != nil {
 		log.Printf("Error getting local file content from \"%s\": %s", colorDataFilePath, err)
 		return nil, err
 	}
 
-	var data map[string]repoHelper.VimColorSchemeData
-	err = json.Unmarshal([]byte(vimColorSchemeOutput), &data)
+	var data map[string]ColorschemeData
+	err = json.Unmarshal([]byte(colorschemeOutput), &data)
 	if err != nil {
 		return nil, err
 	}
@@ -325,11 +335,3 @@ func cleanUp() {
 		log.Panic(err)
 	}
 }
-
-func getGenerateRepositoryObject(repository repoHelper.Repository) bson.M {
-	return bson.M{
-		"vimColorSchemes": repository.VimColorSchemes,
-		"generateValid":   repository.GenerateValid,
-		"generatedAt":     time.Now(),
-	}
-}

cli/import.go

@@ -1,21 +1,22 @@
 package cli
 
 import (
+	"context"
 	"log"
 	"math"
 	"strings"
 
 	"github.com/vimcolorschemes/worker/internal/database"
 	"github.com/vimcolorschemes/worker/internal/dotenv"
 	"github.com/vimcolorschemes/worker/internal/github"
-
-	"go.mongodb.org/mongo-driver/bson"
+	"github.com/vimcolorschemes/worker/internal/store"
 
 	gogithub "github.com/google/go-github/v68/github"
 )
 
 var repositoryCountLimit int
 var repositoryCountLimitPerPage int
+var repositoryStore *store.RepositoryStore
 
 var queries = []string{
 	"vim theme",
@@ -31,6 +32,8 @@ var queries = []string{
 }
 
 func init() {
+	repositoryStore = store.NewRepositoryStore(database.Connect())
+
 	repositoryCountLimitValue, err := dotenv.GetInt("GITHUB_REPOSITORY_COUNT_LIMIT")
 	if err != nil {
 		repositoryCountLimitValue = 100
@@ -41,7 +44,7 @@ func init() {
 }
 
 // Import potential vim color scheme repositories from Github
-func Import(_force bool, _debug bool, repoKey string) bson.M {
+func Import(_force bool, _debug bool, repoKey string) database.JSONB {
 	log.Printf("Repository limit: %d", repositoryCountLimit)
 
 	var repositories []*gogithub.Repository
@@ -62,22 +65,20 @@ func Import(_force bool, _debug bool, repoKey string) bson.M {
 	log.Print("Upserting ", len(repositories), " repositories")
 	for _, repository := range repositories {
 		log.Print("Upserting ", *repository.Name)
-		repositoryUpdateObject := getImportRepositoryObject(repository)
-		database.UpsertRepository(*repository.ID, repositoryUpdateObject)
+		err := repositoryStore.Upsert(context.TODO(), store.Repository{
+			ID:          *repository.ID,
+			Name:        *repository.Name,
+			Owner:       *repository.Owner.Login,
+			Description: *repository.Description,
+			CreatedAt:   *repository.CreatedAt.GetTime(),
+			UpdatedAt:   *repository.PushedAt.GetTime(),
+			GithubURL:   *repository.HTMLURL,
+		})
+		if err != nil {
+			log.Println("Error upserting repository:", err)
+		}
 	}
 
-	return bson.M{"repositoryCount": len(repositories)}
-}
-
-func getImportRepositoryObject(repository *gogithub.Repository) bson.M {
-	return bson.M{
-		"_id":             repository.GetID(),
-		"owner.name":      repository.GetOwner().GetLogin(),
-		"owner.avatarURL": repository.GetOwner().GetAvatarURL(),
-		"name":            repository.GetName(),
-		"description":     repository.GetDescription(),
-		"githubURL":       repository.GetHTMLURL(),
-		"githubCreatedAt": repository.GetCreatedAt().Time,
-		"pushedAt":        repository.GetPushedAt().Time,
-	}
+	log.Printf("Imported %d repositories", len(repositories))
+	return database.JSONB{"count": len(repositories)}
 }