Skip to content

Commit 3a32804

Browse files
tankerkiller125tonyaelliekatosdev
authored
Merge VNEXT into Main (#464)
* [VNEXT] feat: Multi-DB type support (#291) * feat: Multi-DB type URL formats and config * fix: remove legacy sqlite path config and minor other things * fix: dumb eslint issues * fix: dumb eslint issues * fix: application can be tested with sqlite * fix: minor config formatting * chore: some cleanup * feat: postgres migration creation now works The migration creation for postgres now works properly. Removed MySQL support, having too many issues with it at this time. * chore: revert some strings back to bytes as they should be * feat: improve languages support * feat: add locale time ago formatting and the local name for the language in language dropdown * Update FUNDING.yml * chore: remove some more mysql stuff * fix: coderabbit security recommendations * fix: validate postgres sslmode * Update migrations.go * fix: postgres migration creation now works * fix: errors in raw sql queries * fix: lint error, and simpler SQL query * fix: migrations directory string * fix: stats related test * fix: sql query * Update TextArea.vue * Update TextField.vue * chore: run integration testing on multiple postgresql versions * chore: jobs should run for vnext branch PRs * fix: missed $ for Postgres testing * fix: environment variable for db ssl mode * fix: lint issue from a merge * chore: trying to fix postgresql testing * chore: trying to fix postgresql testing * fix: trying to fix postgresql testing * fix: trying to fix postgresql testing --------- Co-authored-by: tonya <[email protected]> * fix: publish docker vnext branch * Add upgrade guide documentation * chore: add new config options to documentation * Update vnext (#314) * feat: make 404 follow theme and add a return home page * feat: sanitise translations when using v-html * chore: Add native API docs to website * chore: remove try it button from api docs --------- Co-authored-by: tonyaellie <[email protected]> * Update Dockerfile Update dockerfile to test the theory of data folder breaking in vnext * fix: broken docker image * fix: statistics * feat: support mm, cm and inches for label generation * [VNEXT] feat: Multi-DB type support (#291) * feat: Multi-DB type URL formats and config * fix: remove legacy sqlite path config and minor other things * fix: dumb eslint issues * fix: dumb eslint issues * fix: application can be tested with sqlite * fix: minor config formatting * chore: some cleanup * feat: postgres migration creation now works The migration creation for postgres now works properly. Removed MySQL support, having too many issues with it at this time. * chore: revert some strings back to bytes as they should be * feat: improve languages support * feat: add locale time ago formatting and the local name for the language in language dropdown * Update FUNDING.yml * chore: remove some more mysql stuff * fix: coderabbit security recommendations * fix: validate postgres sslmode * Update migrations.go * fix: postgres migration creation now works * fix: errors in raw sql queries * fix: lint error, and simpler SQL query * fix: migrations directory string * fix: stats related test * fix: sql query * Update TextArea.vue * Update TextField.vue * chore: run integration testing on multiple postgresql versions * chore: jobs should run for vnext branch PRs * fix: missed $ for Postgres testing * fix: environment variable for db ssl mode * fix: lint issue from a merge * chore: trying to fix postgresql testing * chore: trying to fix postgresql testing * fix: trying to fix postgresql testing * fix: trying to fix postgresql testing --------- Co-authored-by: tonya <[email protected]> * fix: publish docker vnext branch * Add upgrade guide documentation * chore: add new config options to documentation * Update Dockerfile Update dockerfile to test the theory of data folder breaking in vnext * fix: broken docker image * fix: statistics * feat: support mm, cm and inches for label generation * Update go dependencies * Update documentation * Slight update to docker actions * Small doc update * More doc changes * Sort out migrations * Temp fix to broken stats test * Update dependencies * Update documentation * Fix broken merge * Fix docker image sqlite path * Fix minor taskfile issue --------- Co-authored-by: tonya <[email protected]> Co-authored-by: Katos <[email protected]>
1 parent 33ecc49 commit 3a32804

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+3047
-2589
lines changed

‎.github/workflows/docker-publish-rootless.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,21 @@ on:
99
- 'backend/**'
1010
- 'frontend/**'
1111
- 'Dockerfile'
12-
- 'Dockerfile.rootless'
1312
- '.dockerignore'
14-
- '.github/workflows/**'
13+
- '.github/workflows/docker-publish-rootless.yaml'
14+
ignore:
15+
- 'docs/**'
1516
tags: [ 'v*.*.*' ]
1617
pull_request:
1718
branches: [ "main" ]
1819
paths:
1920
- 'backend/**'
2021
- 'frontend/**'
2122
- 'Dockerfile'
22-
- 'Dockerfile.rootless'
2323
- '.dockerignore'
24-
- '.github/workflows/**'
24+
- '.github/workflows/docker-publish-rootless.yaml'
25+
ignore:
26+
- 'docs/**'
2527

2628
permissions:
2729
contents: read # Access to repository contents

‎.github/workflows/docker-publish.yaml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ on:
1010
- 'frontend/**'
1111
- 'Dockerfile'
1212
- '.dockerignore'
13-
- '.github/workflows/**'
13+
- '.github/workflows/docker-publish.yaml'
14+
ignore:
15+
- 'docs/**'
1416
tags: [ 'v*.*.*' ]
1517
pull_request:
1618
branches: [ "main" ]
@@ -19,7 +21,9 @@ on:
1921
- 'frontend/**'
2022
- 'Dockerfile'
2123
- '.dockerignore'
22-
- '.github/workflows/**'
24+
- '.github/workflows/docker-publish.yaml'
25+
ignore:
26+
- 'docs/**'
2327

2428
env:
2529
DOCKERHUB_REPO: sysadminsmedia/homebox

‎.github/workflows/partial-frontend.yaml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,20 @@ jobs:
3232
integration-tests:
3333
name: Integration Tests
3434
runs-on: ubuntu-latest
35+
services:
36+
postgres:
37+
image: postgres:17
38+
env:
39+
POSTGRES_USER: homebox
40+
POSTGRES_PASSWORD: homebox
41+
POSTGRES_DB: homebox
42+
ports:
43+
- 5432:5432
44+
options: >-
45+
--health-cmd pg_isready
46+
--health-interval 10s
47+
--health-timeout 5s
48+
--health-retries 5
3549
steps:
3650
- name: Checkout
3751
uses: actions/checkout@v4
@@ -62,3 +76,53 @@ jobs:
6276

6377
- name: Run Integration Tests
6478
run: task test:ci
79+
integration-tests-pgsql:
80+
strategy:
81+
matrix:
82+
database_version: [17,16,15]
83+
name: Integration Tests PGSQL ${{ matrix.database_version }}
84+
runs-on: ubuntu-latest
85+
services:
86+
postgres:
87+
image: postgres:${{ matrix.database_version }}
88+
env:
89+
POSTGRES_USER: homebox
90+
POSTGRES_PASSWORD: homebox
91+
POSTGRES_DB: homebox
92+
options: >-
93+
--health-cmd pg_isready
94+
--health-interval 10s
95+
--health-timeout 5s
96+
--health-retries 5
97+
ports:
98+
- 5432:5432
99+
steps:
100+
- name: Checkout
101+
uses: actions/checkout@v4
102+
with:
103+
fetch-depth: 0
104+
105+
- name: Install Task
106+
uses: arduino/setup-task@v1
107+
with:
108+
repo-token: ${{ secrets.GITHUB_TOKEN }}
109+
110+
- name: Set up Go
111+
uses: actions/setup-go@v5
112+
with:
113+
go-version: "1.21"
114+
115+
- uses: actions/setup-node@v4
116+
with:
117+
node-version: 18
118+
119+
- uses: pnpm/[email protected]
120+
with:
121+
version: 9.12.2
122+
123+
- name: Install dependencies
124+
run: pnpm install
125+
working-directory: frontend
126+
127+
- name: Run Integration Tests
128+
run: task test:ci:postgresql

‎.github/workflows/pull-requests.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@ on:
44
pull_request:
55
branches:
66
- main
7+
- vnext
78

89
paths:
910
- 'backend/**'
1011
- 'frontend/**'
12+
- '.github/workflows/**'
1113

1214
jobs:
1315
backend-tests:

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
backend/.data/*
33
config.yml
44
homebox.db
5+
homebox.db-journal
6+
homebox.db-shm
7+
homebox.db-wal
58
.idea
69
.DS_Store
710
test-mailer.json

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
6161
FROM public.ecr.aws/docker/library/alpine:latest
6262
ENV HBOX_MODE=production
6363
ENV HBOX_STORAGE_DATA=/data/
64-
ENV HBOX_STORAGE_SQLITE_URL=/data/homebox.db?_pragma=busy_timeout=2000&_pragma=journal_mode=WAL&_fk=1&_time_format=sqlite
64+
ENV HBOX_DATABASE_SQLITE_PATH=/data/homebox.db?_pragma=busy_timeout=2000&_pragma=journal_mode=WAL&_fk=1&_time_format=sqlite
6565

6666
# Install necessary runtime dependencies
6767
RUN apk --no-cache add ca-certificates wget

‎Dockerfile.rootless‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ RUN mkdir /data
6363
FROM public.ecr.aws/docker/library/alpine:latest
6464
ENV HBOX_MODE=production
6565
ENV HBOX_STORAGE_DATA=/data/
66-
ENV HBOX_STORAGE_SQLITE_URL=/data/homebox.db?_pragma=busy_timeout=2000&_pragma=journal_mode=WAL&_fk=1&_time_format=sqlite
66+
ENV HBOX_DATABASE_SQLITE_PATH=/data/homebox.db?_pragma=busy_timeout=2000&_pragma=journal_mode=WAL&_fk=1&_time_format=sqlite
6767

6868
# Install necessary runtime dependencies
6969
RUN apk --no-cache add ca-certificates wget

‎Taskfile.yml‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ version: "3"
22

33
env:
44
HBOX_LOG_LEVEL: debug
5-
HBOX_STORAGE_SQLITE_URL: .data/homebox.db?_pragma=busy_timeout=1000&_pragma=journal_mode=WAL&_fk=1&_time_format=sqlite
5+
HBOX_DATABASE_DRIVER: sqlite3
6+
HBOX_DATABASE_SQLITE_PATH: .data/homebox.db?_pragma=busy_timeout=1000&_pragma=journal_mode=WAL&_fk=1&_time_format=sqlite
67
HBOX_OPTIONS_ALLOW_REGISTRATION: true
78
UNSAFE_DISABLE_PASSWORD_PROJECTION: "yes_i_am_sure"
89
tasks:
@@ -61,6 +62,24 @@ tasks:
6162
- go run ./app/api/ {{ .CLI_ARGS }}
6263
silent: false
6364

65+
go:run:postgresql:
66+
env:
67+
HBOX_DEMO: true
68+
HBOX_DATABASE_DRIVER: postgres
69+
HBOX_DATABASE_USERNAME: homebox
70+
HBOX_DATABASE_PASSWORD: homebox
71+
HBOX_DATABASE_DATABASE: homebox
72+
HBOX_DATABASE_HOST: localhost
73+
HBOX_DATABASE_PORT: 5432
74+
HBOX_DATABASE_SSL_MODE: disable
75+
desc: Starts the backend api server with postgresql (depends on generate task)
76+
dir: backend
77+
deps:
78+
- generate
79+
cmds:
80+
- go run ./app/api/ {{ .CLI_ARGS }}
81+
silent: false
82+
6483
go:test:
6584
desc: Runs all go tests using gotestsum - supports passing gotestsum args
6685
dir: backend
@@ -115,6 +134,21 @@ tasks:
115134
cmds:
116135
- cd backend && go run app/tools/migrations/main.go {{ .CLI_ARGS }}
117136

137+
db:migration:postgresql:
138+
env:
139+
HBOX_DATABASE_DRIVER: postgres
140+
HBOX_DATABASE_USERNAME: homebox
141+
HBOX_DATABASE_PASSWORD: homebox
142+
HBOX_DATABASE_DATABASE: homebox
143+
HBOX_DATABASE_HOST: localhost
144+
HBOX_DATABASE_PORT: 5432
145+
HBOX_DATABASE_SSL_MODE: disable
146+
desc: Runs the database diff engine to generate a SQL migration files for postgresql
147+
deps:
148+
- db:generate
149+
cmds:
150+
- cd backend && go run app/tools/migrations/main.go {{ .CLI_ARGS }}
151+
118152
ui:watch:
119153
desc: Starts the vitest test runner in watch mode
120154
dir: frontend
@@ -144,7 +178,24 @@ tasks:
144178
cmds:
145179
- cd backend && go build ./app/api
146180
- backend/api &
147-
- sleep 5
181+
- sleep 10
182+
- cd frontend && pnpm run test:ci
183+
silent: true
184+
185+
test:ci:postgresql:
186+
env:
187+
HBOX_DATABASE_DRIVER: postgres
188+
HBOX_DATABASE_USERNAME: homebox
189+
HBOX_DATABASE_PASSWORD: homebox
190+
HBOX_DATABASE_DATABASE: homebox
191+
HBOX_DATABASE_HOST: 127.0.0.1
192+
HBOX_DATABASE_PORT: 5432
193+
HBOX_DATABASE_SSL_MODE: disable
194+
desc: Runs end-to-end test on a live server with postgresql (only for use in CI)
195+
cmds:
196+
- cd backend && go build ./app/api
197+
- backend/api &
198+
- sleep 10
148199
- cd frontend && pnpm run test:ci
149200
silent: true
150201

‎backend/app/api/main.go‎

Lines changed: 64 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@ import (
44
"bytes"
55
"context"
66
"fmt"
7+
"github.com/google/uuid"
78
"net/http"
89
"os"
910
"path/filepath"
11+
"strings"
1012
"time"
1113

1214
atlas "ariga.io/atlas/sql/migrate"
@@ -28,6 +30,7 @@ import (
2830
"github.com/sysadminsmedia/homebox/backend/internal/sys/config"
2931
"github.com/sysadminsmedia/homebox/backend/internal/web/mid"
3032

33+
_ "github.com/lib/pq"
3134
_ "github.com/sysadminsmedia/homebox/backend/pkgs/cgofreesqlite"
3235
)
3336

@@ -46,15 +49,29 @@ func build() string {
4649
return fmt.Sprintf("%s, commit %s, built at %s", version, short, buildTime)
4750
}
4851

49-
// @title Homebox API
50-
// @version 1.0
51-
// @description Track, Manage, and Organize your Things.
52-
// @contact.name Don't
53-
// @BasePath /api
54-
// @securityDefinitions.apikey Bearer
55-
// @in header
56-
// @name Authorization
57-
// @description "Type 'Bearer TOKEN' to correctly set the API Key"
52+
func validatePostgresSSLMode(sslMode string) bool {
53+
validModes := map[string]bool{
54+
"": true,
55+
"disable": true,
56+
"allow": true,
57+
"prefer": true,
58+
"require": true,
59+
"verify-ca": true,
60+
"verify-full": true,
61+
}
62+
return validModes[strings.ToLower(strings.TrimSpace(sslMode))]
63+
}
64+
65+
// @title Homebox API
66+
// @version 1.0
67+
// @description Track, Manage, and Organize your Things.
68+
// @contact.name Don't
69+
// @BasePath /api
70+
// @securityDefinitions.apikey Bearer
71+
// @in header
72+
// @name Authorization
73+
// @description "Type 'Bearer TOKEN' to correctly set the API Key"
74+
5875
func main() {
5976
zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack
6077

@@ -80,13 +97,32 @@ func run(cfg *config.Config) error {
8097
log.Fatal().Err(err).Msg("failed to create data directory")
8198
}
8299

83-
c, err := ent.Open("sqlite3", cfg.Storage.SqliteURL)
100+
if strings.ToLower(cfg.Database.Driver) == "postgres" {
101+
if !validatePostgresSSLMode(cfg.Database.SslMode) {
102+
log.Fatal().Str("sslmode", cfg.Database.SslMode).Msg("invalid sslmode")
103+
}
104+
}
105+
106+
// Set up the database URL based on the driver because for some reason a common URL format is not used
107+
databaseURL := ""
108+
switch strings.ToLower(cfg.Database.Driver) {
109+
case "sqlite3":
110+
databaseURL = cfg.Database.SqlitePath
111+
case "postgres":
112+
databaseURL = fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s", cfg.Database.Host, cfg.Database.Port, cfg.Database.Username, cfg.Database.Password, cfg.Database.Database, cfg.Database.SslMode)
113+
default:
114+
log.Fatal().Str("driver", cfg.Database.Driver).Msg("unsupported database driver")
115+
}
116+
117+
c, err := ent.Open(strings.ToLower(cfg.Database.Driver), databaseURL)
84118
if err != nil {
85119
log.Fatal().
86120
Err(err).
87-
Str("driver", "sqlite").
88-
Str("url", cfg.Storage.SqliteURL).
89-
Msg("failed opening connection to sqlite")
121+
Str("driver", strings.ToLower(cfg.Database.Driver)).
122+
Str("host", cfg.Database.Host).
123+
Str("port", cfg.Database.Port).
124+
Str("database", cfg.Database.Database).
125+
Msg("failed opening connection to {driver} database at {host}:{port}/{database}")
90126
}
91127
defer func(c *ent.Client) {
92128
err := c.Close()
@@ -95,9 +131,14 @@ func run(cfg *config.Config) error {
95131
}
96132
}(c)
97133

98-
temp := filepath.Join(os.TempDir(), "migrations")
134+
// Always create a random temporary directory for migrations
135+
tempUUID, err := uuid.NewUUID()
136+
if err != nil {
137+
return err
138+
}
139+
temp := filepath.Join(os.TempDir(), fmt.Sprintf("homebox-%s", tempUUID.String()))
99140

100-
err = migrations.Write(temp)
141+
err = migrations.Write(temp, cfg.Database.Driver)
101142
if err != nil {
102143
return err
103144
}
@@ -117,17 +158,18 @@ func run(cfg *config.Config) error {
117158
if err != nil {
118159
log.Error().
119160
Err(err).
120-
Str("driver", "sqlite").
121-
Str("url", cfg.Storage.SqliteURL).
161+
Str("driver", cfg.Database.Driver).
162+
Str("url", databaseURL).
122163
Msg("failed creating schema resources")
123164
return err
124165
}
125166

126-
err = os.RemoveAll(temp)
127-
if err != nil {
128-
log.Error().Err(err).Msg("failed to remove temporary directory for database migrations")
129-
return err
130-
}
167+
defer func() {
168+
err := os.RemoveAll(temp)
169+
if err != nil {
170+
log.Error().Err(err).Msg("failed to remove temporary directory for database migrations")
171+
}
172+
}()
131173

132174
collectFuncs := []currencies.CollectorFunc{
133175
currencies.CollectDefaults(),

0 commit comments

Comments
 (0)