Skip to content

Commit 9c39fec

Browse files
0x416e746f6e0x416e746f6e
committed
feat: implement soft-deletes
1 parent a4c9e23 commit 9c39fec

15 files changed

+135
-65
lines changed

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ snapshot:
1414

1515
.PHONY: help
1616
help:
17+
@go run github.com/flashbots/gh-artifacts-sync/cmd --help
1718
@go run github.com/flashbots/gh-artifacts-sync/cmd serve --help
1819

1920
.PHONY: serve

‎cmd/serve.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,26 @@ func CommandServe(cfg *config.Config) *cli.Command {
5454
Usage: "a `path` to the directory where scheduled jobs will be persisted",
5555
Value: "./jobs",
5656
},
57+
58+
&cli.StringFlag{ // --dir-soft-delete-downloads
59+
Aliases: []string{"dir.soft_delete_downloads"},
60+
Category: strings.ToUpper(categoryDir),
61+
Destination: &cfg.SoftDelete.Downloads,
62+
EnvVars: []string{envPrefix + strings.ToUpper(categoryDir) + "_SOFT_DELETE_DOWNLOADS"},
63+
Name: categoryDir + "-soft-delete-downloads",
64+
Usage: "a `path` to the directory where finalised downloaded will be moved to instead of deleting",
65+
Value: "",
66+
},
67+
68+
&cli.StringFlag{ // --dir-soft-delete-jobs
69+
Aliases: []string{"dir.soft_delete_jobs"},
70+
Category: strings.ToUpper(categoryDir),
71+
Destination: &cfg.SoftDelete.Jobs,
72+
EnvVars: []string{envPrefix + strings.ToUpper(categoryDir) + "_SOFT_DELETE_JOBS"},
73+
Name: categoryDir + "-soft-delete-jobs",
74+
Usage: "a `path` to the directory where complete jobs be moved instead of deleting",
75+
Value: "",
76+
},
5777
}
5878

5979
githubFlags := []cli.Flag{ // --github-xxx

‎config/config.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ type Config struct {
1616
Log *Log `yaml:"log" json:"log"`
1717
Repositories map[string]*Repository `yaml:"repositories" json:"repositories"`
1818
Server *Server `yaml:"server" json:"server"`
19+
SoftDelete *Dir `yaml:"soft_delete" json:"soft_delete"`
1920
}
2021

2122
var (
@@ -24,10 +25,11 @@ var (
2425

2526
func New() *Config {
2627
return &Config{
27-
Dir: &Dir{},
28-
Github: &Github{App: &GithubApp{}},
29-
Log: &Log{},
30-
Server: &Server{},
28+
Dir: &Dir{},
29+
Github: &Github{App: &GithubApp{}},
30+
Log: &Log{},
31+
Server: &Server{},
32+
SoftDelete: &Dir{},
3133
}
3234
}
3335

‎config/dir.go‎

Lines changed: 17 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -9,48 +9,39 @@ import (
99
)
1010

1111
type Dir struct {
12-
Downloads string `yaml:"downloads" json:"downloads"`
13-
Jobs string `yaml:"jobs" json:"jobs"`
12+
Downloads string `yaml:"downloads" json:"downloads"`
13+
Jobs string `yaml:"jobs" json:"jobs"`
1414
}
1515

1616
var (
17-
errDirNotDirectory = errors.New("not a directory")
17+
errDirFailedToAccess = errors.New("failed to access a directory")
18+
errDirFailedToCreate = errors.New("failed to create a directory")
19+
errDirNotDirectory = errors.New("not a directory")
1820
)
1921

2022
func (cfg *Dir) Validate() error {
2123
errs := make([]error, 0)
2224

23-
{ // artifacts
24-
if info, err := os.Stat(cfg.Downloads); err != nil {
25-
if !os.IsNotExist(err) {
26-
if errMkdir := os.Mkdir(cfg.Downloads, 0640); errMkdir != nil {
27-
errs = append(errs, err, errMkdir)
28-
}
29-
} else {
30-
errs = append(errs, err)
31-
}
32-
} else {
33-
if !info.IsDir() {
34-
errs = append(errs, fmt.Errorf("%w: %s",
35-
errDirNotDirectory, cfg.Downloads,
36-
))
37-
}
25+
for _, dir := range []string{cfg.Downloads, cfg.Jobs} {
26+
if dir == "" {
27+
continue
3828
}
39-
}
40-
41-
{ // jobs
42-
if info, err := os.Stat(cfg.Jobs); err != nil {
29+
if info, err := os.Stat(dir); err != nil {
4330
if !os.IsNotExist(err) {
44-
if errMkdir := os.Mkdir(cfg.Jobs, 0640); errMkdir != nil {
45-
errs = append(errs, err, errMkdir)
31+
if errMkdir := os.Mkdir(dir, 0640); errMkdir != nil {
32+
errs = append(errs, fmt.Errorf("%w: %s: %w",
33+
errDirFailedToCreate, dir, err,
34+
))
4635
}
4736
} else {
48-
errs = append(errs, err)
37+
errs = append(errs, fmt.Errorf("%w: %s: %w",
38+
errDirFailedToAccess, dir, err,
39+
))
4940
}
5041
} else {
5142
if !info.IsDir() {
5243
errs = append(errs, fmt.Errorf("%w: %s",
53-
errDirNotDirectory, cfg.Jobs,
44+
errDirNotDirectory, dir,
5445
))
5546
}
5647
}

‎readme.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,10 @@ USAGE:
6565
OPTIONS:
6666
DIR
6767

68-
--dir-artifacts path, --dir.artifacts path a path to the directory where downloaded artifacts will be temporarily stored (default: "./artifacts") [$GH_ARTIFACTS_SYNC_DIR_ARTIFACTS]
69-
--dir-jobs path, --dir.jobs path a path to the directory where scheduled jobs will be persisted (default: "./jobs") [$GH_ARTIFACTS_SYNC_DIR_JOBS]
68+
--dir-downloads path, --dir.downloads path a path to the directory where downloaded artifacts will be temporarily stored (default: "./downloads") [$GH_ARTIFACTS_SYNC_DIR_DOWNLOADS]
69+
--dir-jobs path, --dir.jobs path a path to the directory where scheduled jobs will be persisted (default: "./jobs") [$GH_ARTIFACTS_SYNC_DIR_JOBS]
70+
--dir-soft-delete-downloads path, --dir.soft_delete_downloads path a path to the directory where finalised downloaded will be moved to instead of deleting [$GH_ARTIFACTS_SYNC_DIR_SOFT_DELETE_DOWNLOADS]
71+
--dir-soft-delete-jobs path, --dir.soft_delete_jobs path a path to the directory where complete jobs be moved instead of deleting [$GH_ARTIFACTS_SYNC_DIR_SOFT_DELETE_JOBS]
7072

7173
GITHUB
7274

‎server/download_github_artifact.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
"github.com/flashbots/gh-artifacts-sync/job"
1212
)
1313

14-
func (s *Server) downloadGithubWorkflowArtifact(
14+
func (s *Server) downloadGithubArtifact(
1515
ctx context.Context,
1616
j *job.SyncWorkflowArtifact,
1717
) (string, error) {

‎server/download_github_container.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import (
2222
"go.uber.org/zap"
2323
)
2424

25-
func (s *Server) downloadGithubContainerRegistryPackage(
25+
func (s *Server) downloadGithubContainer(
2626
ctx context.Context,
2727
j *job.SyncContainerRegistryPackage,
2828
) (string, error) {

‎server/download_github_release.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
"github.com/flashbots/gh-artifacts-sync/job"
1212
)
1313

14-
func (s *Server) downloadGithubReleaseAsset(
14+
func (s *Server) downloadGithubRelease(
1515
ctx context.Context,
1616
j *job.SyncReleaseAsset,
1717
) (string, error) {

‎server/handle_cleanup_unparseable.go‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package server
22

33
import (
44
"context"
5-
"os"
65

76
"github.com/flashbots/gh-artifacts-sync/job"
87
"github.com/flashbots/gh-artifacts-sync/logutils"
@@ -19,12 +18,7 @@ func (s *Server) handleCleanupUnparseableJob(
1918
zap.String("path", job.Path(j)),
2019
)
2120

22-
if err := os.Remove(job.Path(j)); err != nil {
23-
l.Error("Failed to remove unparseable job",
24-
zap.Error(err),
25-
zap.String("path", job.Path(j)),
26-
)
27-
}
21+
s.RemoveJob(ctx, j)
2822

2923
return nil
3024
}

‎server/handle_sync_artifact.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ package server
22

33
import (
44
"context"
5-
"errors"
6-
"os"
75

86
"github.com/flashbots/gh-artifacts-sync/job"
97
"github.com/flashbots/gh-artifacts-sync/logutils"
@@ -25,10 +23,11 @@ func (s *Server) handleSyncWorkflowArtifact(
2523

2624
l.Info("Synchronising workflow artifact...")
2725

28-
zname, err := s.downloadGithubWorkflowArtifact(ctx, j)
26+
zname, err := s.downloadGithubArtifact(ctx, j)
2927
if err != nil {
3028
l.Error("Failed to download workflow artifact", zap.Error(err))
31-
return errors.Join(err, os.Remove(zname))
29+
s.RemoveDownload(ctx, zname)
30+
return err
3231
}
3332

3433
if err := s.uploadFromZipAndDelete(ctx, j, zname); err != nil {

0 commit comments

Comments
 (0)