Skip to content

Commit ade3c74

Browse files
nicoburniskenicoburniske
authored
GOL-188 Compilation Server (#150)
* initial compilation server * progress * remove cancellation token, fix config, and integrate template_id instead of UUID in grpc * read grpc host from config * rename compilation service to template compilation service * integrate into golem-service * add metrics * add cargo lock * port changes * remove duplicate entry from grpc build * fix unit tests for config * remove cargo config * remove golem-service dir * re-add dependency to cargo toml * rename template-compilation to template-compilation-service * update dockerfile to use 8080 * fix docker compose for local dev and add logs * add template compilation service to ci
1 parent 9e2108a commit ade3c74

File tree

29 files changed

+1115
-14
lines changed

29 files changed

+1115
-14
lines changed

‎.env‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ POSTGRES_PORT=5432
33
GOLEM_ROUTER_PORT=9881
44
TEMPLATE_SERVICE_HTTP_PORT=8083
55
TEMPLATE_SERVICE_GRPC_PORT=9090
6+
TEMPLATE_COMPILATION_SERVICE_HTTP_PORT=8084
7+
TEMPLATE_COMPILATION_SERVICE_GRPC_PORT=9091
68
SHARD_MANAGER_HTTP_PORT=8081
79
SHARD_MANAGER_GRPC_PORT=9002
810
WORKER_EXECUTOR_HTTP_PORT=8082

‎.github/workflows/ci.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ jobs:
7575
${{ matrix.platform.target }}/release/worker-executor \
7676
${{ matrix.platform.target }}/release/golem-template-service \
7777
${{ matrix.platform.target }}/release/golem-worker-service \
78+
${{ matrix.platform.target }}/release/golem-template-compilation-service
7879
- uses: actions/upload-artifact@v4
7980
name: Upload Targets
8081
with:
@@ -181,6 +182,20 @@ jobs:
181182
platforms: ${{ env.PLATFORMS }}
182183
tags: ${{ steps.meta-golem-worker-service.outputs.tags }}
183184
labels: ${{ steps.meta-golem-worker-service.outputs.labels }}
185+
- name: Extract metadata (tags, labels) for golem template compilation service
186+
id: meta-golem-template-compilation-service
187+
uses: docker/metadata-action@v5
188+
with:
189+
images: golemservices/golem-template-compilation-service
190+
- name: Build and push golem template compilation service image
191+
uses: docker/build-push-action@v5
192+
with:
193+
context: .
194+
file: ./golem-template-compilation-service/docker/Dockerfile
195+
push: true
196+
platforms: ${{ env.PLATFORMS }}
197+
tags: ${{ steps.meta-golem-template-compilation-service.outputs.tags }}
198+
labels: ${{ steps.meta-golem-template-compilation-service.outputs.labels }}
184199
- name: Extract metadata (tags, labels) for golem router
185200
id: meta-golem-router
186201
uses: docker/metadata-action@v5
@@ -294,6 +309,7 @@ jobs:
294309
path: |
295310
target/release/golem-template-service
296311
target/release/golem-worker-service
312+
target/release/golem-template-compilation-service
297313
target/release/golem-worker-service-yaml
298314
target/release/golem-template-service-yaml
299315
target/release/golem-shard-manager

‎Cargo.lock‎

Lines changed: 28 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ members = [
1010
"golem-template-service",
1111
"golem-shard-manager",
1212
"golem-worker-service",
13+
"golem-template-compilation-service"
1314
]
1415

1516
exclude = [

‎docker-compose-postgres.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ services:
6767
- GOLEM__TEMPLATE_STORE__TYPE="Local"
6868
- GOLEM__TEMPLATE_STORE__CONFIG__OBJECT_PREFIX=""
6969
- GOLEM__TEMPLATE_STORE__CONFIG__ROOT_PATH=/template_store
70+
- GOLEM__COMPILATION__TYPE="Enabled"
71+
- GOLEM__COMPILATION__CONFIG__HOST=golem-template-compilation-service
72+
- GOLEM__COMPILATION__CONFIG__PORT=${TEMPLATE_COMPILATION_SERVICE_GRPC_PORT}
7073
- GOLEM__DB__TYPE=Postgres
7174
- GOLEM__DB__CONFIG__DATABASE=golem_db
7275
- GOLEM__DB__CONFIG__MAX_CONNECTIONS=10
@@ -109,6 +112,26 @@ services:
109112
- "${WORKER_SERVICE_GRPC_PORT}:${WORKER_SERVICE_GRPC_PORT}"
110113
depends_on:
111114
- redis
115+
116+
golem-template-compilation-service:
117+
build:
118+
context: .
119+
dockerfile: golem-template-compilation-service/docker/Dockerfile
120+
environment:
121+
- RUST_BACKTRACE=1
122+
- RUST_LOG=info,h2=warn,hyper=warn,tower=warn
123+
- GOLEM__TEMPLATE_SERVICE__HOST=golem-template-service
124+
- GOLEM__TEMPLATE_SERVICE__PORT=${TEMPLATE_SERVICE_GRPC_PORT}
125+
- GOLEM__TEMPLATE_SERVICE__ACCESS_TOKEN="5C832D93-FF85-4A8F-9803-513950FDFDB1"
126+
- GOLEM__COMPILED_TEMPLATE_SERVICE__CONFIG__ROOT="data/templates"
127+
- GOLEM__COMPILED_TEMPLATE_SERVICE__TYPE="Local"
128+
- GOLEM__HTTP_PORT=${TEMPLATE_COMPILATION_SERVICE_HTTP_PORT}
129+
- GOLEM__GRPC_PORT=${TEMPLATE_COMPILATION_SERVICE_GRPC_PORT}
130+
ports:
131+
- "${TEMPLATE_COMPILATION_SERVICE_HTTP_PORT}:${TEMPLATE_COMPILATION_SERVICE_HTTP_PORT}"
132+
- "${TEMPLATE_COMPILATION_SERVICE_GRPC_PORT}:${TEMPLATE_COMPILATION_SERVICE_GRPC_PORT}"
133+
depends_on:
134+
- golem-template-service
112135

113136
golem-worker-executor:
114137
build:

‎docker-compose-sqlite.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,9 @@ services:
5555
- GOLEM__TEMPLATE_STORE__TYPE="Local"
5656
- GOLEM__TEMPLATE_STORE__CONFIG__OBJECT_PREFIX=""
5757
- GOLEM__TEMPLATE_STORE__CONFIG__ROOT_PATH=/template_store
58+
- GOLEM__COMPILATION__TYPE="Enabled"
59+
- GOLEM__COMPILATION__CONFIG__HOST=golem-template-compilation-service
60+
- GOLEM__COMPILATION__CONFIG__PORT=${TEMPLATE_COMPILATION_SERVICE_GRPC_PORT}
5861
- GOLEM__DB__TYPE=Sqlite
5962
- GOLEM__DB__CONFIG__DATABASE=/app/golem_db/golem.sqlite
6063
- GOLEM__DB__CONFIG__MAX_CONNECTIONS=10
@@ -94,6 +97,26 @@ services:
9497
depends_on:
9598
- redis
9699

100+
golem-template-compilation-service:
101+
build:
102+
context: .
103+
dockerfile: golem-template-compilation-service/docker/Dockerfile
104+
environment:
105+
- RUST_BACKTRACE=1
106+
- RUST_LOG=info,h2=warn,hyper=warn,tower=warn
107+
- GOLEM__TEMPLATE_SERVICE__HOST=golem-template-service
108+
- GOLEM__TEMPLATE_SERVICE__PORT=${TEMPLATE_SERVICE_GRPC_PORT}
109+
- GOLEM__TEMPLATE_SERVICE__ACCESS_TOKEN="5C832D93-FF85-4A8F-9803-513950FDFDB1"
110+
- GOLEM__COMPILED_TEMPLATE_SERVICE__CONFIG__ROOT="data/templates"
111+
- GOLEM__COMPILED_TEMPLATE_SERVICE__TYPE="Local"
112+
- GOLEM__HTTP_PORT=${TEMPLATE_COMPILATION_SERVICE_HTTP_PORT}
113+
- GOLEM__GRPC_PORT=${TEMPLATE_COMPILATION_SERVICE_GRPC_PORT}
114+
ports:
115+
- "${TEMPLATE_COMPILATION_SERVICE_HTTP_PORT}:${TEMPLATE_COMPILATION_SERVICE_HTTP_PORT}"
116+
- "${TEMPLATE_COMPILATION_SERVICE_GRPC_PORT}:${TEMPLATE_COMPILATION_SERVICE_GRPC_PORT}"
117+
depends_on:
118+
- golem-template-service
119+
97120
golem-worker-executor:
98121
build:
99122
context: .

‎golem-api-grpc/build.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
3636
"proto/golem/template/versioned_name.proto",
3737
"proto/golem/template/versioned_template_id.proto",
3838
"proto/golem/template/template_service.proto",
39+
"proto/golem/templatecompilation/template_compilation_service.proto",
3940
"proto/golem/worker/calling_convention.proto",
4041
"proto/golem/worker/complete_parameters.proto",
4142
"proto/golem/worker/invocation_key.proto",

‎golem-api-grpc/proto/golem/template/template_service.proto‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,4 +125,3 @@ message UpdateTemplateResponse {
125125
golem.template.TemplateError error = 2;
126126
}
127127
}
128-

‎golem-api-grpc/proto/golem/templatecompilation/template_compilation_service.proto‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
syntax = "proto3";
2+
3+
import "golem/common/empty.proto";
4+
import "golem/common/error_body.proto";
5+
import "golem/template/template_id.proto";
6+
7+
package golem.templatecompilation;
8+
9+
service TemplateCompilationService {
10+
rpc EnqueueCompilation(TemplateCompilationRequest) returns (TemplateCompilationResponse);
11+
}
12+
13+
message TemplateCompilationRequest {
14+
golem.template.TemplateId template_id = 1;
15+
int32 template_version = 2;
16+
}
17+
18+
message TemplateCompilationResponse {
19+
oneof result {
20+
golem.common.Empty success = 1;
21+
TemplateCompilationError failure = 2;
22+
}
23+
}
24+
25+
message TemplateCompilationError {
26+
oneof error {
27+
golem.common.ErrorsBody bad_request = 1;
28+
golem.common.ErrorBody not_found = 2;
29+
golem.common.ErrorBody internal_error = 3;
30+
}
31+
}

‎golem-template-compilation-service/Cargo.toml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
[package]
2+
name = "golem-template-compilation-service"
3+
version = "0.0.0"
4+
edition = "2021"
5+
6+
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
7+
8+
[[bin]]
9+
name = "golem-template-compilation-service"
10+
path = "src/server.rs"
11+
12+
[dependencies]
13+
golem-api-grpc = { version = "0.0.0", path = "../golem-api-grpc" }
14+
golem-common = { version = "0.0.0", path = "../golem-common" }
15+
golem-worker-executor-base = { version = "0.0.0", path = "../golem-worker-executor-base" }
16+
17+
async-trait = { workspace = true }
18+
futures = { workspace = true }
19+
http = { workspace = true }
20+
http_02 = { package = "http", version = "0.2.11" }
21+
tokio = { workspace = true }
22+
23+
tracing = { workspace = true }
24+
uuid = { workspace = true }
25+
wasmtime = { version = "=17.0.0", features = ["component-model"] }
26+
tonic = { workspace = true }
27+
tonic-health = { workspace = true }
28+
thiserror = { workspace = true }
29+
serde = { workspace = true }
30+
figment = { workspace = true }
31+
console-subscriber = { workspace = true }
32+
tracing-subscriber = { workspace = true, features = [
33+
"env-filter",
34+
"fmt",
35+
"std",
36+
"json",
37+
] }
38+
prometheus = { workspace = true }
39+
futures-util = "0.3.30"
40+
lazy_static.workspace = true

0 commit comments

Comments
 (0)