Docker amd64 and arm64 build (#59)

Author: justcoon

.dockerignore

@@ -7,4 +7,10 @@ README.md
 !target/x86_64-unknown-linux-gnu/debug/cloud-server-oss
 !target/x86_64-unknown-linux-gnu/release/golem-shard-manager
 !target/x86_64-unknown-linux-gnu/release/worker-executor-oss
-!target/x86_64-unknown-linux-gnu/release/cloud-server-oss
+!target/x86_64-unknown-linux-gnu/release/cloud-server-oss
+!target/aarch64-unknown-linux-gnu/debug/golem-shard-manager
+!target/aarch64-unknown-linux-gnu/debug/worker-executor-oss
+!target/aarch64-unknown-linux-gnu/debug/cloud-server-oss
+!target/aarch64-unknown-linux-gnu/release/golem-shard-manager
+!target/aarch64-unknown-linux-gnu/release/worker-executor-oss
+!target/aarch64-unknown-linux-gnu/release/cloud-server-oss

.github/workflows/ci.yaml

@@ -29,10 +29,18 @@ jobs:
         with:
           toolchain: stable
           override: true
+      - name: Install Rust Targets
+        run: |
+          rustup target add x86_64-unknown-linux-gnu
+          rustup target add aarch64-unknown-linux-gnu
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install gcc-aarch64-linux-gnu
+        run: |
+          sudo apt-get update
+          sudo apt-get install gcc-aarch64-linux-gnu
       - uses: actions/cache@v3
         with:
           path: |
@@ -42,11 +50,17 @@ jobs:
             ~/.cargo/git/db/
             target/
           key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }}
-      - name: Install Target
-        run: rustup target add x86_64-unknown-linux-gnu
-      - name: Build Executables
-        run: cargo build --release --target ${{ env.BUILD_TARGET }}
-      - name: Set version
+      - name: Build amd64 Executables
+        run: |
+          cargo build --release --target x86_64-unknown-linux-gnu
+      - name: Build arm64 Executables
+        run: |
+          cargo build --release --target aarch64-unknown-linux-gnu --config "target.aarch64-unknown-linux-gnu.linker = 'aarch64-linux-gnu-gcc'"
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Set Docker version
         id: set-version
         run: |
           if [ "${{ github.event_name }}" == 'push' ] && [ "${{ github.ref_type }}" == 'tag' ]; then
@@ -56,60 +70,55 @@ jobs:
             COMMIT_SHORT_HASH=$(git rev-parse --short=7 HEAD)
             echo "DOCKER_VERSION=${COMMIT_SHORT_HASH}" >> $GITHUB_ENV
           fi
-      - uses: satackey/action-docker-layer-caching@v0.0.11
+      - uses: jpribyl/action-docker-layer-caching@v0.1.1
         continue-on-error: true
-
       - name: Log in to Docker Hub
-        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
       - name: Extract metadata (tags, labels) for worker executor
         id: meta-worker-executor
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        uses: docker/metadata-action@v5
         with:
           images: golemservices/golem-worker-executor
-
       - name: Build and push worker executor image
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./golem-worker-executor-oss/docker/amd64/Dockerfile
+          file: ./golem-worker-executor-oss/docker/Dockerfile
           push: true
+          platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta-worker-executor.outputs.tags }}
           labels: ${{ steps.meta-worker-executor.outputs.labels }}
-
       - name: Extract metadata (tags, labels) for shard manager
         id: meta-shard-manager
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        uses: docker/metadata-action@v5
         with:
           images: golemservices/golem-shard-manager
-
       - name: Build and push shard manager image
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./golem-shard-manager/docker/amd64/Dockerfile
+          file: ./golem-shard-manager/docker/Dockerfile
           push: true
+          platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta-shard-manager.outputs.tags }}
           labels: ${{ steps.meta-shard-manager.outputs.labels }}
-
       - name: Extract metadata (tags, labels) for golem server
         id: meta-golem-server
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        uses: docker/metadata-action@v5
         with:
           images: golemservices/golem-server
-
       - name: Build and push cloud server image
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./golem-cloud-server-oss/docker/amd64/Dockerfile
+          file: ./golem-cloud-server-oss/docker/Dockerfile
           push: true
+          platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta-golem-server.outputs.tags }}
           labels: ${{ steps.meta-golem-server.outputs.labels }}
-
   build:
     runs-on: ubuntu-latest
     steps:

CONTRIBUTING.md

@@ -1,13 +1,12 @@
-
 ## Local Testing
 
-To spin up services using the latest code 
+To spin up services using the latest code
 
 ```bash
 # Clone golem-services
 cd golem-services
 # Find more info below if you are having issues running this command(example: Running from MAC may fail)
-# Target has to be x86_64-unknown-linux-gnu 
+# Target has to be x86_64-unknown-linux-gnu or aarch64-unknown-linux-gnu-gcc
 cargo build --release --target x86_64-unknown-linux-gnu
 
 docker-compose up --build
@@ -33,9 +32,11 @@ Make sure to do `docker-compose pull` next time to make sure you are pulling the
 ### Cargo Build
 
 ### MAC
-If you are running ` cargo build --target x86_64-unknown-linux-gnu` (cross compiling to Linux) from MAC, you may encounter
+If you are running ` cargo build --target ARCH-unknown-linux-gnu` (cross compiling to Linux) from MAC, you may encounter
 some missing dependencies. If interested, refer, https://github.com/messense/homebrew-macos-cross-toolchains
 
+### Intel MAC
+
 Typically, the following should allow you to run it successfully.
 
 ```bash
@@ -54,6 +55,26 @@ rustup target add x86_64-unknown-linux-gnu
 cargo build --target x86_64-unknown-linux-gnu
 ```
 
+### ARM MAC
+
+Typically, the following should allow you to run it successfully.
+
+```bash
+brew tap messense/macos-cross-toolchains
+brew install aarch64-unknown-linux-gnu
+export CC_AARCH64_UNKNOWN_LINUX_GNU=aarch64-unknown-linux-gnu-gcc
+export CXX_AARCH64_UNKNOWN_LINUX_GNU=aarch64-unknown-linux-gnu-g++
+export AR_AARCH64_UNKNOWN_LINUX_GNU=aarch64-unknown-linux-gnu-ar
+export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-unknown-linux-gnu-gcc
+```
+
+From the root of the project
+
+```bash
+rustup target add aarch64-unknown-linux-gnu-gcc
+cargo build --target aarch64-unknown-linux-gnu-gcc
+```
+
 ### LINUX
 
 From the root of the project
@@ -66,4 +87,4 @@ cargo build --target x86_64-unknown-linux-gnu
 ### WINDOWS
 TBD
 
-We will be trying cargo chef and offload cargo build to docker context without impacting the build time.
+We will be trying cargo chef and offload cargo build to docker context without impacting the build time.

docker-compose.yml

@@ -11,7 +11,7 @@ services:
   golem-shard-manager:
     build:
       context: .
-      dockerfile: golem-shard-manager/docker/amd64/Dockerfile
+      dockerfile: golem-shard-manager/docker/Dockerfile
     environment:
       - WASMTIME_BACKTRACE_DETAILS=1
       - RUST_BACKTRACE=1
@@ -27,7 +27,7 @@ services:
   golem-cloud-server-oss:
     build:
       context: .
-      dockerfile: golem-cloud-server-oss/docker/amd64/Dockerfile
+      dockerfile: golem-cloud-server-oss/docker/Dockerfile
     environment:
       - ENVIRONMENT=local
       - RUST_LOG="debug,h2=warn,hyper=warn,tower=warn"
@@ -52,7 +52,7 @@ services:
   golem-worker-executor-oss:
     build:
       context: .
-      dockerfile: golem-worker-executor-oss/docker/amd64/Dockerfile
+      dockerfile: golem-worker-executor-oss/docker/Dockerfile
     environment:
       - ENVIRONMENT=local
       - RUST_LOG=info

golem-cloud-server-oss/docker/Dockerfile

@@ -0,0 +1,26 @@
+FROM debian:bookworm-slim as base
+
+ARG TARGETARCH
+
+FROM base AS platform-amd64
+
+ENV RUST_TARGET=x86_64-unknown-linux-gnu
+
+FROM base AS platform-arm64
+
+ENV RUST_TARGET=aarch64-unknown-linux-gnu
+
+FROM platform-${TARGETARCH} AS final
+
+WORKDIR /app
+COPY /target/$RUST_TARGET/release/cloud-server-oss ./
+COPY /golem-cloud-server-oss/config/cloud-server.toml ./config/cloud-server.toml
+
+RUN apt-get update && apt-get install -y libssl-dev
+RUN apt-get update && apt-get install -y ca-certificates
+RUN update-ca-certificates
+
+EXPOSE 8080
+EXPOSE 9090
+
+ENTRYPOINT ["./cloud-server-oss"]

golem-shard-manager/docker/Dockerfile

@@ -0,0 +1,27 @@
+FROM debian:bookworm-slim as base
+
+ARG TARGETARCH
+
+FROM base AS platform-amd64
+
+ENV RUST_TARGET=x86_64-unknown-linux-gnu
+
+FROM base AS platform-arm64
+
+ENV RUST_TARGET=aarch64-unknown-linux-gnu
+
+FROM platform-${TARGETARCH} AS final
+
+WORKDIR /app
+COPY /target/$RUST_TARGET/release/golem-shard-manager ./
+COPY /golem-shard-manager/config/shard-manager.toml ./config/shard-manager.toml
+
+RUN apt-get update && apt-get install -y libssl-dev
+RUN apt-get update && apt-get install -y ca-certificates
+RUN update-ca-certificates
+
+EXPOSE 8081
+EXPOSE 9001
+
+ENTRYPOINT ["./golem-shard-manager"]
+

golem-worker-executor-oss/docker/Dockerfile

@@ -0,0 +1,25 @@
+FROM debian:bookworm-slim as base
+
+ARG TARGETARCH
+
+FROM base AS platform-amd64
+
+ENV RUST_TARGET=x86_64-unknown-linux-gnu
+
+FROM base AS platform-arm64
+
+ENV RUST_TARGET=aarch64-unknown-linux-gnu
+
+FROM platform-${TARGETARCH} AS final
+
+WORKDIR /app
+COPY /target/$RUST_TARGET/release/worker-executor-oss ./
+COPY /golem-worker-executor-oss/config/worker-executor.toml ./config/worker-executor.toml
+
+RUN apt-get update && apt-get install -y libssl-dev
+RUN apt-get update && apt-get install -y ca-certificates
+RUN update-ca-certificates
+
+EXPOSE 9000
+
+ENTRYPOINT ["./worker-executor-oss"]