Merge pull request #45 from omnia-network/tests/fix-load-tests

tests: fix load tests

Author: ilbertt

.github/actions/setup-dfx/action.yml

@@ -0,0 +1,10 @@
+name: Setup DFX
+description: Setup DFX
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Setup DFX
+      uses: dfinity/setup-dfx@main
+      with:
+        dfx-version: 0.24.1

.github/actions/setup-node/action.yml

@@ -0,0 +1,11 @@
+name: Setup Node.js
+description: Setup Node.js
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 20
+        cache: "npm"
+        cache-dependency-path: "tests/package-lock.json"

.github/workflows/integration-tests.yml

@@ -1,6 +1,5 @@
 name: Integration tests
 
-# only run when a commit is pushed to the main branch
 on:
   push:
     branches:
@@ -20,15 +19,9 @@ jobs:
         with:
           cache-key: "build-native"
 
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 20
-          cache: "npm"
-          cache-dependency-path: "tests/package-lock.json"
+      - uses: ./.github/actions/setup-node
 
-      - uses: dfinity/setup-dfx@main
-        with:
-          dfx-version: 0.24.1
+      - uses: ./.github/actions/setup-dfx
 
       - name: Install mops
         run: npm i ic-mops -g
@@ -37,7 +30,7 @@ jobs:
         run: npx mocv use latest
 
       - name: Prepare environment for integration tests
-        run: ./scripts/prepare_integration_tests.sh
+        run: ./scripts/prepare_tests.sh
 
       - name: Run integration tests
         run: ./scripts/ci_cd_test_integration.sh

.github/workflows/load-tests.yml

@@ -0,0 +1,37 @@
+name: Load tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  load-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - uses: ./.github/actions/setup-rust
+        with:
+          cache-key: "build-native"
+
+      - uses: ./.github/actions/setup-node
+
+      - uses: ./.github/actions/setup-dfx
+
+      - name: Prepare environment for load tests
+        run: ./scripts/prepare_tests.sh
+
+      - name: Execute load tests
+        run: ./scripts/ci_cd_test_load.sh
+
+      - name: Upload load test results
+        if: always()  # This ensures the upload happens even if tests fail
+        uses: actions/upload-artifact@v4
+        with:
+          name: load-test-results
+          path: tests/reports/gateway_load_tests.json

.github/workflows/rust-tests.yml

@@ -1,6 +1,5 @@
 name: Rust tests
 
-# only run when a commit is pushed to the main branch
 on:
   push:
     branches:

README.md

@@ -19,10 +19,13 @@ Make sure you have the **Rust toolchain** installed. You can find instructions [
 1. Run the gateway:
 
     In **debug** mode:
+
     ```bash
     cargo run
     ```
+
     In **release** mode:
+
     ```bash
     cargo build --release
 
@@ -77,6 +80,7 @@ docker run -p 8080:8080 omniadevs/ic-websocket-gateway --ic-network-url http://h
 ```
 
 > Note: if you're on an ARM machine, you have to add the `--platform` flag to the command above, since the published image is built only for the `x86_64` architecture:
+>
 > ```bash
 > docker run --platform linux/amd64 -p 8080:8080 omniadevs/ic-websocket-gateway --ic-network-url http://host.docker.internal:4943
 > ```
@@ -89,9 +93,9 @@ Make sure you have [Docker Compose](https://docs.docker.com/compose/install/) in
 
 The following compose files are available:
 
-- [docker-compose.yml](./docker-compose.yml)
-- [docker-compose-local.yml](./docker-compose-local.yml)
-- [docker-compose-prod.yml](./docker-compose-prod.yml)
+-   [docker-compose.yml](./docker-compose.yml)
+-   [docker-compose-local.yml](./docker-compose-local.yml)
+-   [docker-compose-prod.yml](./docker-compose-prod.yml)
 
 The following sections describe how to use the different compose files to run the gateway with Docker Compose.
 
@@ -100,36 +104,34 @@ A visual representation of the containers in the compose files is provided in th
 ### Local
 
 To run the gateway in a local environment with Docker Compose, follow these steps:
+
 1. To run all the required local structure you can execute the [start_local_docker_environment.sh](./scripts/start_local_docker_environment.sh) with the command:
-    
-   ```
-    ./scripts/start_local_docker_environment.sh 
+
+    ```
+     ./scripts/start_local_docker_environment.sh
     ```
 
     This script simply run a dfx local replica and execute the gateway with the following command:
 
     ```
     docker compose -f docker-compose.yml -f docker-compose-local.yml --env-file .env.local up -d --build
     ```
-   
+
 2. To stop and clean all the local environment, the bash script [stop_local_docker_environment.sh](./scripts/stop_local_docker_environment.sh) is provided. You can execute it with the command:
-    
-   ```
-    ./scripts/stop_local_docker_environment.sh 
+
+    ```
+     ./scripts/stop_local_docker_environment.sh
     ```
 
 3. The Gateway will print its principal in the container logs, just as explained in the [Standalone](#standalone) section.
-4. If you want to verify that everything started correctly, the bash script [run_test_canister.sh](./scripts/run_test_canister.sh) is provided. This script assumes that the gateway is already running and reachable locally. You can execute it with the command:
-    
-   ```
-    ./scripts/run_test_canister.sh
-    ```
+4. You can verify that things are working properly by running the tests, see [Testing](#Testing)
 
 ### Production
 
 This configuration uses the [omniadevs/ic-websocket-gateway](https://hub.docker.com/r/omniadevs/ic-websocket-gateway) image.
 
 To run the gateway in a production environment with Docker Compose, follow these steps:
+
 1. Set the environment variables:
 
     ```
@@ -140,21 +142,22 @@ To run the gateway in a production environment with Docker Compose, follow these
 3. Open the `443` port (or the port that you set in the `LISTEN_PORT` environment variable) on your server and make it reachable from the Internet.
 4. To run all the required production containers you can execute the [start_prod_docker_environment.sh](./scripts/start_prod_docker_environment.sh) with the command:
 
-   ```
-    ./scripts/start_prod_docker_environment.sh 
+    ```
+     ./scripts/start_prod_docker_environment.sh
     ```
 
-   This script first generates the `telemetry/prometheus/prometheus-prod.yml` config file from the `telemetry/prometheus/prometheus-template.yml` template (step required to perform the environment variables substitution) and then runs the gateway with the following command:
+    This script first generates the `telemetry/prometheus/prometheus-prod.yml` config file from the `telemetry/prometheus/prometheus-template.yml` template (step required to perform the environment variables substitution) and then runs the gateway with the following command:
 
     ```
     docker compose -f docker-compose.yml -f docker-compose-prod.yml --env-file .env up -d
     ```
 
 5. To stop and clean the containers, the bash script [stop_prod_docker_environment.sh](./scripts/stop_prod_docker_environment.sh) is provided. You can execute it with the command:
 
-   ```
-    ./scripts/stop_prod_docker_environment.sh 
     ```
+     ./scripts/stop_prod_docker_environment.sh
+    ```
+
 6. The Gateway will print its principal in the container logs, just as explained in the [Standalone](#standalone) section.
 
 #### Obtain a TLS certificate
@@ -198,13 +201,15 @@ The gateway uses the [opentelemetry](https://docs.rs/opentelemetry) crate and [G
 If you're deploying the gateway with Docker (see the [Docker](#docker) section), make sure you set the following varibales in the `.env` file:
 
 **Local**:
+
 ```
 OPENTELEMETRY_COLLECTOR_ENDPOINT=grpc://otlp_collector:4317
 GRAFANA_TEMPO_ENDPOINT=tempo:4318
 GRAFANA_TEMPO_LOCAL=true
 ```
 
 **Production**:
+
 ```
 OPENTELEMETRY_COLLECTOR_ENDPOINT=grpc://otlp_collector:4317
 GRAFANA_TEMPO_ENDPOINT=your-grafana-cloud-tempo-endpoint
@@ -240,19 +245,21 @@ After installing Node.js and dfx, you can run the integration tests as follows:
 
 1. Prepare the test environment by installing dependencies and building the components by running the following command:
     ```
-    ./scripts/prepare_integration_tests.sh
+    ./scripts/prepare_tests.sh
     ```
 2. Set the environment variables:
     ```
     cp tests/.env.example tests/.env
     ```
-   When running the tests, the `tests/.env` file is modified by dfx, which will add some variables.
+    When running the tests, the `tests/.env` file will be modified by dfx to add some variables.
 3. Run integration tests using the Rust test canister:
+
     ```
     ./scripts/integration_test_rs.sh
     ```
 
-    If you instead want to run tests using the Motoko test canister, run the following command instead:
+    If you want to run tests using the Motoko test canister, run the following command instead:
+
     ```
     ./scripts/integration_test_mo.sh
     ```
@@ -263,21 +270,21 @@ Tests canisters used in the integration tests can be found in the [tests/src/tes
 
 ### Local test script
 
-After setting up and running the tests for the first time following the steps above, you can use the following command to run the unit and integration tests (using Rust test canister) together:
+After setting up and running the tests for the first time following the steps above, you can use the following command to run the unit and integration tests (using both the Rust and Motoko test canisters):
 
 ```
 ./scripts/local_test.sh
 ```
 
 ### Load tests
 
-Load tests are provided in the [tests/src/load](./tests/src/load/) folder. You can run them with:
+We use [Artillery](https://github.com/artilleryio/artillery) to run load tests. The load test configuration is provided in the [tests/gateway_load_tests.yml](./tests/gateway_load_tests.yml) file and the source code is provided in the [tests/src/load](./tests/src/load/) folder. You can run them with:
 
 ```
 ./scripts/run_load_test.sh
 ```
 
-This script requires you to set up the test environment manually, because you usually want to keep an eye on the logs of the different components. You have to start the local replica, start the gateway and deploy the test canister. The [scripts/integration_test_rs.sh](./scripts/integration_test_rs.sh) is a good reference for how to do that.
+This script requires you to set up the test environment manually, because you usually want to keep an eye on the logs of the different components. You have to start the local replica, start the gateway and deploy the test canister. The [scripts/ci_cd_test_load.sh](./scripts/ci_cd_test_load.sh) is a good reference for how to do that.
 
 # How it works
 

scripts/ci_cd_test_integration.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-# requires running the prepare_integration_tests.sh script first
+# requires running the prepare_tests.sh script first
 
 echo "Starting local replica..."
 dfx start --clean --background

scripts/ci_cd_test_load.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+set -e
+
+# requires running the prepare_tests.sh script first
+
+echo "Starting local replica..."
+dfx start --clean > /dev/null 2>&1 &
+
+echo "Waiting for replica to start..."
+sleep 10
+
+echo "Starting gateway in the background..."
+cargo run --release > /dev/null 2>&1 &
+gateway_pid=$!
+
+echo "Waiting for gateway to start..."
+sleep 1
+
+export DFX_NETWORK=local
+export WS_GATEWAY_URL=ws://127.0.0.1:8080
+export IC_URL=http://127.0.0.1:4943
+export FETCH_IC_ROOT_KEY=true
+
+echo "Deploying test canister (Rust)..."
+cd tests
+dfx deploy test_canister_rs --no-wallet
+# generate test_canister JS declarations,
+# which will be used for both Rust and Motoko tests
+npm run generate:test_canister_rs
+
+# Compile load test script
+echo "Compiling load test script..."
+npm run load:bundle
+
+# Create reports directory if it doesn't exist
+mkdir -p reports
+
+# Run load tests
+echo "Running load tests..."
+LOG_LEVEL=error npx artillery run gateway_load_tests.yml --output reports/gateway_load_tests.json
+
+echo "Results:"
+cat reports/gateway_load_tests.json
+
+# Check that no users have failed
+echo -e "\nChecking that no users have failed..."
+failed=$(jq '.aggregate.counters."vusers.failed"' reports/gateway_load_tests.json)
+if [ "$failed" != 0 ]; then
+    echo "ERROR: Load test failed with $failed users failing."
+    exit 1
+else
+    echo "All users were successful."
+fi
+
+echo -e "\nStopping gateway..."
+kill $gateway_pid
+
+echo "Stopping local replica..."
+dfx stop

scripts/integration_test_mo.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+# requires running the prepare_tests.sh script first
+
 echo "Starting local replica"
 dfx start --clean --background
 
@@ -9,6 +11,7 @@ pid=$!
 
 echo "Deploying Motoko test canister"
 cd tests
+npx mops install
 npm run generate:test_canister_mo
 dfx deploy test_canister_mo --no-wallet
 

scripts/integration_test_rs.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+# requires running the prepare_tests.sh script first
+
 echo "Starting local replica"
 dfx start --clean --background