Merge pull request #29 from omnia-network/lorenzo

ilbertt · web-flow · commit 366243f75b3a · 2024-03-21T17:40:35.000+01:00
Implementation of required Dashboards and Integration with Grafana Prometheus Cloud
diff --git a/.env.example b/.env.example
@@ -27,3 +27,7 @@ GRAFANA_TEMPO_ENDPOINT=https://your_tempo_endpoint
 # Set this to the access token obtained from Grafana Tempo
 GRAFANA_TEMPO_ACCESS_TOKEN=your_tempo_basic_auth_token
 GRAFANA_TEMPO_LOCAL=false
+
+GRAFANA_PROMETHEUS_ENDPOINT=https://your_grafana_prometheus_cloud_enpoint
+GRAFANA_PROMETHEUS_USERNAME=your_grafana_prometheus_cloud_username
+GRAFANA_PROMETHEUS_TOKEN=your_grafana_prometheus_cloud_token
diff --git a/.gitignore b/.gitignore
@@ -12,3 +12,5 @@ scripts/gateway_test.log
 
 .env
 .idea
+
+prometheus-prod.yml
diff --git a/README.md b/README.md
@@ -91,14 +91,24 @@ To run the gateway in a prod environment with Docker Compose, follow these steps
 
 2. To run the [docker-compose-prod.yml](./docker-compose-prod.yml) file you need a public domain (that you will put in the `DOMAIN_NAME` environment variable) and a TLS certificate for that domain (because it is configured to make the gateway run with TLS enabled). See [Obtain a TLS certificate](#obtain-a-tls-certificate) for more details.
 3. Open the `443` port (or the port that you set in the `LISTEN_PORT` environment variable) on your server and make it reachable from the Internet.
-4. Run the gateway:
-   
+4. To run all the required production structure you can execute the [start_prod_docker_environment.sh](./scripts/start_prod_docker_environment.sh) with the command:
+
+   ```
+    ./scripts/start_prod_docker_environment.sh 
+    ```
+
+   This script firstly generated the `telemetry/prometheus/prometheus-prod.yml` config file starting from the `telemetry/prometheus/prometheus-template.yml` (step required to perform the variable substitution) and then simply run the gateway with the following command:
+
     ```
-    docker compose -f docker-compose.yml -f docker-compose-prod.yml up
+    docker compose -f docker-compose.yml -f docker-compose-prod.yml --env-file .env up -d --build
     ```
 
-   This command, similar to the local one, allows you to run a docker compose generated from 2 different docker compose files.
-5. The Gateway will print its principal in the container logs, just as explained above.
+5. To stop and clean all the local environment a bash script [stop_prod_docker_environment.sh](./scripts/stop_prod_docker_environment.sh) is provided. You can execute it with the command:
+
+   ```
+    ./scripts/stop_prod_docker_environment.sh 
+    ```
+6. The Gateway will print its principal in the container logs, just as explained above.
 
 ### Obtain a TLS certificate
 
@@ -185,7 +195,11 @@ After installing Node.js and dfx, you can run the integration tests as follows:
     ```
     ./scripts/prepare_integration_tests.sh
     ```
-2. The `tests/.env` file should now contain some variables created by `dfx`. Add the variables from the [.env.example](./tests/.env.example) file to the `.env` file and leave them unchanged.
+2. Set the environment variables:
+    ```
+    cp tests/.env.example tests/.env
+    ```
+   When running the tests, the `tests/.env` file is modified by dfx, which will add some variables.
 3. Run integration tests using the Rust test canister:
     ```
     ./scripts/integration_test_rs.sh
diff --git a/docker-compose-local.yml b/docker-compose-local.yml
@@ -59,14 +59,8 @@ services:
       - ic-ws-gateway-network
 
   prometheus:
-    image: prom/prometheus
-    container_name: prometheus
-    ports:
-      - "9090:9090"
     volumes:
-      - "./telemetry/prometheus.yml:/etc/prometheus/prometheus.yml"
-    networks:
-      - ic-ws-gateway-network
+      - "./telemetry/prometheus/prometheus-local.yml:/etc/prometheus/prometheus.yml"
 
 networks:
   ic-ws-gateway-network:
diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml
@@ -21,3 +21,7 @@ services:
   otlp_collector:
     env_file:
       - .env
+
+  prometheus:
+    volumes:
+      - "./telemetry/prometheus/prometheus-prod.yml:/etc/prometheus/prometheus.yml"
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -22,6 +22,23 @@ services:
     networks:
       - ic-ws-gateway-network
 
+  # https://grafana.com/docs/grafana-cloud/send-data/metrics/metrics-prometheus/
+  prometheus:
+    image: prom/prometheus
+    # If you want to build from local customized Dockerfile enabling the config file (prometheus.yml) environment variable substitution directly
+    # inside the image and not outside it, you can use the provided Dockerfile image. (telemetry/prometheus/Dockerfile)
+#    image: local/prometheus
+#    build:
+#      context: ./
+#      dockerfile: ./telemetry/prometheus/Dockerfile
+    container_name: prometheus
+    ports:
+      - "9090:9090"
+    env_file:
+      - .env
+    networks:
+      - ic-ws-gateway-network
+
 networks:
   ic-ws-gateway-network:
     name: ic-ws-gateway-network
diff --git a/scripts/run_test_canister.sh b/scripts/run_test_canister.sh
@@ -7,5 +7,3 @@ dfx deploy test_canister_rs --no-wallet
 
 echo "Running integration test"
 npm run integration:test
-
-dfx stop
diff --git a/scripts/start_local_docker_environment.sh b/scripts/start_local_docker_environment.sh
@@ -3,7 +3,7 @@
 echo "Starting local replica"
 dfx start --clean --background
 
-echo "Removing previous local image"
+echo "Removing previous local websocket image"
 docker rmi local/ic-websocket-gateway
 
 echo "Starting docker environment"
diff --git a/scripts/start_prod_docker_environment.sh b/scripts/start_prod_docker_environment.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+echo "Creating the telemetry/prometheus/prometheus-prod.yml file"
+# Load and export variables from .env file
+set -a  # Automatically export all variables
+source .env
+set +a  # Stop automatically exporting
+
+# Run envsubst
+envsubst < ./telemetry/prometheus/prometheus-template.yml > ./telemetry/prometheus/prometheus-prod.yml
+echo "File telemetry/prometheus/prometheus-prod.yml successfully created"
+
+echo "Starting docker environment"
+docker compose -f docker-compose.yml -f docker-compose-prod.yml --env-file .env up -d
diff --git a/scripts/stop_prod_docker_environment.sh b/scripts/stop_prod_docker_environment.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+echo "Stopping docker environment"
+docker compose -f docker-compose.yml -f docker-compose-prod.yml down
diff --git a/src/gateway-state/src/lib.rs b/src/gateway-state/src/lib.rs
@@ -109,6 +109,18 @@ impl GatewayState {
         // therefore there is no need to do anything else here
     }
 
+    pub fn get_clients_count(&self, canister_id: CanisterPrincipal) -> usize {
+        if let Some(poller_state) = self.inner.data.get(&canister_id) {
+            poller_state.len()
+        } else {
+            0
+        }
+    }
+
+    pub fn get_active_pollers_count(&self) -> usize {
+        self.inner.data.len()
+    }
+
     /// SAFETY:
     ///
     /// The [Dashmap::entry](https://docs.rs/dashmap/5.5.3/src/dashmap/lib.rs.html#1147-1163) method gets a write lock on the whole shard in which the entry is.
diff --git a/src/ic-websocket-gateway/src/client_session_handler.rs b/src/ic-websocket-gateway/src/client_session_handler.rs
@@ -1,4 +1,3 @@
-use std::collections::HashMap;
 use crate::{
     canister_poller::CanisterPoller,
     client_session::{ClientSession, IcWsError, IcWsSessionState},
@@ -28,6 +27,8 @@ pub struct ClientSessionHandler {
     gateway_state: GatewayState,
     /// Polling interval in milliseconds
     polling_interval_ms: u64,
+    // Time to the start of the connection
+    start_connection_time: Instant,
 }
 
 impl ClientSessionHandler {
@@ -36,12 +37,14 @@ impl ClientSessionHandler {
         agent: Arc<Agent>,
         gateway_state: GatewayState,
         polling_interval_ms: u64,
+        start_connection_time: Instant,
     ) -> Self {
         Self {
             id,
             agent,
             gateway_state,
             polling_interval_ms,
+            start_connection_time,
         }
     }
 
@@ -109,7 +112,7 @@ impl ClientSessionHandler {
         mut client_channel_tx: Option<Sender<IcWsCanisterMessage>>,
         client_session_span: Span,
     ) -> Result<(), String> {
-        let mut clients_session_time: HashMap<ClientKey, Instant>  = HashMap::new();
+        let client_start_session_time = Instant::now();
 
         // keeps trying to update the client session state
         // if a new state is returned, execute the corresponding logic
@@ -186,27 +189,23 @@ impl ClientSessionHandler {
                     client_session_span.in_scope(|| {
                         debug!("Client session opened");
 
+                        let canister_id = self.get_canister_id(&client_session);
                         let client_key = self.get_client_key(&client_session);
 
-                        gauge!("clients_connected").increment(1.0);
+                        // Clients connection metrics
+                        let clients_connected = self.gateway_state.get_clients_count(canister_id);
+                        debug!("Clients connected: {}", clients_connected.to_string());
+                        gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64);
 
-                        clients_session_time.insert(client_key.clone(), Instant::now());
+                        // Calculate the time it took to open the connection and record it using the timer started in ws_listener.rs
+                        let delta = self.start_connection_time.elapsed();
+                        histogram!("connection_opening_time", "client_key" => client_key.to_string()).record(delta);
                     });
                     // do not return anything as the session is still alive
                 },
                 Ok(Some(IcWsSessionState::Closed)) => {
                     client_session_span.in_scope(|| {
                         debug!("Client session closed");
-
-                        let client_key = self.get_client_key(&client_session);
-
-                        gauge!("clients_connected").decrement(1.0);
-
-                        let value = clients_session_time.get(&client_key.clone());
-
-                        let delta = value.unwrap().elapsed();
-                        histogram!("connection_duration", "client_key" => client_key.to_string()).record(delta);
-
                     });
 
                     let canister_id = self.get_canister_id(&client_session);
@@ -216,6 +215,14 @@ impl ClientSessionHandler {
                         .remove_client(canister_id, client_key.clone());
                     debug!("Client removed from gateway state");
 
+                    // Clients connection metrics
+                    let clients_connected = self.gateway_state.get_clients_count(canister_id);
+                    debug!("Clients connected: {}", clients_connected.to_string());
+                    gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64);
+
+                    let delta = client_start_session_time.elapsed();
+                    histogram!("connection_duration", "client_key" => client_key.to_string()).record(delta);
+
                     self.call_ws_close(&canister_id, client_key).await;
 
                     // return Ok as the session was closed correctly
@@ -246,6 +253,15 @@ impl ClientSessionHandler {
                         .remove_client_if_exists(canister_id, client_key)
                     {
                         debug!("Client removed from gateway state");
+
+                        // Clients connection metrics
+                        let clients_connected = self.gateway_state.get_clients_count(canister_id);
+                        debug!("Clients connected: {}", clients_connected.to_string());
+                        gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64);
+
+                        let delta = client_start_session_time.elapsed();
+                        histogram!("connection_duration", "client_key" => client_key.to_string()).record(delta);
+
                         self.call_ws_close(&canister_id, client_key).await;
 
                         // return Err as the session had an error and cannot be updated anymore
@@ -301,6 +317,11 @@ impl ClientSessionHandler {
         let agent = Arc::clone(&self.agent);
         let gateway_state = self.gateway_state.clone();
         let polling_interval_ms = self.polling_interval_ms;
+
+        let active_pollers = gateway_state.get_active_pollers_count();
+        debug!("Active pollers: {}", active_pollers.to_string());
+        gauge!("active_pollers").set(active_pollers as f64);
+
         tokio::spawn(async move {
             // we pass both the whole gateway state and the poller state for the specific canister
             // the poller can access the poller state to determine which clients are connected
@@ -312,7 +333,7 @@ impl ClientSessionHandler {
                 agent,
                 canister_id,
                 poller_state,
-                gateway_state,
+                gateway_state.clone(),
                 polling_interval_ms,
             );
             if let Err(e) = poller.run_polling().await {
@@ -326,6 +347,10 @@ impl ClientSessionHandler {
             // the poller takes care of notifying the session handlers when an error is detected
             // and removing its corresponding entry from the gateway state
             // therefore, this task can simply terminate without doing anything
+
+            let active_pollers = gateway_state.get_active_pollers_count();
+            debug!("Active pollers: {}", active_pollers.to_string());
+            gauge!("active_pollers").set(active_pollers as f64);
         });
     }
 }
diff --git a/src/ic-websocket-gateway/src/gateway_metrics.rs b/src/ic-websocket-gateway/src/gateway_metrics.rs
@@ -8,21 +8,32 @@ use metrics_util::{MetricKindMask};
 pub fn init_metrics(port: Option<u16>) -> Result<(), Box<dyn Error>> {
     let builder = PrometheusBuilder::new()
         .with_http_listener(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port.unwrap_or(9000)));
+
+    // Set the idle timeout for counters and histograms to 30 seconds then the metrics are removed from the registry
     builder
         .idle_timeout(
             MetricKindMask::ALL,
-            Some(Duration::from_secs(10)),
+            Some(Duration::from_secs(30)),
         )
         .install()
         .expect("failed to install Prometheus recorder");
 
     describe_gauge!("clients_connected", "The number of clients currently connected");
+    describe_gauge!("active_pollers", "The number of pollers currently active");
     describe_histogram!(
         "connection_duration",
         "The duration of the client connection"
     );
+    describe_histogram!(
+        "tls_resolution_time",
+        "The resolution time of the TLS handshake"
+    );
+    describe_histogram!(
+        "connection_opening_time",
+        "The time it takes to open a connection"
+    );
 
-    gauge!("clients_connected").set(0.0);
+    gauge!("active_pollers").set(0.0);
 
     Ok(())
 }
diff --git a/src/ic-websocket-gateway/src/ws_listener.rs b/src/ic-websocket-gateway/src/ws_listener.rs
diff --git a/telemetry/dashboards/ic_metrics.json b/telemetry/dashboards/ic_metrics.json
diff --git a/telemetry/prometheus/Dockerfile b/telemetry/prometheus/Dockerfile
diff --git a/telemetry/prometheus/prometheus-local.yml b/telemetry/prometheus/prometheus-local.yml
diff --git a/telemetry/prometheus/prometheus-template.yml b/telemetry/prometheus/prometheus-template.yml
diff --git a/tests/src/integration/client.test.ts b/tests/src/integration/client.test.ts

-Original file line number
+Diff line change
 .env
 .idea
++
 +prometheus-prod.yml