Add FFE SDK resilience test for RC unavailability

leoromanovsky · leoromanovsky · commit 51c0eae07145 · 2025-11-26T13:54:33.000-08:00
- Add StaticErrorMockedResponse class for simulating error responses
- Add Test_FFE_RC_Unavailable test class
- Register test in all manifest files
diff --git a/manifests/cpp.yml b/manifests/cpp.yml
@@ -97,5 +97,6 @@ tests/:
     Test_FFE_Exposure_Events: missing_feature
     Test_FFE_Exposure_Events_Empty: missing_feature
     Test_FFE_Exposure_Events_Errors: missing_feature
+    Test_FFE_RC_Unavailable: missing_feature
   test_telemetry.py:
     Test_TelemetryEnhancedConfigReporting: missing_feature
diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml
@@ -78,6 +78,7 @@ tests/:
     Test_FFE_Exposure_Events: missing_feature
     Test_FFE_Exposure_Events_Empty: missing_feature
     Test_FFE_Exposure_Events_Errors: missing_feature
+    Test_FFE_RC_Unavailable: missing_feature
   test_graphql.py: missing_feature
   test_identify.py: irrelevant
   test_ipv6.py: missing_feature (APMAPI-869)
diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml
@@ -277,6 +277,7 @@ tests/:
     Test_FFE_Exposure_Events: missing_feature
     Test_FFE_Exposure_Events_Empty: missing_feature
     Test_FFE_Exposure_Events_Errors: missing_feature
+    Test_FFE_RC_Unavailable: missing_feature
   test_graphql.py: missing_feature
   test_identify.py: irrelevant
   test_ipv6.py: missing_feature (APMAPI-869)
diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml
@@ -695,6 +695,7 @@ tests/:
     Test_FFE_Exposure_Events: missing_feature
     Test_FFE_Exposure_Events_Empty: missing_feature
     Test_FFE_Exposure_Events_Errors: missing_feature
+    Test_FFE_RC_Unavailable: missing_feature
   test_graphql.py:
     Test_GraphQLOperationErrorReporting:
       '*': missing_feature
diff --git a/manifests/golang.yml b/manifests/golang.yml
@@ -885,6 +885,7 @@ tests/:
     Test_FFE_Exposure_Events: v2.5.0-dev
     Test_FFE_Exposure_Events_Empty: v2.5.0-dev
     Test_FFE_Exposure_Events_Errors: v2.5.0-dev
+    Test_FFE_RC_Unavailable: v2.4.0
   test_graphql.py:
     Test_GraphQLOperationErrorReporting:
       '*': irrelevant
diff --git a/manifests/java.yml b/manifests/java.yml
@@ -2334,6 +2334,7 @@ tests/:
     Test_FFE_Exposure_Events: missing_feature
     Test_FFE_Exposure_Events_Empty: missing_feature
     Test_FFE_Exposure_Events_Errors: missing_feature
+    Test_FFE_RC_Unavailable: v1.56.0
   test_graphql.py:
     Test_GraphQLOperationErrorReporting:
       '*': missing_feature
diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml
@@ -1475,6 +1475,9 @@ tests/:
     Test_FFE_Exposure_Events_Errors:
       '*': incomplete_test_app
       express4: *ref_5_77_0  # only target express 4
+    Test_FFE_RC_Unavailable:
+      '*': incomplete_test_app
+      express4: *ref_5_77_0  # only target express 4
   test_graphql.py:
     Test_GraphQLOperationErrorReporting:
       '*': irrelevant
diff --git a/manifests/php.yml b/manifests/php.yml
@@ -688,6 +688,7 @@ tests/:
     Test_FFE_Exposure_Events: missing_feature
     Test_FFE_Exposure_Events_Empty: missing_feature
     Test_FFE_Exposure_Events_Errors: missing_feature
+    Test_FFE_RC_Unavailable: missing_feature
   test_graphql.py:
     Test_GraphQLOperationErrorReporting:
       '*': missing_feature
diff --git a/manifests/python.yml b/manifests/python.yml
@@ -1279,6 +1279,7 @@ tests/:
     Test_FFE_Exposure_Events: v4.0.0
     Test_FFE_Exposure_Events_Empty: v4.0.0
     Test_FFE_Exposure_Events_Errors: v4.0.0
+    Test_FFE_RC_Unavailable: v4.0.0
   test_graphql.py:
     Test_GraphQLOperationErrorReporting:
       '*': missing_feature
diff --git a/manifests/ruby.yml b/manifests/ruby.yml
@@ -882,6 +882,7 @@ tests/:
     Test_FFE_Exposure_Events: missing_feature
     Test_FFE_Exposure_Events_Empty: missing_feature
     Test_FFE_Exposure_Events_Errors: missing_feature
+    Test_FFE_RC_Unavailable: missing_feature
   test_graphql.py:
     Test_GraphQLOperationErrorReporting:
       "*": missing_feature
diff --git a/manifests/rust.yml b/manifests/rust.yml
@@ -96,4 +96,5 @@ tests/:
     Test_FFE_Exposure_Events: missing_feature
     Test_FFE_Exposure_Events_Empty: missing_feature
     Test_FFE_Exposure_Events_Errors: missing_feature
+    Test_FFE_RC_Unavailable: missing_feature
   test_v1_payloads.py: missing_feature
diff --git a/tests/test_feature_flag_exposures.py b/tests/test_feature_flag_exposures.py
@@ -1,12 +1,15 @@
 """Test Feature Flag Exposure (FFE) exposure events in weblog end-to-end scenario."""
 
+import json
+
 from utils import (
     weblog,
     interfaces,
     scenarios,
     features,
     remote_config as rc,
 )
+from utils.proxy.mocked_response import StaticErrorMockedResponse, StaticJsonMockedResponse
 
 
 RC_PRODUCT = "FFE_FLAGS"
@@ -428,3 +431,89 @@ def test_ffe_malformed_remote_config_rejection(self):
             assert event["subject"]["id"] == self.targeting_key, (
                 f"Expected subject '{self.targeting_key}', got '{event['subject']['id']}'"
             )
+
+
+@scenarios.feature_flag_exposure
+@features.feature_flag_exposure
+class Test_FFE_RC_Unavailable:
+    """Test FFE SDK resilience when the Remote Configuration service becomes unavailable.
+
+    This test validates that the local flag configuration cache inside the tracer
+    can continue to perform evaluations even when RC returns errors.
+    """
+
+    def setup_ffe_rc_unavailable_graceful_degradation(self):
+        """Set up FFE with valid config, then simulate RC unavailability and verify cached config still works."""
+        rc.rc_state.reset().apply()
+
+        self.flag_key = "test-flag"  # From UFC_FIXTURE_DATA, returns "on"
+        self.not_delivered_flag_key = "test-flag-not-delivered"
+        self.default_value = "default_fallback"
+
+        config_state = rc.rc_state.set_config(f"{RC_PATH}/ffe-test/config", UFC_FIXTURE_DATA).apply()
+        assert config_state.state == rc.ApplyState.ACKNOWLEDGED, f"RC config not acknowledged: {config_state.state}"
+
+        # Baseline: evaluate flag with RC working
+        self.baseline_eval = weblog.post(
+            "/ffe",
+            json={
+                "flag": self.flag_key,
+                "variationType": "STRING",
+                "defaultValue": self.default_value,
+                "targetingKey": "user-1",
+                "attributes": {},
+            },
+        )
+
+        # Simulate RC unavailability by returning 503 errors
+        StaticErrorMockedResponse(path="/v0.7/config", status_code=503).send()
+
+        # Evaluate cached flag while RC is unavailable
+        self.cached_eval = weblog.post(
+            "/ffe",
+            json={
+                "flag": self.flag_key,
+                "variationType": "STRING",
+                "defaultValue": self.default_value,
+                "targetingKey": "user-2",
+                "attributes": {},
+            },
+        )
+
+        # Evaluate a flag that was not delivered via RC
+        self.not_delivered_eval = weblog.post(
+            "/ffe",
+            json={
+                "flag": self.not_delivered_flag_key,
+                "variationType": "STRING",
+                "defaultValue": self.default_value,
+                "targetingKey": "user-3",
+                "attributes": {},
+            },
+        )
+
+        # Restore normal RC behavior (empty response)
+        StaticJsonMockedResponse(path="/v0.7/config", mocked_json={}).send()
+
+    def test_ffe_rc_unavailable_graceful_degradation(self):
+        """Test that cached flag configs continue working when RC is unavailable."""
+        expected_value = "on"
+
+        assert self.baseline_eval.status_code == 200, f"Baseline request failed: {self.baseline_eval.text}"
+        assert self.cached_eval.status_code == 200, f"Cached eval request failed: {self.cached_eval.text}"
+        assert self.not_delivered_eval.status_code == 200, f"Not delivered eval failed: {self.not_delivered_eval.text}"
+
+        baseline_result = json.loads(self.baseline_eval.text)
+        assert baseline_result["value"] == expected_value, (
+            f"Baseline: expected '{expected_value}', got '{baseline_result['value']}'"
+        )
+
+        cached_result = json.loads(self.cached_eval.text)
+        assert cached_result["value"] == expected_value, (
+            f"Cached eval during RC outage: expected '{expected_value}' from cache, got '{cached_result['value']}'"
+        )
+
+        not_delivered_result = json.loads(self.not_delivered_eval.text)
+        assert not_delivered_result["value"] == self.default_value, (
+            f"Not delivered flag: expected default '{self.default_value}', got '{not_delivered_result['value']}'"
+        )
diff --git a/utils/proxy/mocked_response.py b/utils/proxy/mocked_response.py
@@ -50,6 +50,8 @@ def build_from_json(source: dict) -> "MockedResponse":
             return MockedResponse.from_json(source)
         if mocked_response_type == SequentialRemoteConfigJsonMockedResponse.__name__:
             return SequentialRemoteConfigJsonMockedResponse.from_json(source)
+        if mocked_response_type == StaticErrorMockedResponse.__name__:
+            return StaticErrorMockedResponse.from_json(source)
 
         raise ValueError(f"Unknown MockedResponse type: {mocked_response_type}")
 
@@ -132,6 +134,30 @@ def to_json(self) -> dict:
         }
 
 
+class StaticErrorMockedResponse(MockedResponse):
+    """Returns an error status code for requests to the given path"""
+
+    def __init__(self, path: str, status_code: int = 503, error_message: str = "Service Unavailable"):
+        super().__init__(path=path, mocked_headers={"Content-Type": "application/json"})
+        self.status_code = status_code
+        self.error_message = error_message
+
+    def _apply_status_code(self, flow: HTTPFlow) -> None:
+        flow.response.status_code = self.status_code
+
+    def execute(self, flow: HTTPFlow) -> None:
+        super().execute(flow)
+        flow.response.content = json.dumps({"error": self.error_message}).encode()
+
+    def to_json(self) -> dict:
+        return {
+            "type": self.__class__.__name__,
+            "path": self.path,
+            "status_code": self.status_code,
+            "error_message": self.error_message,
+        }
+
+
 class _InternalMockedResponse(MockedResponse):
     def send(self) -> None:
         raise ValueError("This mocked response cannot be sent directly")
