http190 improve completion code for response parse errors

Signed-off-by: davidradl <[email protected]>

Author: davidradl

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+- Add UNABLE_TO_DESERIALIZE_RESPONSE http-completion-state. If you have used 
+`gid.connector.http.source.lookup.continue-on-error` in a previous release, please review your SQL or applications to 
+account for this new http-completion-state value. 
+
 ## [0.23.0] - 2025-11-07
 
 -   Ability to specify http versions for http lookups. 

README.md

@@ -190,20 +190,24 @@ Metadata columns can be specified and hold http information. They are optional r
 
 | Key                   | Data Type                        | Description                            |
 |-----------------------|----------------------------------|----------------------------------------|
-| error-string          | STRING NULL                      | A message associated with the error    |
+| error-string          | STRING NULL                      | A string associated with the error     |
 | http-status-code      | INT NULL                         | The HTTP status code                   |
 | http-headers-map      | MAP <STRING, ARRAY<STRING>> NULL | The headers returned with the response |
 | http-completion-state | STRING NULL                      | The completion state of the http call. |
 
 ##### http-completion-state possible values
 
-| Value             | Description            |
-|:------------------|------------------------|
-| SUCCESS           | Success                |
-| HTTP_ERROR_STATUS | HTTP error status code |
-| EXCEPTION         | An Exception occurred  |
+| Value                          | Description                         |
+|:-------------------------------|-------------------------------------|
+| SUCCESS                        | Success                             |
+| HTTP_ERROR_STATUS              | HTTP error status code              |
+| EXCEPTION                      | An Exception occurred               |
+| UNABLE_TO_DESERIALIZE_RESPONSE | Unable to deserialize HTTP response |
 
 If the `error-string` metadata column is defined on the table and the call succeeds then it will have a null value.
+When the HTTP response cannot be deserialized, then the `http-completion-state` will be `UNABLE_TO_DESERIALIZE_RESPONSE`
+and the `error-string` will be the response body. Note that `UNABLE_TO_DESERIALIZE_RESPONSE` is a new enum value added
+since [0.22.0], please ensure you amend your applications and SQL appropriately. 
 
 When a http lookup call fails and populates the metadata columns with the error information, the expected enrichment columns from the http call
 are not populated, this means that they will be null for nullable columns and hold a default value for the type for non-nullable columns. 

src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpCompletionState.java

@@ -3,5 +3,6 @@
 public enum HttpCompletionState {
     HTTP_ERROR_STATUS,
     EXCEPTION,
-    SUCCESS
+    SUCCESS,
+    UNABLE_TO_DESERIALIZE_RESPONSE
 }

src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java

@@ -235,8 +235,8 @@ private HttpRowDataWrapper processHttpResponse(
                     rowData = deserialize(responseBody);
                 } catch (IOException e) {
                     if (!this.continueOnError) throw e;
-                    httpCompletionState = HttpCompletionState.EXCEPTION;
-                    errMessage = e.getMessage();
+                    httpCompletionState = HttpCompletionState.UNABLE_TO_DESERIALIZE_RESPONSE;
+                    errMessage = responseBody;
                 }
                 return HttpRowDataWrapper.builder()
                         .data(rowData)

src/test/java/com/getindata/connectors/http/internal/table/lookup/HttpLookupTableSourceITCaseTest.java

@@ -78,6 +78,7 @@ class HttpLookupTableSourceITCaseTest {
 
         return row1Id.compareTo(row2Id);
     };
+    public static final String A_TEST_STRING_THAT_IS_NOT_JSON = "A test string that is not json";
 
     private StreamTableEnvironment tEnv;
 
@@ -1081,10 +1082,11 @@ private void  assertEnrichedRowsDeserException(Collection<Row> collectedRows ) {
                 assertThat(row.getField("balance")).isNull();
                 // metadata
                 assertThat(row.getField("errStr"))
-                    .isEqualTo("Failed to deserialize JSON 'A test string that is not json'.");
+                    .isEqualTo(A_TEST_STRING_THAT_IS_NOT_JSON);
                 assertThat(row.getField("headers")).isNotNull();
                 assertThat(row.getField("statusCode")).isEqualTo(200);
-                assertEquals(row.getField("completionState"), HttpCompletionState.EXCEPTION.name());
+                assertEquals(row.getField("completionState"), HttpCompletionState.UNABLE_TO_DESERIALIZE_RESPONSE
+                    .name());
             }
         }
         );
@@ -1393,7 +1395,7 @@ private void setupServerStubForSpec(TestSpec spec) {
             if (StringUtils.isNullOrWhitespaceOnly(spec.methodName) || spec.methodName.equalsIgnoreCase("GET")) {
                 wireMockServer.stubFor(get(urlPathEqualTo(ENDPOINT))
                         .withHeader("Content-Type", equalTo("application/json"))
-                        .willReturn(aResponse().withBody("A test string that is not json").withStatus(200))
+                        .willReturn(aResponse().withBody(A_TEST_STRING_THAT_IS_NOT_JSON).withStatus(200))
                 );
             } else {
                 setUpServerBodyStub(