refactor: make unmarshalling easier and override json to RawMessage

go/apps/api/routes/v2_apis_list_keys/handler.go

@@ -2,7 +2,6 @@ package handler
 
 import (
 	"context"
-	"encoding/json"
 	"net/http"
 
 	"github.com/oapi-codegen/nullable"
@@ -317,9 +316,9 @@ func (h *Handler) buildKeyResponseData(keyData *db.KeyData, plaintext string) op
 		}
 
 		if len(keyData.Identity.Meta) > 0 {
-			var identityMeta map[string]any
-			_ = json.Unmarshal(keyData.Identity.Meta, &identityMeta) // Ignore error, default to nil
-			if identityMeta != nil {
+			if identityMeta, err := db.UnmarshalNullableJSONTo[map[string]any](keyData.Identity.Meta); err != nil {
+				h.Logger.Error("failed to unmarshal identity meta", "error", err)
+			} else {
 				response.Identity.Meta = &identityMeta
 			}
 		}
@@ -386,9 +385,13 @@ func (h *Handler) buildKeyResponseData(keyData *db.KeyData, plaintext string) op
 
 	// Set meta
 	if keyData.Key.Meta.Valid {
-		var meta map[string]any
-		_ = json.Unmarshal([]byte(keyData.Key.Meta.String), &meta) // Ignore error, default to nil
-		if meta != nil {
+		meta, err := db.UnmarshalNullableJSONTo[map[string]any](keyData.Key.Meta.String)
+		if err != nil {
+			h.Logger.Error("failed to unmarshal key meta",
+				"keyId", keyData.Key.ID,
+				"error", err,
+			)
+		} else {
 			response.Meta = &meta
 		}
 	}

go/apps/api/routes/v2_identities_create_identity/200_test.go

@@ -138,8 +138,7 @@ func TestCreateIdentitySuccessfully(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, identity.ExternalID, req.ExternalId)
 
-		var dbMeta map[string]any
-		err = json.Unmarshal(identity.Meta, &dbMeta)
+		dbMeta, err := db.UnmarshalNullableJSONTo[map[string]any](identity.Meta)
 		require.NoError(t, err)
 		require.Equal(t, *meta, dbMeta)
 	})
@@ -239,8 +238,7 @@ func TestCreateIdentitySuccessfully(t *testing.T) {
 		require.Equal(t, identity.ExternalID, req.ExternalId)
 
 		// Verify metadata
-		var dbMeta map[string]any
-		err = json.Unmarshal(identity.Meta, &dbMeta)
+		dbMeta, err := db.UnmarshalNullableJSONTo[map[string]any](identity.Meta)
 		require.NoError(t, err)
 		require.Equal(t, *meta, dbMeta)
 
@@ -315,8 +313,7 @@ func TestCreateIdentitySuccessfully(t *testing.T) {
 		require.Equal(t, identity.ExternalID, req.ExternalId)
 
 		// Verify complex metadata is correctly stored and retrieved
-		var dbMeta map[string]any
-		err = json.Unmarshal(identity.Meta, &dbMeta)
+		dbMeta, err := db.UnmarshalNullableJSONTo[map[string]any](identity.Meta)
 		require.NoError(t, err)
 
 		// Convert expected and actual to JSON strings for comparison to handle potential subtle differences in map types
@@ -425,8 +422,7 @@ func TestCreateIdentitySuccessfully(t *testing.T) {
 		require.NoError(t, err)
 
 		// Verify metadata
-		var dbMeta map[string]any
-		err = json.Unmarshal(identity.Meta, &dbMeta)
+		dbMeta, err := db.UnmarshalNullableJSONTo[map[string]any](identity.Meta)
 		require.NoError(t, err)
 
 		// Convert to JSON for comparison

go/apps/api/routes/v2_identities_get_identity/handler.go

@@ -2,7 +2,6 @@ package handler
 
 import (
 	"context"
-	"encoding/json"
 	"net/http"
 
 	"github.com/unkeyed/unkey/go/apps/api/openapi"
@@ -74,9 +73,12 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
 	identity := results[0]
 
 	// Parse ratelimits JSON
-	var ratelimits []db.RatelimitInfo
-	if ratelimitBytes, ok := identity.Ratelimits.([]byte); ok && ratelimitBytes != nil {
-		_ = json.Unmarshal(ratelimitBytes, &ratelimits) // Ignore error, default to empty array
+	ratelimits, err := db.UnmarshalNullableJSONTo[[]db.RatelimitInfo](identity.Ratelimits)
+	if err != nil {
+		h.Logger.Error("failed to unmarshal ratelimits",
+			"identityId", identity.ID,
+			"error", err,
+		)
 	}
 
 	// Check permissions using either wildcard or the specific identity ID
@@ -96,17 +98,12 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
 		return err
 	}
 
-	// Parse metadata
-	var metaMap map[string]any
-	if len(identity.Meta) > 0 {
-		err = json.Unmarshal(identity.Meta, &metaMap)
-		if err != nil {
-			return fault.Wrap(err,
-				fault.Internal("unable to unmarshal metadata"), fault.Public("We're unable to parse the identity's metadata."),
-			)
-		}
-	} else {
-		metaMap = make(map[string]any)
+	metaMap, err := db.UnmarshalNullableJSONTo[map[string]any](identity.Meta)
+	if err != nil {
+		h.Logger.Error("failed to unmarshal identity meta",
+			"identityId", identity.ID,
+			"error", err,
+		)
 	}
 
 	// Format ratelimits for the response

go/apps/api/routes/v2_identities_list_identities/handler.go

@@ -3,7 +3,6 @@ package handler
 import (
 	"context"
 	"database/sql"
-	"encoding/json"
 	"errors"
 	"net/http"
 
@@ -17,8 +16,10 @@ import (
 	"github.com/unkeyed/unkey/go/pkg/zen"
 )
 
-type Request = openapi.V2IdentitiesListIdentitiesRequestBody
-type Response = openapi.V2IdentitiesListIdentitiesResponseBody
+type (
+	Request  = openapi.V2IdentitiesListIdentitiesRequestBody
+	Response = openapi.V2IdentitiesListIdentitiesResponseBody
+)
 
 // Handler implements zen.Route interface for the v2 identities list identities endpoint
 type Handler struct {
@@ -62,7 +63,6 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
 		IDCursor:    cursor,
 		Limit:       int32(limit + 1), // nolint:gosec
 	})
-
 	if err != nil {
 		return fault.Wrap(err,
 			fault.Internal("unable to list identities"), fault.Public("We're unable to list the identities."),
@@ -136,18 +136,15 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
 		}
 
 		// Add metadata if available
-		if len(identity.Meta) > 0 {
-			// Initialize the Meta field with an empty map
-			metaMap := make(map[string]interface{})
+		metaMap, err := db.UnmarshalNullableJSONTo[map[string]any](identity.Meta)
+		if err != nil {
+			h.Logger.Error("failed to unmarshal identity meta",
+				"identityId", identity.ID,
+				"error", err,
+			)
+			// Continue with empty meta
+		} else {
 			newIdentity.Meta = &metaMap
-
-			// Unmarshal the identity metadata into the map
-			err = json.Unmarshal(identity.Meta, &metaMap)
-			if err != nil {
-				return fault.Wrap(err,
-					fault.Internal("unable to unmarshal identity metadata"), fault.Public("We're unable to parse the metadata for the identity."),
-				)
-			}
 		}
 
 		// Append the identity to the results

go/apps/api/routes/v2_keys_get_key/handler.go

@@ -2,7 +2,6 @@ package handler
 
 import (
 	"context"
-	"encoding/json"
 	"net/http"
 	"sort"
 
@@ -178,8 +177,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
 		}
 
 		if len(keyData.Identity.Meta) > 0 {
-			var identityMeta map[string]any
-			if err := json.Unmarshal(keyData.Identity.Meta, &identityMeta); err != nil {
+			if identityMeta, err := db.UnmarshalNullableJSONTo[map[string]any](keyData.Identity.Meta); err != nil {
 				h.Logger.Error("failed to unmarshal identity meta", "error", err)
 			} else {
 				response.Identity.Meta = &identityMeta
@@ -249,9 +247,12 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
 
 	// Set meta
 	if keyData.Key.Meta.Valid {
-		var meta map[string]any
-		if err := json.Unmarshal([]byte(keyData.Key.Meta.String), &meta); err != nil {
-			h.Logger.Error("failed to unmarshal key meta", "error", err)
+		meta, err := db.UnmarshalNullableJSONTo[map[string]any](keyData.Key.Meta.String)
+		if err != nil {
+			h.Logger.Error("failed to unmarshal key meta",
+				"keyId", keyData.Key.ID,
+				"error", err,
+			)
 		} else {
 			response.Meta = &meta
 		}

go/apps/api/routes/v2_keys_verify_key/handler.go

@@ -2,7 +2,6 @@ package handler
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 
@@ -21,8 +20,10 @@ import (
 	"github.com/unkeyed/unkey/go/pkg/zen"
 )
 
-type Request = openapi.V2KeysVerifyKeyRequestBody
-type Response = openapi.V2KeysVerifyKeyResponseBody
+type (
+	Request  = openapi.V2KeysVerifyKeyRequestBody
+	Response = openapi.V2KeysVerifyKeyResponseBody
+)
 
 const DefaultCost = 1
 
@@ -191,13 +192,15 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
 	}
 
 	if key.Key.Meta.Valid {
-		err = json.Unmarshal([]byte(key.Key.Meta.String), &keyData.Meta)
+		meta, err := db.UnmarshalNullableJSONTo[map[string]any](key.Key.Meta.String)
 		if err != nil {
-			return fault.Wrap(err, fault.Code(codes.App.Internal.UnexpectedError.URN()),
-				fault.Internal("unable to unmarshal key meta"),
-				fault.Public("We encountered an error while trying to unmarshal the key meta data."),
+			h.Logger.Error("failed to unmarshal key meta",
+				"keyId", key.Key.ID,
+				"error", err,
 			)
+			// Continue with empty meta (zero value)
 		}
+		keyData.Meta = &meta
 	}
 
 	if key.Key.IdentityID.Valid {
@@ -227,15 +230,15 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
 			keyData.Identity.Ratelimits = ptr.P(identityRatelimits)
 		}
 
-		if len(key.Key.IdentityMeta) > 0 {
-			err = json.Unmarshal(key.Key.IdentityMeta, &keyData.Identity.Meta)
-			if err != nil {
-				return fault.Wrap(err, fault.Code(codes.App.Internal.UnexpectedError.URN()),
-					fault.Internal("unable to unmarshal identity meta"),
-					fault.Public("We encountered an error while trying to unmarshal the identity meta data."),
-				)
-			}
+		meta, err := db.UnmarshalNullableJSONTo[map[string]any](key.Key.IdentityMeta)
+		if err != nil {
+			h.Logger.Error("failed to unmarshal identity meta",
+				"identityId", key.Key.IdentityID.String,
+				"error", err,
+			)
+			// Continue with empty meta
 		}
+		keyData.Identity.Meta = &meta
 	}
 
 	if len(key.RatelimitResults) > 0 {

go/apps/api/routes/v2_keys_whoami/handler.go

@@ -2,7 +2,6 @@ package handler
 
 import (
 	"context"
-	"encoding/json"
 	"net/http"
 	"sort"
 
@@ -169,8 +168,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
 		}
 
 		if len(keyData.Identity.Meta) > 0 {
-			var identityMeta map[string]any
-			if err := json.Unmarshal(keyData.Identity.Meta, &identityMeta); err != nil {
+			if identityMeta, err := db.UnmarshalNullableJSONTo[map[string]any](keyData.Identity.Meta); err != nil {
 				h.Logger.Error("failed to unmarshal identity meta", "error", err)
 			} else {
 				response.Identity.Meta = &identityMeta
@@ -238,8 +236,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
 
 	// Set meta
 	if keyData.Key.Meta.Valid {
-		var meta map[string]any
-		if err := json.Unmarshal([]byte(keyData.Key.Meta.String), &meta); err != nil {
+		if meta, err := db.UnmarshalNullableJSONTo[map[string]any](keyData.Key.Meta.String); err != nil {
 			h.Logger.Error("failed to unmarshal key meta", "error", err)
 		} else {
 			response.Meta = &meta

go/pkg/db/custom_types.go

@@ -1,12 +1,14 @@
 package db
 
 import (
+	"encoding/json"
+	"fmt"
+
 	dbtype "github.com/unkeyed/unkey/go/pkg/db/types"
 )
 
-// These types mirror the database models and support JSON serialization and deserialization.
+// RoleInfo  types mirror the database models and support JSON serialization and deserialization.
 // They are used to unmarshal aggregated results (e.g., JSON arrays) returned by database queries.
-
 type RoleInfo struct {
 	ID          string            `json:"id"`
 	Name        string            `json:"name"`
@@ -29,3 +31,50 @@ type RatelimitInfo struct {
 	Duration   int64             `json:"duration"`
 	AutoApply  bool              `json:"auto_apply"`
 }
+
+// UnmarshalNullableJSONTo unmarshals JSON data from database columns into Go types.
+// It handles the common pattern where database queries return JSON as []byte that needs
+// to be deserialized into structs, slices, or maps.
+//
+// The function accepts 'any' type because database drivers return interface{} for JSON columns,
+// even though the underlying value is typically []byte.
+//
+// Returns:
+//   - (T, nil) on successful unmarshal
+//   - (zero, nil) if data is nil or empty []byte (these are valid null/empty states)
+//   - (zero, error) if type assertion fails or JSON unmarshal fails
+//
+// Example usage:
+//
+//	roles, err := UnmarshalNullableJSONTo[[]RoleInfo](row.Roles)
+//	if err != nil {
+//	    logger.Error("failed to unmarshal roles", "error", err)
+//	    return err
+//	}
+func UnmarshalNullableJSONTo[T any](data any) (T, error) {
+	var zero T
+	if data == nil {
+		return zero, nil
+	}
+
+	var bytes []byte
+	switch v := data.(type) {
+	case []byte:
+		bytes = v
+	case string:
+		bytes = []byte(v)
+	default:
+		return zero, fmt.Errorf("type assertion failed during unmarshal: expected []byte or string, got %T", data)
+	}
+
+	if len(bytes) == 0 {
+		return zero, nil
+	}
+
+	var result T
+	if err := json.Unmarshal(bytes, &result); err != nil {
+		return zero, fmt.Errorf("json unmarshal failed: %w", err)
+	}
+
+	return result, nil
+}