From 565a797247e74a651f384fa32da346d3a7cf9d28 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 27 Jun 2025 10:00:25 +0200 Subject: [PATCH 1/2] Add array streaming helpers An implementation of https://github.com/status-im/nim-json-serialization/issues/112 that introduces a rich set of helpers for creating arrays using a backwards-compatible approach that maintains the current semantic model of the array/object writer being responsible for introducing plumbing. Since the "outer" level drives the plumbing, we add writers for every combination of value and context: `beginArray`, `beginArrayElement` and `beginArrayMember` for creating an array in top-level, array and object respectively and the same for other values. In this model, `writeValue` itself performs no begin/end tracking - instead, it is up to the entity that creates the array/object to do so (`writeRecordValue`, `stepwiseArrayCreation` and so on). Here's an example of writing a `writeValue` overload that writes an array nested in an object: ```nim proc writeValue(w: var JsonWriter, t: MyType) = w.beginArray() for i in 0 ..< t.children: writer.beginObjectElement() writer.writeMember("id", i) writer.writeMember("name", "item" & t.childName[i]) writer.endObjectElement() writer.endArray() ``` The writing API is quite regular but requires calling special functions depending on the context which is easy to forget - calling the wrong variation results in invalid JSON! The backwards-compatiblity issue can be seen in a new test case based on a real-world example for writing byte arrays as hex strings. Further examples are available in the documentation. --- docs/examples/reference0.nim | 5 +- docs/examples/streamwrite0.nim | 18 ++ docs/examples/streamwrite1.nim | 16 ++ docs/src/SUMMARY.md | 1 + docs/src/reference.md | 5 +- docs/src/streaming.md | 65 +++++ json_serialization/types.nim | 26 +- json_serialization/writer.nim | 421 +++++++++++++++++++++++---------- tests/test_json_flavor.nim | 12 +- 9 files changed, 419 insertions(+), 150 deletions(-) create mode 100644 docs/examples/streamwrite0.nim create mode 100644 docs/examples/streamwrite1.nim create mode 100644 docs/src/streaming.md diff --git a/docs/examples/reference0.nim b/docs/examples/reference0.nim index d37a093..b0f126a 100644 --- a/docs/examples/reference0.nim +++ b/docs/examples/reference0.nim @@ -18,11 +18,8 @@ type var conf = defaultJsonReaderConf conf.nestedDepthLimit = 0 -let native = - Json.decode(rawJson, NimServer, flags = defaultJsonReaderFlags, conf = conf) - # decode into native Nim -#let native = Json.decode(rawJson, NimServer) +let native = Json.decode(rawJson, NimServer) # decode into mixed Nim + JsonValueRef let mixed = Json.decode(rawJson, MixedServer) diff --git a/docs/examples/streamwrite0.nim b/docs/examples/streamwrite0.nim new file mode 100644 index 0000000..ffd25f4 --- /dev/null +++ b/docs/examples/streamwrite0.nim @@ -0,0 +1,18 @@ +import json_serialization, faststreams/outputs + +let file = fileOutput("output.json") +var writer = JsonWriter[DefaultFlavor].init(file, pretty = true) + +writer.beginArray() + +for i in 0 ..< 2: + writer.beginObjectElement() + + writer.writeMember("id", i) + writer.writeMember("name", "item" & $i) + + writer.endObjectElement() + +writer.endArray() + +file.close() diff --git a/docs/examples/streamwrite1.nim b/docs/examples/streamwrite1.nim new file mode 100644 index 0000000..0684763 --- /dev/null +++ b/docs/examples/streamwrite1.nim @@ -0,0 +1,16 @@ +import json_serialization, faststreams/outputs + +let file = fileOutput("output.json") +var writer = JsonWriter[DefaultFlavor].init(file) + +# ANCHOR: Nesting +writer.writeObject: + writer.writeMember("status", "ok") + writer.writeArrayMember("data"): + for i in 0 ..< 2: + writer.writeObjectElement: + writer.writeMember("id", i) + writer.writeMember("name", "item" & $i) +# ANCHOR_END: Nesting + +file.close() diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 3689678..432241e 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -3,6 +3,7 @@ # User guide - [Getting started](./getting_started.md) +- [Streaming](./streaming.md) - [Reference](./reference.md) # Developer guide diff --git a/docs/src/reference.md b/docs/src/reference.md index 9af94b4..ce1b6a1 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -80,7 +80,7 @@ You can adjust these defaults to suit your needs: ### Common API -Similar to parsing, the [common serialization API]() is used to produce JSON documents. +Similar to parsing, the [common serialization API](https://github.com/status-im/nim-serialization?tab=readme-ov-file#common-api) is used to produce JSON documents. ```nim {{#include ../examples/reference0.nim:Encode}} @@ -257,9 +257,6 @@ writeField(w: var JsonWriter, name: string, value: auto) iterator stepwiseArrayCreation[C](w: var JsonWriter, collection: C): auto writeIterable(w: var JsonWriter, collection: auto) writeArray[T](w: var JsonWriter, elements: openArray[T]) - -writeNumber[F,T](w: var JsonWriter[F], value: JsonNumber[T]) -writeJsonValueRef[F,T](w: var JsonWriter[F], value: JsonValueRef[T]) ``` ## Enums diff --git a/docs/src/streaming.md b/docs/src/streaming.md new file mode 100644 index 0000000..db17241 --- /dev/null +++ b/docs/src/streaming.md @@ -0,0 +1,65 @@ +# Streaming + +`JsonWriter` can be used to incrementally write JSON data. + +Incremental processing is ideal for large documents or when you want to avoid building the entire JSON structure in memory. + + + +## Writing + +You can use `JsonWriter` to write JSON objects, arrays, and values step by step, directly to a file or any output stream. + +The process is similar to when you override `writeValue` to provide custom serialization. + +### Example: Writing a JSON Array of Objects + +Suppose you want to write a large array of objects to a file, one at a time: + +```nim +{{#include ../examples/streamwrite0.nim}} +``` + +Resulting file (`output.json`): +```json +[ + { + "id": 0, + "name": "item0" + }, + { + "id": 1, + "name": "item1" + } +] +``` + +```admonish warning "Elements in objects and array" +In the example, we see `beginArray`, `beginElement` and `writeMember`. The functions follow a pattern: +* functions without suffix, like `beginArray`, are used at the top-level +* functions with `Element` suffix are used inside arrays +* functions with `Member` suffix and accomanying name are used in objects + +Thus, if we were writing an array inside another array, we would have used `beginArray` for the outer array and `beginArrayMember` for the inner array. These rules also apply when implementing `writeValue`. +``` + +### Example: Writing Nested Structures + +Objects and arrays can be nested arbitrarily. + +Here is the same array of JSON objects, nested in an envelope containing an additional `status` field. + +Instead of manually placing `begin`/`end` pairs, we're using the convenience helpers `writeObjectElement` and `writeArrayMember`, along with `writeElement` to manage the required element markers: + +```nim +{{#include ../examples/streamwrite1.nim:Nesting}} +``` + +This produces a the following output - notice the more compact representation when `pretty = true` is not used: +```json +{"status":"ok","data":[{"id":0,"name":"item0"},{"id":1,"name":"item1"}]} +``` + +```admonish tip +Similar to `begin`, we're using the `Element` suffix in arrays! +``` diff --git a/json_serialization/types.nim b/json_serialization/types.nim index 841a6bb..68697c6 100644 --- a/json_serialization/types.nim +++ b/json_serialization/types.nim @@ -7,6 +7,8 @@ # This file may not be copied, modified, or distributed except according to # those terms. +{.push gcsafe, raises: [].} + import std/tables, serialization/errors @@ -18,13 +20,13 @@ export type JsonError* = object of SerializationError - # This is a special type to parse whatever - # json value into string. JsonString* = distinct string + ## A string containing valid JSON. + ## Used to preserve and pass on parts of a JSON document to another parser + ## or layer without interpreting it further - # This is a special type to parse whatever - # json value into nothing/skip it. JsonVoid* = object + ## Marker used for skipping a JSON value during parsing JsonSign* {.pure.} = enum None @@ -63,11 +65,11 @@ type stringLengthLimit*: int JsonValueKind* {.pure.} = enum - String, - Number, - Object, - Array, - Bool, + String + Number + Object + Array + Bool Null JsonObjectType*[T: string or uint64] = OrderedTable[string, JsonValueRef[T]] @@ -88,7 +90,6 @@ type of JsonValueKind.Null: discard - const minPortableInt* = -9007199254740991 # -2**53 + 1 maxPortableInt* = 9007199254740991 # +2**53 - 1 @@ -110,8 +111,6 @@ const stringLengthLimit: 0, ) -{.push gcsafe, raises: [].} - template `==`*(lhs, rhs: JsonString): bool = string(lhs) == string(rhs) @@ -172,4 +171,7 @@ func `==`*(lhs, rhs: JsonValueRef): bool = of JsonValueKind.Null: true +template `$`*(s: JsonString): string = + string(s) + {.pop.} diff --git a/json_serialization/writer.nim b/json_serialization/writer.nim index b900bc5..59826dc 100644 --- a/json_serialization/writer.nim +++ b/json_serialization/writer.nim @@ -19,30 +19,43 @@ export outputs, format, types, JsonString, DefaultFlavor type - JsonWriterState = enum - RecordExpected - RecordStarted - AfterField + CollectionKind = enum + Array + Object + JsonWriter*[Flavor = DefaultFlavor] = object stream*: OutputStream hasTypeAnnotations: bool hasPrettyOutput*: bool # read-only - nestingLevel*: int # read-only - state: JsonWriterState + stack: seq[CollectionKind] + ## Stack that keeps track of nested arrays/objects + empty: bool + ## True before any members / elements have been written to an object / array + wantName: bool + ## The next output should be a name (for an object member) Json.setWriter JsonWriter, PreferredOutput = string +template nestingLevel(w: JsonWriter): int = + w.stack.len * 2 + func init*(W: type JsonWriter, stream: OutputStream, pretty = false, typeAnnotations = false): W = W(stream: stream, hasPrettyOutput: pretty, - hasTypeAnnotations: typeAnnotations, - nestingLevel: if pretty: 0 else: -1, - state: RecordExpected) - -proc writeValue*(w: var JsonWriter, value: auto) {.raises: [IOError].} + hasTypeAnnotations: typeAnnotations) + +proc writeValue*[V: not void](w: var JsonWriter, value: V) {.raises: [IOError].} + ## Write value as JSON, without adornments for arrays and objects. + ## + ## See also `writeElement` and `writeMember`. +proc writeElement*[V: not void](w: var JsonWriter, value: V) {.raises: [IOError].} + ## Write `value` as a JSON element in an array or object member pair, adorning + ## it with `beginElement`/`endElement` as needed. +proc writeMember*[V: not void](w: var JsonWriter, name: string, value: V) {.raises: [IOError].} + ## Write `name` and `value` as a JSON member / field of an object. # If it's an optional field, test for it's value before write something. # If it's non optional field, the field is always written. @@ -52,22 +65,137 @@ template append(x: untyped) = write w.stream, x template indent = - for i in 0 ..< w.nestingLevel: - append ' ' + if w.hasPrettyOutput: + append "\n" + for i in 0 ..< w.nestingLevel: + append ' ' + +func inArray(w: JsonWriter): bool = + w.stack.len > 0 and w.stack[^1] == Array + +func inObject(w: JsonWriter): bool = + w.stack.len > 0 and w.stack[^1] == Object + +proc beginElement*(w: var JsonWriter) {.raises: [IOError].} = + ## Start writing an array element or the value part of an object member. + ## + ## Must be closed with a corresponding `endElement`. + ## + ## See also `writeElement`, `writeMember`. + doAssert not w.wantName + + if w.inArray: + if w.empty: + w.empty = false + else: + append ',' + + indent() -template `$`*(s: JsonString): string = - string(s) +proc endElement*(w: var JsonWriter) = + ## Matching `end` call for `beginElement` + w.wantName = w.inObject + +proc beginObject*(w: var JsonWriter, nested: static bool = false) {.raises: [IOError].} = + ## Start writing an object, to be followed by member fields. + ## + ## Must be closed with a matching `endObject`. + ## + ## See also `writeObject`. + ## + ## Use `writeMember`, `writeArrayMember`, `writeObjectMember` to add member + ## fields to the object. + ## + ## Use `nested = true` when creating the object nested inside another object + ## or array, or use `beginObjectElement` / `beginObjectMember`. + ## + ## Use `nested = false` (the default) in top-level calls both in `writeValue` + ## implementations and when streaming. + when nested: + w.beginElement() -proc writeFieldName*(w: var JsonWriter, name: string) {.raises: [IOError].} = - # this is implemented as a separate proc in order to - # keep the code bloat from `writeField` to a minimum - doAssert w.state != RecordExpected + append '{' - if w.state == AfterField: - append ',' + w.empty = true + w.wantName = true - if w.hasPrettyOutput: - append '\n' + w.stack.add(Object) + +proc beginObject*(w: var JsonWriter, O: type, nested: static bool = false) {.raises: [IOError].} = + w.beginObject(nested) + if w.hasTypeAnnotations: w.writeMember("$type", typetraits.name(O)) + +proc endObject*(w: var JsonWriter, nested: static bool = false) {.raises: [IOError].} = + doAssert w.stack.pop() == Object + + if not w.empty: + indent() + + w.empty = false + append '}' + + when nested: + w.endElement() + +proc beginArray*(w: var JsonWriter, nested: static bool = false) {.raises: [IOError].} = + when nested: + w.beginElement() + + append '[' + + w.empty = true + + w.stack.add(Array) + +proc endArray*(w: var JsonWriter, nested: static bool = false) {.raises: [IOError].} = + doAssert w.stack.pop() == Array + + if not w.empty: + indent() + + w.empty = false + + append ']' + + when nested: + w.endElement() + +template writeElement*[T: void](w: var JsonWriter, body: T) = + w.beginElement() + body + w.endElement() + +proc writeElement*[V: not void](w: var JsonWriter, value: V) {.raises: [IOError].} = + mixin writeValue + + w.writeElement: + w.writeValue(value) + +template beginObjectElement*(w: var JsonWriter) = + ## Begin an object nested inside an array or after a member name + beginObject(w, nested = true) + +template beginObjectElement*(w: var JsonWriter, O: type) = + ## Begin an object nested inside an array or after a member name + beginObject(w, O, nested = true) + +template endObjectElement*(w: var JsonWriter) = endObject(w, nested = true) + +template beginArrayElement*(w: var JsonWriter) = beginArray(w, nested = true) + +template endArrayElement*(w: var JsonWriter) = endArray(w, nested = true) + +proc writeName*(w: var JsonWriter, name: string) {.raises: [IOError].} = + ## Write the name part of the member of an object, to be followed by the value + doAssert w.inObject() + doAssert w.wantName + + w.wantName = false + + if w.empty: + w.empty = false + else: + append ',' indent() @@ -77,10 +205,21 @@ proc writeFieldName*(w: var JsonWriter, name: string) {.raises: [IOError].} = append ':' if w.hasPrettyOutput: append ' ' - w.state = RecordExpected +template writeMember*[T: void](w: var JsonWriter, name: string, body: T) = + ## Write a member field of an object, ie the name followed by the value. + ## + ## Optional fields may be omitted depending on the Flavor. + mixin writeValue + + w.writeName(name) + w.writeElement: + body -proc writeField*( - w: var JsonWriter, name: string, value: auto) {.raises: [IOError].} = +proc writeMember*[V: not void]( + w: var JsonWriter, name: string, value: V) {.raises: [IOError].} = + ## Write a member field of an object, ie the name followed by the value. + ## + ## Optional fields may get omitted depending on the Flavor. mixin writeValue mixin flavorOmitsOptionalFields, shouldWriteObjectField @@ -90,95 +229,96 @@ proc writeField*( when flavorOmitsOptionalFields(Flavor): if shouldWriteObjectField(value): - w.writeFieldName(name) - w.writeValue(value) - w.state = AfterField - else: - w.writeFieldName(name) - w.writeValue(value) - w.state = AfterField + w.writeName(name) + w.writeElement(value) -template fieldWritten*(w: var JsonWriter) = - w.state = AfterField + else: + w.writeName(name) + w.writeElement(value) -proc beginRecord*(w: var JsonWriter) {.raises: [IOError].} = - doAssert w.state == RecordExpected +template beginObjectMember*(w: var JsonWriter, name: string) = + w.writeName(name) + w.beginObjectElement() - append '{' - if w.hasPrettyOutput: - w.nestingLevel += 2 +template beginObjectMember*(w: var JsonWriter, name: string, O: type) = + w.writeName(name) + w.beginObjectElement(O) - w.state = RecordStarted +template endObjectMember*(w: var JsonWriter) = + w.endObjectElement() -proc beginRecord*(w: var JsonWriter, T: type) {.raises: [IOError].} = - w.beginRecord() - if w.hasTypeAnnotations: w.writeField("$type", typetraits.name(T)) +template beginArrayMember*(w: var JsonWriter, name: string) = + w.writeName(name) + w.beginArrayElement() -proc endRecord*(w: var JsonWriter) {.raises: [IOError].} = - doAssert w.state != RecordExpected - - if w.hasPrettyOutput: - append '\n' - w.nestingLevel -= 2 - indent() - - append '}' - -template endRecordField*(w: var JsonWriter) = - endRecord(w) - w.state = AfterField +template endArrayMember*(w: var JsonWriter) = + w.endArrayElement() iterator stepwiseArrayCreation*[C](w: var JsonWriter, collection: C): auto {.raises: [IOError].} = - append '[' - - if w.hasPrettyOutput: - append '\n' - w.nestingLevel += 2 - indent() + ## Iterate over the members of a collection, expecting each member to be + ## written directly to the stream (akin to using `writeValue` and not + ## `writeElement`) + w.beginArray() + for e in collection: + w.beginElement() + yield e + w.endElement() + w.endArray() - var first = true +proc writeIterable*(w: var JsonWriter, collection: auto) {.raises: [IOError].} = + mixin writeValue + w.beginArray() for e in collection: - if not first: - append ',' - if w.hasPrettyOutput: - append '\n' - indent() + w.writeElement(e) + w.endArray() - w.state = RecordExpected - yield e - first = false +template writeArray*[T: void](w: var JsonWriter, body: T) = + w.beginArray() + body + w.endArray() - if w.hasPrettyOutput: - append '\n' - w.nestingLevel -= 2 - indent() +template writeArrayElement*[T: void](w: var JsonWriter, body: T) = + w.beginArrayElement() + body + w.beginArrayElement() - append ']' +template writeArrayMember*[T: void](w: var JsonWriter, name: string, body: T) = + w.beginArrayMember(name) + body + w.endArrayMember() -proc writeIterable*(w: var JsonWriter, collection: auto) {.raises: [IOError].} = - mixin writeValue - for e in w.stepwiseArrayCreation(collection): - w.writeValue(e) +proc writeArray*[C: not void](w: var JsonWriter, values: C) {.raises: [IOError].} = + w.writeIterable(values) -proc writeArray*[T](w: var JsonWriter, elements: openArray[T]) {.raises: [IOError].} = - writeIterable(w, elements) +template writeObject*[T: void](w: var JsonWriter, O: type, body: T) = + w.beginObject(O) + body + w.endObject() -template writeObject*(w: var JsonWriter, T: type, body: untyped) = - w.beginRecord(T) +template writeObject*[T: void](w: var JsonWriter, body: T) = + w.beginObject() body - w.endRecord() + w.endObject() -template writeObject*(w: var JsonWriter, body: untyped) = - w.beginRecord() +template writeObjectElement*[T: void](w: var JsonWriter, body: T) = + w.beginObjectElement() body - w.endRecord() + w.endObjectElement() -# this construct catches `array[N, char]` which otherwise won't decompose into -# openArray[char] - we treat any array-like thing-of-characters as a string in -# the output -template isStringLike(v: string|cstring|openArray[char]|seq[char]): bool = true -template isStringLike[N](v: array[N, char]): bool = true -template isStringLike(v: auto): bool = false +template writeObjectElement*[T: void](w: var JsonWriter, O: type, body: T) = + w.beginObjectElement(O) + body + w.endObjectElement() + +template writeObjectMember*[T: void](w: var JsonWriter, name: string, body: T) = + w.beginObjectMember(name) + body + w.endObjectMember() + +template writeObjectMember*[T: void](w: var JsonWriter, name: string, O: type, body: T) = + w.beginObjectMember(name, O) + body + w.endObjectMember() template writeObjectField*[FieldType, RecordType](w: var JsonWriter, record: RecordType, @@ -186,19 +326,22 @@ template writeObjectField*[FieldType, RecordType](w: var JsonWriter, field: FieldType) = mixin writeFieldIMPL, writeValue - w.writeFieldName(fieldName) + w.writeName(fieldName) + + w.beginElement() when RecordType is tuple: w.writeValue(field) else: type R = type record w.writeFieldIMPL(FieldTag[R, fieldName], field, record) + w.endElement() -proc writeRecordValue*(w: var JsonWriter, value: auto) {.raises: [IOError].} = +proc writeRecordValue*(w: var JsonWriter, value: object|tuple) {.raises: [IOError].} = mixin enumInstanceSerializedFields, writeObjectField mixin flavorOmitsOptionalFields, shouldWriteObjectField type RecordType = type value - w.beginRecord RecordType + w.beginObject(RecordType) value.enumInstanceSerializedFields(fieldName, fieldValue): when fieldValue isnot JsonVoid: type @@ -207,19 +350,17 @@ proc writeRecordValue*(w: var JsonWriter, value: auto) {.raises: [IOError].} = when flavorOmitsOptionalFields(Flavor): if shouldWriteObjectField(fieldValue): writeObjectField(w, value, fieldName, fieldValue) - w.state = AfterField else: writeObjectField(w, value, fieldName, fieldValue) - w.state = AfterField else: discard fieldName - w.endRecord() + w.endObject() -proc writeNumber*[F,T](w: var JsonWriter[F], value: JsonNumber[T]) {.raises: [IOError].} = +proc writeValue*(w: var JsonWriter, value: JsonNumber) {.raises: [IOError].} = if value.sign == JsonSign.Neg: append '-' - when T is uint64: + when value.integer is uint64: w.stream.writeText value.integer else: append value.integer @@ -229,7 +370,7 @@ proc writeNumber*[F,T](w: var JsonWriter[F], value: JsonNumber[T]) {.raises: [IO append value.fraction template writeExp(body: untyped) = - when T is uint64: + when value.exponent is uint64: if value.exponent > 0: body else: @@ -240,33 +381,30 @@ proc writeNumber*[F,T](w: var JsonWriter[F], value: JsonNumber[T]) {.raises: [IO append 'e' if value.expSign == JsonSign.Neg: append '-' - when T is uint64: + when value.exponent is uint64: w.stream.writeText value.exponent else: append value.exponent -proc writeJsonValueRef*[F,T](w: var JsonWriter[F], value: JsonValueRef[T]) {.raises: [IOError].} = - if value.isNil: - append "null" - return +proc writeValue*(w: var JsonWriter, value: JsonObjectType) {.raises: [IOError].} = + w.beginObject() + for name, v in value: + w.writeMember(name, v) + w.endObject() +proc writeValue*(w: var JsonWriter, value: JsonValue) {.raises: [IOError].} = case value.kind of JsonValueKind.String: w.writeValue(value.strVal) of JsonValueKind.Number: - w.writeNumber(value.numVal) + w.writeValue(value.numVal) of JsonValueKind.Object: - w.beginRecord typeof(value) - for k, v in value.objVal: - w.writeField(k, v) - w.endRecord() + w.writeValue(value.objVal) of JsonValueKind.Array: - w.writeArray(value.arrayVal) + w.writeValue(value.arrayVal) of JsonValueKind.Bool: - if value.boolVal: - append "true" - else: - append "false" + w.writeValue(value.boolVal) + of JsonValueKind.Null: append "null" @@ -275,7 +413,7 @@ template writeEnumImpl(w: var JsonWriter, value, enumRep) = when enumRep == EnumAsString: w.writeValue $value elif enumRep == EnumAsNumber: - w.stream.writeText(value.int) + w.writeValue value.int elif enumRep == EnumAsStringifiedNumber: w.writeValue $value.int @@ -288,7 +426,20 @@ template writeValue*(w: var JsonWriter, value: enum) = type Flavor = type(w).Flavor writeEnumImpl(w, value, Flavor.flavorEnumRep()) -proc writeValue*(w: var JsonWriter, value: auto) {.raises: [IOError].} = +# this construct catches `array[N, char]` which otherwise won't decompose into +# openArray[char] - we treat any array-like thing-of-characters as a string in +# the output +template isStringLike(v: string|cstring|openArray[char]|seq[char]): bool = true +template isStringLike[N](v: array[N, char]): bool = true +template isStringLike(v: auto): bool = false + +proc writeValue*[V: not void](w: var JsonWriter, value: V) {.raises: [IOError].} = + ## `writeValue` is a low-level operation for encoding `value` as JSON without + ## adorning it with `beginElement`/`endElement`/`writeName`. + ## + ## When writing inside a nested object / array, prefer + ## `writeMember` / `writeElement` respectively that will make the right + ## adornments. mixin writeValue when value is JsonNode: @@ -301,12 +452,6 @@ proc writeValue*(w: var JsonWriter, value: auto) {.raises: [IOError].} = elif value is JsonVoid: discard - elif value is JsonNumber: - w.writeNumber(value) - - elif value is JsonValueRef: - w.writeJsonValueRef(value) - elif value is ref: if value == nil: append "null" @@ -388,9 +533,9 @@ proc writeValue*(w: var JsonWriter, value: auto) {.raises: [IOError].} = writeRecordValue(w, value) else: const typeName = typetraits.name(value.type) - {.fatal: "Failed to convert to JSON an unsupported type: " & typeName.} + {.error: "Failed to convert to JSON an unsupported type: " & typeName.} -proc toJson*(v: auto, pretty = false, typeAnnotations = false, Flavor = DefaultFlavor): string {.raises: [].} = +proc toJson*(v: auto, pretty = false, typeAnnotations = false, Flavor = DefaultFlavor): string = mixin writeValue var @@ -402,6 +547,24 @@ proc toJson*(v: auto, pretty = false, typeAnnotations = false, Flavor = DefaultF raiseAssert "no exceptions from memoryOutput" s.getOutput(string) +# nim-serialization integration / naming + +template beginRecord*(w: var JsonWriter) = beginObject(w, false) +template beginRecord*(w: var JsonWriter, T: type) = beginObject(w, T, false) + +template writeFieldName*(w: var JsonWriter, name: string) = writeName(w, name) + +template writeField*(w: var JsonWriter, name: string, value: auto) = + writeMember(w, name, value) + +template endRecord*(w: var JsonWriter) = w.endObject(false) + +template endRecordField*(w: var JsonWriter) {.deprecated: "endObject".} = + endRecord(w) + +template fieldWritten*(w: var JsonWriter) {.deprecated: "endElement".} = + w.endElement() + template serializesAsTextInJson*(T: type[enum]) = template writeValue*(w: var JsonWriter, val: T) = w.writeValue $val diff --git a/tests/test_json_flavor.nim b/tests/test_json_flavor.nim index 40cc7d2..4c251d3 100644 --- a/tests/test_json_flavor.nim +++ b/tests/test_json_flavor.nim @@ -11,6 +11,7 @@ import std/[strutils, options], unittest2, results, + stew/byteutils, serialization, ../json_serialization/pkg/results, ../json_serialization/std/options, @@ -18,6 +19,11 @@ import createJsonFlavor StringyJson +proc writeValue(w: var JsonWriter[StringyJson], value: seq[byte]) = + w.stream.write('"') + w.stream.write(toHex(value)) + w.stream.write('"') + proc writeValue*( w: var JsonWriter[StringyJson], val: SomeInteger) {.raises: [IOError].} = writeValue(w, $val) @@ -199,4 +205,8 @@ suite "Test JsonFlavor": NullyFields.flavorEnumRep(EnumAsNumber) let z = NullyFields.encode(Banana) - check z == "0" \ No newline at end of file + check z == "0" + + test "custom writer that uses stream": + let value = @[@[byte 0, 1], @[byte 2, 3]] + check: StringyJson.encode(value) == """["0001","0203"]""" From 7573801ec607b4bcfb84b223ad697141fb0bf067 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 27 Jun 2025 10:14:22 +0200 Subject: [PATCH 2/2] 1.6 compat --- json_serialization/writer.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json_serialization/writer.nim b/json_serialization/writer.nim index 59826dc..cc8fa1c 100644 --- a/json_serialization/writer.nim +++ b/json_serialization/writer.nim @@ -453,7 +453,7 @@ proc writeValue*[V: not void](w: var JsonWriter, value: V) {.raises: [IOError].} discard elif value is ref: - if value == nil: + if value.isNil: append "null" else: writeValue(w, value[])