diff --git a/goyaml.v3/patch.go b/goyaml.v3/patch.go index b98c332..cebfdf0 100644 --- a/goyaml.v3/patch.go +++ b/goyaml.v3/patch.go @@ -37,3 +37,7 @@ func (e *Encoder) DefaultSeqIndent() { func yaml_emitter_process_line_comment(emitter *yaml_emitter_t) bool { return yaml_emitter_process_line_comment_linebreak(emitter, false) } + +func (e *Encoder) SetWidth(width int) { + yaml_emitter_set_width(&e.encoder.emitter, width) +} diff --git a/goyaml.v3/patch_test.go b/goyaml.v3/patch_test.go index 3b76276..10de880 100644 --- a/goyaml.v3/patch_test.go +++ b/goyaml.v3/patch_test.go @@ -175,3 +175,17 @@ a: t.Errorf("expected error, got none") } } + +func (s *S) TestSetWidth(c *C) { + longString := "this is very long line with spaces and it must be longer than 80 so we will repeat that it must be longer that 80" + + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + enc.SetWidth(80) + err := enc.Encode(map[string]interface{}{"a": longString}) + c.Assert(err, Equals, nil) + err = enc.Close() + c.Assert(err, Equals, nil) + c.Assert(buf.String(), Equals, "a: this is very long line with spaces and it must be longer than 80 so we will repeat\n that it must be longer that 80\n") +} diff --git a/goyaml.v3/yaml.go b/goyaml.v3/yaml.go index 8cec6da..f12a729 100644 --- a/goyaml.v3/yaml.go +++ b/goyaml.v3/yaml.go @@ -91,8 +91,9 @@ func Unmarshal(in []byte, out interface{}) (err error) { // A Decoder reads and decodes YAML values from an input stream. type Decoder struct { - parser *parser - knownFields bool + parser *parser + knownFields bool + noUniqueKeys bool } // NewDecoder returns a new decoder that reads from r. @@ -111,6 +112,11 @@ func (dec *Decoder) KnownFields(enable bool) { dec.knownFields = enable } +// UniqueKeys ensures that the keys in the yaml document are unique. +func (dec *Decoder) UniqueKeys(enable bool) { + dec.noUniqueKeys = !enable +} + // Decode reads the next YAML-encoded value from its input // and stores it in the value pointed to by v. // @@ -119,6 +125,7 @@ func (dec *Decoder) KnownFields(enable bool) { func (dec *Decoder) Decode(v interface{}) (err error) { d := newDecoder() d.knownFields = dec.knownFields + d.uniqueKeys = !dec.noUniqueKeys defer handleErr(&err) node := dec.parser.parse() if node == nil { diff --git a/yaml.go b/yaml.go index b8d3e53..c26c9a2 100644 --- a/yaml.go +++ b/yaml.go @@ -24,7 +24,8 @@ import ( "reflect" "strconv" - "sigs.k8s.io/yaml/goyaml.v2" + yamlv2 "sigs.k8s.io/yaml/goyaml.v2" + yamlv3 "sigs.k8s.io/yaml/goyaml.v3" ) // Marshal marshals obj into JSON using stdlib json.Marshal, and then converts JSON to YAML using JSONToYAML (see that method for more reference) @@ -37,6 +38,40 @@ func Marshal(obj interface{}) ([]byte, error) { return JSONToYAML(jsonBytes) } +// yamlv3Unmarshal is a YAML unmarshaler that does not error on duplicate keys or on unknown fields. +// This function replicates the behavior of the Unmarshal function in the goyaml v2 library. +func yamlv3Unmarshal(input []byte, output interface{}) error { + decoder := yamlv3.NewDecoder(bytes.NewReader(input)) + decoder.KnownFields(false) + decoder.UniqueKeys(false) + return decoder.Decode(output) +} + +// yamlv3UnmarshalStrict is like Unmarshal except that any fields that are found in the data that +// do not have corresponding struct members, or mapping keys that are duplicates, will result in an error. +// This function replicates the behavior of the UnmarshalStrict function in the goyaml v2 library. +func yamlv3UnmarshalStrict(input []byte, output interface{}) error { + decoder := yamlv3.NewDecoder(bytes.NewReader(input)) + decoder.KnownFields(true) + decoder.UniqueKeys(true) + return decoder.Decode(output) +} + +// yamlv3Marshal is a YAML marshaller that uses compact sequence indentation an indentation of 2 spaces. +// It also sets the width to 80 characters. +// This function replicates the behavior of the Marshal function in the goyaml v2 library. +func yamlv3Marshal(input interface{}) ([]byte, error) { + buf := bytes.Buffer{} + encoder := yamlv3.NewEncoder(&buf) + encoder.CompactSeqIndent() + encoder.SetIndent(2) + encoder.SetWidth(80) + if err := encoder.Encode(input); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + // JSONOpt is a decoding option for decoding from JSON format. type JSONOpt func(*json.Decoder) *json.Decoder @@ -53,7 +88,7 @@ type JSONOpt func(*json.Decoder) *json.Decoder // - YAML non-string keys, e.g. ints, bools and floats, are converted to strings implicitly during the YAML to JSON conversion process. // - There are no compatibility guarantees for returned error values. func Unmarshal(yamlBytes []byte, obj interface{}, opts ...JSONOpt) error { - return unmarshal(yamlBytes, obj, yaml.Unmarshal, opts...) + return unmarshal(yamlBytes, obj, yamlv3Unmarshal, opts...) } // UnmarshalStrict is similar to Unmarshal (please read its documentation for reference), with the following exceptions: @@ -61,7 +96,7 @@ func Unmarshal(yamlBytes []byte, obj interface{}, opts ...JSONOpt) error { // - Duplicate fields in an object yield an error. This is according to the YAML specification. // - If obj, or any of its recursive children, is a struct, presence of fields in the serialized data unknown to the struct will yield an error. func UnmarshalStrict(yamlBytes []byte, obj interface{}, opts ...JSONOpt) error { - return unmarshal(yamlBytes, obj, yaml.UnmarshalStrict, append(opts, DisallowUnknownFields)...) + return unmarshal(yamlBytes, obj, yamlv3UnmarshalStrict, append(opts, DisallowUnknownFields)...) } // unmarshal unmarshals the given YAML byte stream into the given interface, @@ -111,13 +146,13 @@ func JSONToYAML(j []byte) ([]byte, error) { // etc.) when unmarshalling to interface{}, it just picks float64 // universally. go-yaml does go through the effort of picking the right // number type, so we can preserve number type throughout this process. - err := yaml.Unmarshal(j, &jsonObj) + err := yamlv3Unmarshal(j, &jsonObj) if err != nil { return nil, err } // Marshal this object into YAML. - yamlBytes, err := yaml.Marshal(jsonObj) + yamlBytes, err := yamlv3Marshal(jsonObj) if err != nil { return nil, err } @@ -144,13 +179,13 @@ func JSONToYAML(j []byte) ([]byte, error) { // - Unlike Unmarshal, all integers, up to 64 bits, are preserved during this round-trip. // - There are no compatibility guarantees for returned error values. func YAMLToJSON(y []byte) ([]byte, error) { - return yamlToJSONTarget(y, nil, yaml.Unmarshal) + return yamlToJSONTarget(y, nil, yamlv3Unmarshal) } // YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding, // returning an error on any duplicate field names. func YAMLToJSONStrict(y []byte) ([]byte, error) { - return yamlToJSONTarget(y, nil, yaml.UnmarshalStrict) + return yamlToJSONTarget(y, nil, yamlv3UnmarshalStrict) } func yamlToJSONTarget(yamlBytes []byte, jsonTarget *reflect.Value, unmarshalFn func([]byte, interface{}) error) ([]byte, error) { @@ -196,6 +231,18 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in } } + // Transform map[string]interface{} into map[interface{}]interface{}, + // such that the switch statement below can handle it. + // TODO: we should modify the logic in the switch statement to handle + // map[string]interface{} directly, instead of doing this conversion. + if stringMap, ok := yamlObj.(map[string]interface{}); ok { + interfaceMap := make(map[interface{}]interface{}) + for k, v := range stringMap { + interfaceMap[k] = v + } + yamlObj = interfaceMap + } + // If yamlObj is a number or a boolean, check if jsonTarget is a string - // if so, coerce. Else return normal. // If yamlObj is a map or array, find the field that each key is @@ -227,7 +274,7 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in // Stolen from go-yaml to use the same conversion to string as // the go-yaml library uses to convert float to string when // Marshaling. - s := strconv.FormatFloat(typedKey, 'g', -1, 32) + s := strconv.FormatFloat(typedKey, 'g', -1, 64) switch s { case "+Inf": s = ".inf" @@ -339,7 +386,7 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in case int64: s = strconv.FormatInt(typedVal, 10) case float64: - s = strconv.FormatFloat(typedVal, 'g', -1, 32) + s = strconv.FormatFloat(typedVal, 'g', -1, 64) case uint64: s = strconv.FormatUint(typedVal, 10) case bool: @@ -370,13 +417,13 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in // Big int/int64/uint64 do not lose precision as in the json-yaml roundtripping case. // // string, bool and any other types are unchanged. -func JSONObjectToYAMLObject(j map[string]interface{}) yaml.MapSlice { +func JSONObjectToYAMLObject(j map[string]interface{}) yamlv2.MapSlice { if len(j) == 0 { return nil } - ret := make(yaml.MapSlice, 0, len(j)) + ret := make(yamlv2.MapSlice, 0, len(j)) for k, v := range j { - ret = append(ret, yaml.MapItem{Key: k, Value: jsonToYAMLValue(v)}) + ret = append(ret, yamlv2.MapItem{Key: k, Value: jsonToYAMLValue(v)}) } return ret } diff --git a/yaml_test.go b/yaml_test.go index 00cb371..5d49363 100644 --- a/yaml_test.go +++ b/yaml_test.go @@ -175,15 +175,13 @@ func testYAMLToJSON(t *testing.T, f testYAMLToJSONFunc, tests map[string]yamlToJ type MarshalTest struct { A string B int64 - // Would like to test float64, but it's not supported in go-yaml. - // (See https://github.com/go-yaml/yaml/issues/83.) - C float32 + C float64 } func TestMarshal(t *testing.T) { - f32String := strconv.FormatFloat(math.MaxFloat32, 'g', -1, 32) - s := MarshalTest{"a", math.MaxInt64, math.MaxFloat32} - e := []byte(fmt.Sprintf("A: a\nB: %d\nC: %s\n", math.MaxInt64, f32String)) + f64String := strconv.FormatFloat(math.MaxFloat64, 'g', -1, 64) + s := MarshalTest{"a", math.MaxInt64, math.MaxFloat64} + e := []byte(fmt.Sprintf("A: a\nB: %d\nC: %s\n", math.MaxInt64, f64String)) y, err := Marshal(s) if err != nil { @@ -213,6 +211,8 @@ type UnmarshalTaggedStruct struct { IntBig2 string `json:"1000000000000000000000000000000000000"` IntBig2Scientific string `json:"1e+36"` Float3dot3 string `json:"3.3"` + FloatMax32 string `json:"3.4028234663852886e+38"` + FloatMax64 string `json:"1.7976931348623157e+308"` } type UnmarshalStruct struct { @@ -296,12 +296,12 @@ func TestUnmarshal(t *testing.T) { "tagged casematched boolean key (yes)": { encoded: []byte("Yes: test"), decodeInto: new(UnmarshalTaggedStruct), - decoded: UnmarshalTaggedStruct{TrueLower: "test"}, + decoded: UnmarshalTaggedStruct{YesUpper: "test"}, // In yamlv2, this incorrectly set the TrueLower field instead }, "tagged non-casematched boolean key (yes)": { encoded: []byte("yes: test"), decodeInto: new(UnmarshalTaggedStruct), - decoded: UnmarshalTaggedStruct{TrueLower: "test"}, + decoded: UnmarshalTaggedStruct{YesLower: "test"}, // In yamlv2, this incorrectly set the TrueLower field instead }, "tagged integer key": { encoded: []byte("3: test"), @@ -323,6 +323,16 @@ func TestUnmarshal(t *testing.T) { decodeInto: new(UnmarshalTaggedStruct), decoded: UnmarshalTaggedStruct{Float3dot3: "test"}, }, + "tagged max float32 key": { + encoded: []byte("3.4028234663852886e+38: test"), + decodeInto: new(UnmarshalTaggedStruct), + decoded: UnmarshalTaggedStruct{FloatMax32: "test"}, + }, + "tagged max float64 key": { + encoded: []byte("1.7976931348623157e+308: test"), + decodeInto: new(UnmarshalTaggedStruct), + decoded: UnmarshalTaggedStruct{FloatMax64: "test"}, + }, // decode into string field "string value into string field": { @@ -343,7 +353,7 @@ func TestUnmarshal(t *testing.T) { "boolean value (no) into string field": { encoded: []byte("a: no"), decodeInto: new(UnmarshalStruct), - decoded: UnmarshalStruct{A: "false"}, + decoded: UnmarshalStruct{A: "no"}, // In yamlv2, this incorrectly set the value to "false" }, // decode into complex fields @@ -390,7 +400,7 @@ func TestUnmarshal(t *testing.T) { encoded: []byte("Yes:"), decodeInto: new(map[string]struct{}), decoded: map[string]struct{}{ - "true": {}, + "Yes": {}, // In yamlv2, this incorrectly set the key to "true" }, }, "string map: decode integer key": { @@ -440,6 +450,38 @@ func TestUnmarshal(t *testing.T) { }, }, + // decoding floats + "decode max float32 into float32": { + encoded: []byte("3.4028234663852886e+38"), + decodeInto: new(float32), + decoded: float32(math.MaxFloat32), + }, + "decode max float32 into interface": { + encoded: []byte("3.4028234663852886e+38"), + decodeInto: new(interface{}), + decoded: float64(math.MaxFloat32), + }, + "decode max float32 into string": { + encoded: []byte("3.4028234663852886e+38"), + decodeInto: new(string), + decoded: "3.4028234663852886e+38", + }, + "decode max float64 into float64": { + encoded: []byte("1.7976931348623157e+308"), + decodeInto: new(float64), + decoded: math.MaxFloat64, + }, + "decode max float64 into interface": { + encoded: []byte("1.7976931348623157e+308"), + decodeInto: new(interface{}), + decoded: math.MaxFloat64, + }, + "decode max float64 into string": { + encoded: []byte("1.7976931348623157e+308"), + decodeInto: new(string), + decoded: "1.7976931348623157e+308", + }, + // duplicate (non-casematched) keys (NOTE: this is very non-ideal behaviour!) "decode duplicate (non-casematched) into nested struct 1": { encoded: []byte("a:\n a: 1\n b: 1\n c: test\n\nA:\n a: 2"), @@ -639,8 +681,8 @@ func TestYAMLToJSON(t *testing.T) { }, "boolean value (no)": { yaml: "t: no\n", - json: `{"t":false}`, - yamlReverseOverwrite: strPtr("t: false\n"), + json: `{"t":"no"}`, // In yamlv2, this was interpreted as the boolean false + yamlReverseOverwrite: strPtr("t: \"no\"\n"), }, "integer value (2^53 + 1)": { yaml: "t: 9007199254740993\n", @@ -668,8 +710,8 @@ func TestYAMLToJSON(t *testing.T) { }, "boolean key (no)": { yaml: "no: a", - json: `{"false":"a"}`, - yamlReverseOverwrite: strPtr("\"false\": a\n"), + json: `{"no":"a"}`, // In yamlv2, this was incorrectly converted to "false" + yamlReverseOverwrite: strPtr("\"no\": a\n"), }, "integer key": { yaml: "1: a",