diff --git a/.gitignore b/.gitignore index bd0517d9..d5f6a50c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /.cache/ +/.claude/ /yts/testdata/ /go-yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3031a8c9..36f6efb0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -105,10 +105,11 @@ foo: &a1 bar - Fork and clone the repository - Make your changes - Run tests, linters and formatters - - `make test-all` - - `make lint` - `make fmt` - `make tidy` + - `make lint` + - `make test` + - You can use `make check` to run all of the above - Submit a [Pull Request](https://github.com/yaml/go-yaml/pulls) diff --git a/GNUmakefile b/GNUmakefile index 17c23b18..bd5a3988 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -30,7 +30,6 @@ MAKES-CLEAN := $(CLI-BINARY) MAKES-REALCLEAN := $(dir $(YTS-DIR)) # Setup and include go.mk and shell.mk: -GO-FILES := $(shell find -not \( -path ./.cache -prune \) -name '*.go' | sort) GO-CMDS-SKIP := test fmt vet ifndef GO-VERSION-NEEDED GO-NO-DEP-GO := true @@ -58,19 +57,25 @@ SHELL-NAME := makes go-yaml include $(MAKES)/clean.mk include $(MAKES)/shell.mk -MAKES-CLEAN := $(dir $(YTS-DIR)) $(GOLANGCI-LINT) +MAKES-CLEAN += $(GOLANGCI-LINT) + +MAKE := $(MAKE) --no-print-directory v ?= count ?= 1 # Test rules: -test: $(GO-DEPS) - go test$(if $v, -v) -vet=off ./... +check: + $(MAKE) fmt + $(MAKE) tidy + $(MAKE) lint + $(MAKE) test -test-data: $(YTS-DIR) +test: test-unit test-cli test-yts-all -test-all: test test-yts-all +test-unit: $(GO-DEPS) + go test$(if $v, -v) test-yts: $(GO-DEPS) $(YTS-DIR) go test$(if $v, -v) ./yts -count=$(count) @@ -83,12 +88,14 @@ test-yts-fail: $(GO-DEPS) $(YTS-DIR) @echo 'Testing yaml-test-suite failures' @RUNFAILING=1 bash -c "$$yts_pass_fail" +test-cli: $(GO-DEPS) cli + go test$(if $v, -v) ./cmd/go-yaml -count=$(count) + +get-test-data: $(YTS-DIR) + # Install golangci-lint for GitHub Actions: golangci-lint-install: $(GOLANGCI-LINT) -fmt: $(GOLANGCI-LINT-VERSIONED) - $< fmt ./... - lint: $(GOLANGCI-LINT-VERSIONED) $< run ./... diff --git a/cmd/go-yaml/cli_test.go b/cmd/go-yaml/cli_test.go new file mode 100644 index 00000000..c844f72d --- /dev/null +++ b/cmd/go-yaml/cli_test.go @@ -0,0 +1,193 @@ +package main + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "go.yaml.in/yaml/v4" +) + +// TestCase represents a single test case from a test file +type TestCase struct { + Name string `yaml:"name"` + Text string `yaml:"text"` + Token string `yaml:"token,omitempty"` + TOKEN string `yaml:"TOKEN,omitempty"` + Event string `yaml:"event,omitempty"` + EVENT string `yaml:"EVENT,omitempty"` + Node string `yaml:"node,omitempty"` + NODE string `yaml:"NODE,omitempty"` + Yaml string `yaml:"yaml,omitempty"` + YAML string `yaml:"YAML,omitempty"` + Json string `yaml:"json,omitempty"` + JSON string `yaml:"JSON,omitempty"` +} + +// TestSuite is a sequence of test cases +type TestSuite []TestCase + +// flagMapping maps test file field names to CLI flags +var flagMapping = map[string]string{ + "token": "-t", + "TOKEN": "-T", + "event": "-e", + "EVENT": "-E", + "node": "-n", + "NODE": "-N", + "yaml": "-y", + "YAML": "-Y", + "json": "-j", + "JSON": "-J", +} + +func TestCLI(t *testing.T) { + // Find all test files in testdata/ + testFiles, err := filepath.Glob("testdata/*.yaml") + if err != nil { + t.Fatalf("Failed to find test files: %v", err) + } + + if len(testFiles) == 0 { + t.Skip("No test files found in testdata/") + } + + // Build the CLI binary if it doesn't exist + binaryPath := "../../go-yaml" + if _, err := os.Stat(binaryPath); os.IsNotExist(err) { + t.Logf("Building go-yaml binary...") + cmd := exec.Command("go", "build", "-o", binaryPath, ".") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("Failed to build go-yaml: %v\n%s", err, output) + } + } + + // Process each test file + for _, testFile := range testFiles { + testFileName := filepath.Base(testFile) + t.Run(testFileName, func(t *testing.T) { + runTestFile(t, testFile, binaryPath) + }) + } +} + +func runTestFile(t *testing.T, testFile, binaryPath string) { + // Read and parse the test file + data, err := os.ReadFile(testFile) + if err != nil { + t.Fatalf("Failed to read test file %s: %v", testFile, err) + } + + var suite TestSuite + if err := yaml.Unmarshal(data, &suite); err != nil { + t.Fatalf("Failed to parse test file %s: %v", testFile, err) + } + + // Run each test case + for _, testCase := range suite { + t.Run(testCase.Name, func(t *testing.T) { + runTestCase(t, testCase, binaryPath) + }) + } +} + +func runTestCase(t *testing.T, tc TestCase, binaryPath string) { + // Test each output format that has an expected value + tests := []struct { + field string + flag string + expected string + }{ + {"token", flagMapping["token"], tc.Token}, + {"TOKEN", flagMapping["TOKEN"], tc.TOKEN}, + {"event", flagMapping["event"], tc.Event}, + {"EVENT", flagMapping["EVENT"], tc.EVENT}, + {"node", flagMapping["node"], tc.Node}, + {"NODE", flagMapping["NODE"], tc.NODE}, + {"yaml", flagMapping["yaml"], tc.Yaml}, + {"YAML", flagMapping["YAML"], tc.YAML}, + {"json", flagMapping["json"], tc.Json}, + {"JSON", flagMapping["JSON"], tc.JSON}, + } + + for _, test := range tests { + if test.expected == "" { + continue // Skip if no expected output for this format + } + + t.Run(test.field, func(t *testing.T) { + // Run the CLI command + cmd := exec.Command(binaryPath, test.flag) + cmd.Stdin = strings.NewReader(tc.Text) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + t.Fatalf("Command failed: %v\nStderr: %s", err, stderr.String()) + } + + // Normalize output for comparison + actual := normalizeOutput(stdout.String()) + expected := normalizeOutput(test.expected) + + if actual != expected { + t.Errorf("Output mismatch for flag %s\nExpected:\n%s\n\nActual:\n%s\n\nDiff:\n%s", + test.flag, expected, actual, diff(expected, actual)) + } + }) + } +} + +// normalizeOutput trims whitespace and ensures consistent line endings +func normalizeOutput(s string) string { + s = strings.TrimSpace(s) + s = strings.ReplaceAll(s, "\r\n", "\n") + return s +} + +// diff provides a simple diff output for debugging +func diff(expected, actual string) string { + expLines := strings.Split(expected, "\n") + actLines := strings.Split(actual, "\n") + + maxLines := len(expLines) + if len(actLines) > maxLines { + maxLines = len(actLines) + } + + var result strings.Builder + for i := 0; i < maxLines; i++ { + expLine := "" + actLine := "" + + if i < len(expLines) { + expLine = expLines[i] + } + if i < len(actLines) { + actLine = actLines[i] + } + + if expLine != actLine { + result.WriteString("Line ") + result.WriteString(strings.Repeat(" ", len(strings.TrimSpace(expLine))+1)) + result.WriteString("\n") + if expLine != "" { + result.WriteString("- ") + result.WriteString(expLine) + result.WriteString("\n") + } + if actLine != "" { + result.WriteString("+ ") + result.WriteString(actLine) + result.WriteString("\n") + } + } + } + + return result.String() +} diff --git a/cmd/go-yaml/event.go b/cmd/go-yaml/event.go index f810ce63..60d923f7 100644 --- a/cmd/go-yaml/event.go +++ b/cmd/go-yaml/event.go @@ -10,6 +10,7 @@ import ( "strings" "go.yaml.in/yaml/v4" + "go.yaml.in/yaml/v4/internal/libyaml" ) // EventType represents the type of a YAML event @@ -44,15 +45,17 @@ type Event struct { // EventInfo represents the information about a YAML event for YAML encoding type EventInfo struct { - Event string `yaml:"Event"` - Value string `yaml:"Value,omitempty"` - Style string `yaml:"Style,omitempty"` - Tag string `yaml:"Tag,omitempty"` - Anchor string `yaml:"Anchor,omitempty"` - Head string `yaml:"Head,omitempty"` - Line string `yaml:"Line,omitempty"` - Foot string `yaml:"Foot,omitempty"` - Pos string `yaml:"Pos,omitempty"` + Event string `yaml:"event"` + Value string `yaml:"value,omitempty"` + Style string `yaml:"style,omitempty"` + Tag string `yaml:"tag,omitempty"` + Anchor string `yaml:"anchor,omitempty"` + Implicit *bool `yaml:"implicit,omitempty"` + Explicit *bool `yaml:"explicit,omitempty"` + Head string `yaml:"head,omitempty"` + Line string `yaml:"line,omitempty"` + Foot string `yaml:"foot,omitempty"` + Pos string `yaml:"pos,omitempty"` } // ProcessEvents reads YAML from stdin and outputs event information @@ -63,10 +66,24 @@ func ProcessEvents(profuse, compact, unmarshal bool) error { return processEventsDecode(profuse, compact) } -// processEventsDecode uses Decoder.Decode for YAML processing +// processEventsDecode uses yaml.Decoder for YAML processing with implicit field augmentation func processEventsDecode(profuse, compact bool) error { - decoder := yaml.NewDecoder(os.Stdin) - firstDoc := true + // Read all input from stdin + input, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("failed to read stdin: %w", err) + } + + // Get implicit flags from libyaml parser + implicitFlags, err := getDocumentImplicitFlags(input) + if err != nil { + return err + } + + // Use yaml.Decoder to get events with comments + decoder := yaml.NewDecoder(bytes.NewReader(input)) + docIndex := 0 + var allEvents []*Event for { var node yaml.Node @@ -78,104 +95,126 @@ func processEventsDecode(profuse, compact bool) error { return fmt.Errorf("failed to decode YAML: %w", err) } - // Add document separator for all documents except the first - if !firstDoc { - fmt.Println("---") - } - firstDoc = false - + // Get events from node (includes comments) events := processNodeToEvents(&node, profuse) - if compact { - // For compact mode, output each event as a flow style mapping in a sequence + // Augment document start/end events with implicit flags + if docIndex < len(implicitFlags) { for _, event := range events { - info := formatEventInfo(event, profuse) - - // Create a YAML node with flow style for the mapping - compactNode := &yaml.Node{ - Kind: yaml.MappingNode, - Style: yaml.FlowStyle, + switch event.Type { + case "DOCUMENT-START": + event.Implicit = implicitFlags[docIndex].StartImplicit + case "DOCUMENT-END": + event.Implicit = implicitFlags[docIndex].EndImplicit } + } + } + + allEvents = append(allEvents, events...) + docIndex++ + } + + events := allEvents + + if compact { + // For compact mode, output each event as a flow style mapping in a sequence + for _, event := range events { + info := formatEventInfo(event, profuse) + + // Create a YAML node with flow style for the mapping + compactNode := &yaml.Node{ + Kind: yaml.MappingNode, + Style: yaml.FlowStyle, + } + + // Add the Event field + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "event"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Event}) - // Add the Event field + // Add other fields if they exist + if info.Value != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Event"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Event}) - - // Add other fields if they exist - if info.Value != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Value"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Value}) - } - if info.Style != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Style"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Style}) - } - if info.Tag != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Tag"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Tag}) - } - if info.Anchor != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Anchor"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Anchor}) - } - if info.Head != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Head"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Head}) - } - if info.Line != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Line"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Line}) - } - if info.Foot != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Foot"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Foot}) - } - if info.Pos != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Pos"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Pos}) - } + &yaml.Node{Kind: yaml.ScalarNode, Value: "value"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Value}) + } + if info.Style != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "style"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Style}) + } + if info.Tag != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "tag"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Tag}) + } + if info.Anchor != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "anchor"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Anchor}) + } + if info.Implicit != nil { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "implicit"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%t", *info.Implicit)}) + } + if info.Explicit != nil { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "explicit"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%t", *info.Explicit)}) + } + if info.Head != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "head"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Head}) + } + if info.Line != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "line"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Line}) + } + if info.Foot != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "foot"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Foot}) + } + if info.Pos != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "pos"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Pos}) + } - var buf bytes.Buffer - enc := yaml.NewEncoder(&buf) - enc.SetIndent(2) - if err := enc.Encode([]*yaml.Node{compactNode}); err != nil { - enc.Close() - return fmt.Errorf("failed to marshal compact event info: %w", err) - } + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + if err := enc.Encode([]*yaml.Node{compactNode}); err != nil { enc.Close() - fmt.Print(buf.String()) + return fmt.Errorf("failed to marshal compact event info: %w", err) } - } else { - // For non-compact mode, output each event as a separate mapping - for _, event := range events { - info := formatEventInfo(event, profuse) - - var buf bytes.Buffer - enc := yaml.NewEncoder(&buf) - enc.SetIndent(2) - if err := enc.Encode([]*EventInfo{info}); err != nil { - enc.Close() - return fmt.Errorf("failed to marshal event info: %w", err) - } + enc.Close() + fmt.Print(buf.String()) + } + } else { + // For non-compact mode, output each event as a separate mapping + for _, event := range events { + info := formatEventInfo(event, profuse) + + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + if err := enc.Encode([]*EventInfo{info}); err != nil { enc.Close() - fmt.Print(buf.String()) + return fmt.Errorf("failed to marshal event info: %w", err) } + enc.Close() + fmt.Print(buf.String()) } } return nil } -// processEventsUnmarshal uses yaml.Unmarshal for YAML processing +// processEventsUnmarshal uses yaml.Unmarshal for YAML processing with implicit field augmentation func processEventsUnmarshal(profuse, compact bool) error { // Read all input from stdin input, err := io.ReadAll(os.Stdin) @@ -183,9 +222,16 @@ func processEventsUnmarshal(profuse, compact bool) error { return fmt.Errorf("failed to read stdin: %w", err) } + // Get implicit flags from libyaml parser + implicitFlags, err := getDocumentImplicitFlags(input) + if err != nil { + return err + } + // Split input into documents documents := bytes.Split(input, []byte("---")) - firstDoc := true + docIndex := 0 + var allEvents []*Event for _, doc := range documents { // Skip empty documents @@ -193,126 +239,152 @@ func processEventsUnmarshal(profuse, compact bool) error { continue } - // Add document separator for all documents except the first - if !firstDoc { - fmt.Println("---") - } - firstDoc = false - - // For unmarshal mode, use interface{} first to avoid preserving comments - var data interface{} - if err := yaml.Unmarshal(doc, &data); err != nil { - return fmt.Errorf("failed to unmarshal YAML: %w", err) - } - // Convert to yaml.Node for event processing var node yaml.Node if err := yaml.Unmarshal(doc, &node); err != nil { return fmt.Errorf("failed to unmarshal YAML to node: %w", err) } + // Get events from node (includes comments) events := processNodeToEvents(&node, profuse) - if compact { - // For compact mode, output each event as a flow style mapping in a sequence + // Augment document start/end events with implicit flags + if docIndex < len(implicitFlags) { for _, event := range events { - info := formatEventInfo(event, profuse) - - // Create a YAML node with flow style for the mapping - compactNode := &yaml.Node{ - Kind: yaml.MappingNode, - Style: yaml.FlowStyle, + switch event.Type { + case "DOCUMENT-START": + event.Implicit = implicitFlags[docIndex].StartImplicit + case "DOCUMENT-END": + event.Implicit = implicitFlags[docIndex].EndImplicit } + } + } + + allEvents = append(allEvents, events...) + docIndex++ + } + + events := allEvents + + if compact { + // For compact mode, output each event as a flow style mapping in a sequence + for _, event := range events { + info := formatEventInfo(event, profuse) - // Add the Event field + // Create a YAML node with flow style for the mapping + compactNode := &yaml.Node{ + Kind: yaml.MappingNode, + Style: yaml.FlowStyle, + } + + // Add the Event field + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "event"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Event}) + + // Add other fields if they exist + if info.Value != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Event"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Event}) - - // Add other fields if they exist - if info.Value != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Value"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Value}) - } - if info.Style != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Style"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Style}) - } - if info.Tag != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Tag"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Tag}) - } - if info.Anchor != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Anchor"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Anchor}) - } - if info.Head != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Head"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Head}) - } - if info.Line != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Line"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Line}) - } - if info.Foot != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Foot"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Foot}) - } - if info.Pos != "" { - compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Pos"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: info.Pos}) - } + &yaml.Node{Kind: yaml.ScalarNode, Value: "value"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Value}) + } + if info.Style != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "style"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Style}) + } + if info.Tag != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "tag"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Tag}) + } + if info.Anchor != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "anchor"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Anchor}) + } + if info.Implicit != nil { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "implicit"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%t", *info.Implicit)}) + } + if info.Explicit != nil { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "explicit"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%t", *info.Explicit)}) + } + if info.Head != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "head"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Head}) + } + if info.Line != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "line"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Line}) + } + if info.Foot != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "foot"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Foot}) + } + if info.Pos != "" { + compactNode.Content = append(compactNode.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "pos"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: info.Pos}) + } - var buf bytes.Buffer - enc := yaml.NewEncoder(&buf) - enc.SetIndent(2) - if err := enc.Encode([]*yaml.Node{compactNode}); err != nil { - enc.Close() - return fmt.Errorf("failed to marshal compact event info: %w", err) - } + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + if err := enc.Encode([]*yaml.Node{compactNode}); err != nil { enc.Close() - fmt.Print(buf.String()) + return fmt.Errorf("failed to marshal compact event info: %w", err) } - } else { - // For non-compact mode, output each event as a separate mapping - for _, event := range events { - info := formatEventInfo(event, profuse) - - var buf bytes.Buffer - enc := yaml.NewEncoder(&buf) - enc.SetIndent(2) - if err := enc.Encode([]*EventInfo{info}); err != nil { - enc.Close() - return fmt.Errorf("failed to marshal event info: %w", err) - } + enc.Close() + fmt.Print(buf.String()) + } + } else { + // For non-compact mode, output each event as a separate mapping + for _, event := range events { + info := formatEventInfo(event, profuse) + + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + if err := enc.Encode([]*EventInfo{info}); err != nil { enc.Close() - fmt.Print(buf.String()) + return fmt.Errorf("failed to marshal event info: %w", err) } + enc.Close() + fmt.Print(buf.String()) } } return nil } +// adjustColumn converts yaml.Node 1-based column to 0-based (libyaml format) +func adjustColumn(col int) int { + if col > 0 { + return col - 1 + } + return 0 +} + // processNodeToEvents converts a node to a slice of events for compact output func processNodeToEvents(node *yaml.Node, profuse bool) []*Event { var events []*Event // Add document start event + // yaml.Node uses 1-based columns, but we need 0-based (libyaml format) + startCol := adjustColumn(node.Column) events = append(events, &Event{ Type: "DOCUMENT-START", StartLine: node.Line, - StartColumn: node.Column, + StartColumn: startCol, EndLine: node.Line, - EndColumn: node.Column, + EndColumn: startCol, }) // Process the node content @@ -322,9 +394,9 @@ func processNodeToEvents(node *yaml.Node, profuse bool) []*Event { events = append(events, &Event{ Type: "DOCUMENT-END", StartLine: node.Line, - StartColumn: node.Column, + StartColumn: startCol, EndLine: node.Line, - EndColumn: node.Column, + EndColumn: startCol, }) return events @@ -340,13 +412,14 @@ func processNodeToEventsRecursive(node *yaml.Node, profuse bool) []*Event { events = append(events, processNodeToEventsRecursive(child, profuse)...) } case yaml.MappingNode: + startCol := adjustColumn(node.Column) events = append(events, &Event{ Type: "MAPPING-START", StartLine: node.Line, - StartColumn: node.Column, + StartColumn: startCol, EndLine: node.Line, - EndColumn: node.Column, - Style: formatStyle(node.Style), + EndColumn: startCol, + Style: formatStyle(node.Style, profuse), HeadComment: node.HeadComment, LineComment: node.LineComment, FootComment: node.FootComment, @@ -364,18 +437,19 @@ func processNodeToEventsRecursive(node *yaml.Node, profuse bool) []*Event { events = append(events, &Event{ Type: "MAPPING-END", StartLine: node.Line, - StartColumn: node.Column, + StartColumn: startCol, EndLine: node.Line, - EndColumn: node.Column, + EndColumn: startCol, }) case yaml.SequenceNode: + startCol := adjustColumn(node.Column) events = append(events, &Event{ Type: "SEQUENCE-START", StartLine: node.Line, - StartColumn: node.Column, + StartColumn: startCol, EndLine: node.Line, - EndColumn: node.Column, - Style: formatStyle(node.Style), + EndColumn: startCol, + Style: formatStyle(node.Style, profuse), HeadComment: node.HeadComment, LineComment: node.LineComment, FootComment: node.FootComment, @@ -387,22 +461,20 @@ func processNodeToEventsRecursive(node *yaml.Node, profuse bool) []*Event { events = append(events, &Event{ Type: "SEQUENCE-END", StartLine: node.Line, - StartColumn: node.Column, + StartColumn: startCol, EndLine: node.Line, - EndColumn: node.Column, + EndColumn: startCol, }) case yaml.ScalarNode: // Calculate end position for scalars based on value length + // yaml.Node uses 1-based columns, adjust to 0-based + startCol := adjustColumn(node.Column) endLine := node.Line - endColumn := node.Column + endColumn := startCol if node.Value != "" { // For single-line values, add the length to the column if !strings.Contains(node.Value, "\n") { - endColumn += len(node.Value) - } else { - // For multi-line values, we'd need more complex logic - // For now, just use the start position - endColumn = node.Column + endColumn = startCol + len(node.Value) } } @@ -425,23 +497,24 @@ func processNodeToEventsRecursive(node *yaml.Node, profuse bool) []*Event { Anchor: node.Anchor, Tag: tag, StartLine: node.Line, - StartColumn: node.Column, + StartColumn: startCol, EndLine: endLine, EndColumn: endColumn, - Style: formatStyle(node.Style), + Style: formatStyle(node.Style, profuse), HeadComment: node.HeadComment, LineComment: node.LineComment, FootComment: node.FootComment, }) case yaml.AliasNode: // Generate ALIAS event for alias nodes + startCol := adjustColumn(node.Column) events = append(events, &Event{ Type: "ALIAS", Value: node.Value, StartLine: node.Line, - StartColumn: node.Column, + StartColumn: startCol, EndLine: node.Line, - EndColumn: node.Column, + EndColumn: startCol, HeadComment: node.HeadComment, LineComment: node.LineComment, FootComment: node.FootComment, @@ -480,11 +553,176 @@ func formatEventInfo(event *Event, profuse bool) *EventInfo { } if profuse { if event.StartLine == event.EndLine && event.StartColumn == event.EndColumn { - info.Pos = fmt.Sprintf("%d;%d", event.StartLine, event.StartColumn) + // Single position + info.Pos = fmt.Sprintf("%d/%d", event.StartLine, event.StartColumn) + } else if event.StartLine == event.EndLine { + // Range on same line + info.Pos = fmt.Sprintf("%d/%d-%d", event.StartLine, event.StartColumn, event.EndColumn) } else { - info.Pos = fmt.Sprintf("%d;%d-%d;%d", event.StartLine, event.StartColumn, event.EndLine, event.EndColumn) + // Range across different lines + info.Pos = fmt.Sprintf("%d/%d-%d/%d", event.StartLine, event.StartColumn, event.EndLine, event.EndColumn) + } + } + + // Handle implicit/explicit for document start/end events + if event.Type == "DOCUMENT-START" || event.Type == "DOCUMENT-END" { + if profuse { + // For -E mode: show implicit: true when implicit + if event.Implicit { + trueVal := true + info.Implicit = &trueVal + } + } else { + // For -e mode: show explicit: true when NOT implicit + if !event.Implicit { + trueVal := true + info.Explicit = &trueVal + } } } return info } + +// DocumentImplicitFlags holds implicit flags for document start and end events +type DocumentImplicitFlags struct { + StartImplicit bool + EndImplicit bool +} + +// getDocumentImplicitFlags extracts implicit flags for all documents +func getDocumentImplicitFlags(input []byte) ([]*DocumentImplicitFlags, error) { + p := libyaml.NewParser() + if len(input) == 0 { + input = []byte{'\n'} + } + p.SetInputString(input) + + var flags []*DocumentImplicitFlags + var currentDoc *DocumentImplicitFlags + var ev libyaml.Event + + for { + if !p.Parse(&ev) { + return nil, fmt.Errorf("failed to parse YAML: %s", p.Problem) + } + + switch ev.Type { + case libyaml.DOCUMENT_START_EVENT: + currentDoc = &DocumentImplicitFlags{ + StartImplicit: ev.Implicit, + } + flags = append(flags, currentDoc) + case libyaml.DOCUMENT_END_EVENT: + if currentDoc != nil { + currentDoc.EndImplicit = ev.Implicit + } + case libyaml.STREAM_END_EVENT: + ev.Delete() + return flags, nil + } + + ev.Delete() + } +} + +// getEventsFromParser parses YAML input and extracts events with implicit field information +func getEventsFromParser(input []byte, profuse bool) ([]*Event, error) { + p := libyaml.NewParser() + if len(input) == 0 { + input = []byte{'\n'} + } + p.SetInputString(input) + + var events []*Event + var ev libyaml.Event + + for { + if !p.Parse(&ev) { + return nil, fmt.Errorf("failed to parse YAML: %s", p.Problem) + } + + event := convertLibyamlEvent(&ev, profuse) + if event != nil { + events = append(events, event) + } + + if ev.Type == libyaml.STREAM_END_EVENT { + ev.Delete() + break + } + ev.Delete() + } + + return events, nil +} + +// convertLibyamlEvent converts a libyaml event to our Event struct +func convertLibyamlEvent(ev *libyaml.Event, profuse bool) *Event { + // Skip stream events + if ev.Type == libyaml.STREAM_START_EVENT || ev.Type == libyaml.STREAM_END_EVENT { + return nil + } + + event := &Event{ + StartLine: ev.StartMark.Line + 1, // libyaml uses 0-based lines + StartColumn: ev.StartMark.Column, + EndLine: ev.EndMark.Line + 1, + EndColumn: ev.EndMark.Column, + } + + switch ev.Type { + case libyaml.DOCUMENT_START_EVENT: + event.Type = "DOCUMENT-START" + event.Implicit = ev.Implicit + case libyaml.DOCUMENT_END_EVENT: + event.Type = "DOCUMENT-END" + event.Implicit = ev.Implicit + case libyaml.MAPPING_START_EVENT: + event.Type = "MAPPING-START" + event.Anchor = string(ev.Anchor) + event.Tag = string(ev.Tag) + // Style handling for mapping + if ev.MappingStyle() == libyaml.FLOW_MAPPING_STYLE { + event.Style = "Flow" + } + case libyaml.MAPPING_END_EVENT: + event.Type = "MAPPING-END" + case libyaml.SEQUENCE_START_EVENT: + event.Type = "SEQUENCE-START" + event.Anchor = string(ev.Anchor) + event.Tag = string(ev.Tag) + // Style handling for sequence + if ev.SequenceStyle() == libyaml.FLOW_SEQUENCE_STYLE { + event.Style = "Flow" + } + case libyaml.SEQUENCE_END_EVENT: + event.Type = "SEQUENCE-END" + case libyaml.SCALAR_EVENT: + event.Type = "SCALAR" + event.Value = string(ev.Value) + event.Anchor = string(ev.Anchor) + event.Tag = string(ev.Tag) + event.Implicit = ev.Implicit + // Style handling for scalar + switch ev.ScalarStyle() { + case libyaml.PLAIN_SCALAR_STYLE: + if profuse { + event.Style = "Plain" + } + case libyaml.DOUBLE_QUOTED_SCALAR_STYLE: + event.Style = "Double" + case libyaml.SINGLE_QUOTED_SCALAR_STYLE: + event.Style = "Single" + case libyaml.LITERAL_SCALAR_STYLE: + event.Style = "Literal" + case libyaml.FOLDED_SCALAR_STYLE: + event.Style = "Folded" + } + case libyaml.ALIAS_EVENT: + event.Type = "ALIAS" + event.Anchor = string(ev.Anchor) + } + + return event +} diff --git a/cmd/go-yaml/main.go b/cmd/go-yaml/main.go index e3d6b360..e88de603 100644 --- a/cmd/go-yaml/main.go +++ b/cmd/go-yaml/main.go @@ -40,8 +40,9 @@ func main() { eventMode := flag.Bool("e", false, "Event output") eventProfuseMode := flag.Bool("E", false, "Event with line info") - // Node mode + // Node modes nodeMode := flag.Bool("n", false, "Node representation output") + nodeProfuseMode := flag.Bool("N", false, "Node with tag and style for all scalars") // Shared flags longMode := flag.Bool("l", false, "Long (block) formatted output") @@ -59,6 +60,7 @@ func main() { flag.BoolVar(eventMode, "event", false, "Event output") flag.BoolVar(eventProfuseMode, "EVENT", false, "Event with line info") flag.BoolVar(nodeMode, "node", false, "Node representation output") + flag.BoolVar(nodeProfuseMode, "NODE", false, "Node with tag and style for all scalars") flag.BoolVar(longMode, "long", false, "Long (block) formatted output") flag.BoolVar(unmarshalMode, "unmarshal", false, "Use Unmarshal instead of Decode for YAML input") flag.BoolVar(marshalMode, "marshal", false, "Use Marshal instead of Encode for YAML output") @@ -95,14 +97,14 @@ func main() { } // If no stdin and no flags, show help - if (stat.Mode()&os.ModeCharDevice) != 0 && !*nodeMode && !*eventMode && !*eventProfuseMode && !*tokenMode && !*tokenProfuseMode && !*jsonMode && !*jsonPrettyMode && !*yamlMode && !*yamlPreserveMode && !*longMode { + if (stat.Mode()&os.ModeCharDevice) != 0 && !*nodeMode && !*nodeProfuseMode && !*eventMode && !*eventProfuseMode && !*tokenMode && !*tokenProfuseMode && !*jsonMode && !*jsonPrettyMode && !*yamlMode && !*yamlPreserveMode && !*longMode { printHelp() return } // Error if stdin has data but no mode flags are provided - if (stat.Mode()&os.ModeCharDevice) == 0 && !*nodeMode && !*eventMode && !*eventProfuseMode && !*tokenMode && !*tokenProfuseMode && !*jsonMode && !*jsonPrettyMode && !*yamlMode && !*yamlPreserveMode && !*longMode { - fmt.Fprintf(os.Stderr, "Error: stdin has data but no mode specified. Use -n/--node, -e/--event, -E/--EVENT, -t/--token, -T/--TOKEN, -j/--json, -J/--JSON, -y/--yaml, -Y/--YAML flag.\n") + if (stat.Mode()&os.ModeCharDevice) == 0 && !*nodeMode && !*nodeProfuseMode && !*eventMode && !*eventProfuseMode && !*tokenMode && !*tokenProfuseMode && !*jsonMode && !*jsonPrettyMode && !*yamlMode && !*yamlPreserveMode && !*longMode { + fmt.Fprintf(os.Stderr, "Error: stdin has data but no mode specified. Use -n/--node, -N/--NODE, -e/--event, -E/--EVENT, -t/--token, -T/--TOKEN, -j/--json, -J/--JSON, -y/--yaml, -Y/--YAML flag.\n") os.Exit(1) } @@ -153,16 +155,19 @@ func main() { } } else { // Use node formatting mode (default) + profuse := *nodeProfuseMode if *unmarshalMode { // Use Unmarshal mode - if err := ProcessNodeUnmarshal(); err != nil { + if err := ProcessNodeUnmarshal(profuse); err != nil { log.Fatal("Failed to process YAML node:", err) } } else { // Use Decode mode (original behavior) reader := io.Reader(os.Stdin) dec := yaml.NewDecoder(reader) - firstDoc := true + + // Collect all documents + var docs []interface{} for { var node yaml.Node @@ -174,30 +179,39 @@ func main() { log.Fatal("Failed to load YAML node:", err) } - // Add document separator for all documents except the first - if !firstDoc { - fmt.Println("---") + var info interface{} + if profuse { + info = FormatNode(node, profuse) + } else { + info = FormatNodeCompact(node) } - firstDoc = false + docs = append(docs, info) + } - info := FormatNode(node) + // Output as sequence if multiple documents, otherwise output single document + var output interface{} + if len(docs) == 1 { + output = docs[0] + } else { + output = docs + } - // Use encoder with 2-space indentation - var buf bytes.Buffer - enc := yaml.NewEncoder(&buf) - enc.SetIndent(2) - if err := enc.Encode(info); err != nil { - log.Fatal("Failed to marshal node info:", err) - } - enc.Close() - fmt.Print(buf.String()) + // Use encoder with 2-space indentation + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + enc.CompactSeqIndent() + if err := enc.Encode(output); err != nil { + log.Fatal("Failed to marshal node info:", err) } + enc.Close() + fmt.Print(buf.String()) } } } // ProcessNodeUnmarshal reads YAML from stdin using Unmarshal and outputs node structure -func ProcessNodeUnmarshal() error { +func ProcessNodeUnmarshal(profuse bool) error { // Read all input from stdin input, err := io.ReadAll(os.Stdin) if err != nil { @@ -206,7 +220,9 @@ func ProcessNodeUnmarshal() error { // Split input into documents documents := bytes.Split(input, []byte("---")) - firstDoc := true + + // Collect all documents + var docs []interface{} for _, doc := range documents { // Skip empty documents @@ -214,12 +230,6 @@ func ProcessNodeUnmarshal() error { continue } - // Add document separator for all documents except the first - if !firstDoc { - fmt.Println("---") - } - firstDoc = false - // For unmarshal mode, use interface{} first to avoid preserving comments var data interface{} if err := yaml.Unmarshal(doc, &data); err != nil { @@ -232,19 +242,34 @@ func ProcessNodeUnmarshal() error { return fmt.Errorf("failed to unmarshal YAML to node: %w", err) } - info := FormatNode(node) - - // Use encoder with 2-space indentation - var buf bytes.Buffer - enc := yaml.NewEncoder(&buf) - enc.SetIndent(2) - if err := enc.Encode(info); err != nil { - enc.Close() - return fmt.Errorf("failed to marshal node info: %w", err) + var info interface{} + if profuse { + info = FormatNode(node, profuse) + } else { + info = FormatNodeCompact(node) } + docs = append(docs, info) + } + + // Output as sequence if multiple documents, otherwise output single document + var output interface{} + if len(docs) == 1 { + output = docs[0] + } else { + output = docs + } + + // Use encoder with 2-space indentation + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + enc.CompactSeqIndent() + if err := enc.Encode(output); err != nil { enc.Close() - fmt.Print(buf.String()) + return fmt.Errorf("failed to marshal node info: %w", err) } + enc.Close() + fmt.Print(buf.String()) return nil } @@ -281,6 +306,7 @@ Options: -E, --EVENT Event with line info -n, --node Node representation output + -N, --NODE Node with tag and style for all scalars -l, --long Long (block) formatted output diff --git a/cmd/go-yaml/node.go b/cmd/go-yaml/node.go index a2229b0f..03f8819f 100644 --- a/cmd/go-yaml/node.go +++ b/cmd/go-yaml/node.go @@ -6,12 +6,44 @@ import ( "go.yaml.in/yaml/v4" ) +// MapItem is a single item in a MapSlice. +type MapItem struct { + Key string + Value interface{} +} + +// MapSlice is a slice of MapItems that preserves order when marshaled to YAML. +type MapSlice []MapItem + +// MarshalYAML implements yaml.Marshaler for MapSlice to preserve key order. +func (ms MapSlice) MarshalYAML() (interface{}, error) { + // Convert MapSlice to yaml.Node with MappingNode kind to preserve order + node := &yaml.Node{ + Kind: yaml.MappingNode, + } + for _, item := range ms { + // Add key node + keyNode := &yaml.Node{ + Kind: yaml.ScalarNode, + Value: item.Key, + } + // Add value node - let yaml encoder handle the value + // Encode item.Value directly into a yaml.Node + valueNode := &yaml.Node{} + if err := valueNode.Encode(item.Value); err != nil { + return nil, err + } + node.Content = append(node.Content, keyNode, valueNode) + } + return node, nil +} + // NodeInfo represents the information about a YAML node type NodeInfo struct { Kind string `yaml:"kind"` Style string `yaml:"style,omitempty"` - Anchor string `yaml:"anchor,omitempty"` Tag string `yaml:"tag,omitempty"` + Anchor string `yaml:"anchor,omitempty"` Head string `yaml:"head,omitempty"` Line string `yaml:"line,omitempty"` Foot string `yaml:"foot,omitempty"` @@ -20,18 +52,21 @@ type NodeInfo struct { } // FormatNode converts a YAML node into a NodeInfo structure -func FormatNode(n yaml.Node) *NodeInfo { +func FormatNode(n yaml.Node, profuse bool) *NodeInfo { info := &NodeInfo{ Kind: formatKind(n.Kind), } - if style := formatStyle(n.Style); style != "" { - info.Style = style + // Don't set style for Document nodes + if n.Kind != yaml.DocumentNode { + if style := formatStyle(n.Style, profuse); style != "" { + info.Style = style + } } if n.Anchor != "" { info.Anchor = n.Anchor } - if tag := formatTag(n.Tag, n.Style); tag != "" { + if tag := formatTag(n.Tag, n.Style, profuse); tag != "" { info.Tag = tag } if n.HeadComment != "" { @@ -49,7 +84,7 @@ func FormatNode(n yaml.Node) *NodeInfo { } else if n.Content != nil { info.Content = make([]*NodeInfo, len(n.Content)) for i, node := range n.Content { - info.Content[i] = FormatNode(*node) + info.Content[i] = FormatNode(*node, profuse) } } @@ -75,8 +110,11 @@ func formatKind(k yaml.Kind) string { } // formatStyle converts a YAML node style into its string representation. -func formatStyle(s yaml.Style) string { - switch s { +func formatStyle(s yaml.Style, profuse bool) string { + // Remove tagged style bit for checking base style + baseStyle := s &^ yaml.TaggedStyle + + switch baseStyle { case yaml.DoubleQuotedStyle: return "Double" case yaml.SingleQuotedStyle: @@ -87,24 +125,199 @@ func formatStyle(s yaml.Style) string { return "Folded" case yaml.FlowStyle: return "Flow" + case 0: + // Plain style - only show if profuse + if profuse { + return "Plain" + } } return "" } +// formatStyleName converts a YAML node style into a lowercase style name. +// Always returns a style name (defaults to "plain" for style 0). +func formatStyleName(s yaml.Style) string { + // Remove tagged style bit for checking base style + baseStyle := s &^ yaml.TaggedStyle + + switch baseStyle { + case yaml.DoubleQuotedStyle: + return "double" + case yaml.SingleQuotedStyle: + return "single" + case yaml.LiteralStyle: + return "literal" + case yaml.FoldedStyle: + return "folded" + case yaml.FlowStyle: + return "flow" + default: + return "plain" + } +} + // formatTag converts a YAML tag string to its string representation. -func formatTag(tag string, style yaml.Style) string { +func formatTag(tag string, style yaml.Style, profuse bool) string { // Check if the tag was explicit in the input tagWasExplicit := style&yaml.TaggedStyle != 0 - // Show !!str only if it was explicit in the input + // In profuse mode, always show tag + if profuse { + return tag + } + + // Default YAML tags - only show if they were explicit in the input switch tag { - case "!!str", "!!map", "!!seq": + case "!!str", "!!map", "!!seq", "!!int", "!!float", "!!bool", "!!null": if tagWasExplicit { return tag } return "" } - // Show all other tags + // Show all other tags (custom tags) return tag } + +// FormatNodeCompact converts a YAML node into a compact representation. +// Document nodes return their content directly. +// Mapping/Sequence nodes use lowercase keys: "mapping:", "sequence:". +// Scalar nodes use style as key: "plain:", "double:", etc. +func FormatNodeCompact(n yaml.Node) interface{} { + switch n.Kind { + case yaml.DocumentNode: + // Check if document has properties that need to be preserved + hasProperties := n.Anchor != "" || n.HeadComment != "" || n.LineComment != "" || n.FootComment != "" + if tag := formatTag(n.Tag, n.Style, false); tag != "" && tag != "!!str" { + hasProperties = true + } + + // If document has no properties, return content directly (unwrap) + if !hasProperties { + if len(n.Content) > 0 { + return FormatNodeCompact(*n.Content[0]) + } + return nil + } + + // Document has properties - create a result map with ordered keys + result := MapSlice{} + + // Add optional fields in order: tag, anchor, comments + if tag := formatTag(n.Tag, n.Style, false); tag != "" && tag != "!!str" { + result = append(result, MapItem{Key: "tag", Value: tag}) + } + if n.Anchor != "" { + result = append(result, MapItem{Key: "anchor", Value: n.Anchor}) + } + if n.HeadComment != "" { + result = append(result, MapItem{Key: "head", Value: n.HeadComment}) + } + if n.LineComment != "" { + result = append(result, MapItem{Key: "line", Value: n.LineComment}) + } + if n.FootComment != "" { + result = append(result, MapItem{Key: "foot", Value: n.FootComment}) + } + + // Add content if present + if len(n.Content) > 0 { + content := FormatNodeCompact(*n.Content[0]) + // Merge the content into result at the top level + if contentMap, ok := content.(MapSlice); ok { + result = append(result, contentMap...) + } + } + + return result + + case yaml.MappingNode: + result := MapSlice{} + + // Add optional fields in order: tag, anchor, comments + if tag := formatTag(n.Tag, n.Style, false); tag != "" && tag != "!!str" { + result = append(result, MapItem{Key: "tag", Value: tag}) + } + if n.Anchor != "" { + result = append(result, MapItem{Key: "anchor", Value: n.Anchor}) + } + if n.HeadComment != "" { + result = append(result, MapItem{Key: "head", Value: n.HeadComment}) + } + if n.LineComment != "" { + result = append(result, MapItem{Key: "line", Value: n.LineComment}) + } + if n.FootComment != "" { + result = append(result, MapItem{Key: "foot", Value: n.FootComment}) + } + + // Convert content (added last) + var content []interface{} + for _, node := range n.Content { + content = append(content, FormatNodeCompact(*node)) + } + result = append(result, MapItem{Key: "mapping", Value: content}) + return result + + case yaml.SequenceNode: + result := MapSlice{} + + // Add optional fields in order: tag, anchor, comments + if tag := formatTag(n.Tag, n.Style, false); tag != "" && tag != "!!str" { + result = append(result, MapItem{Key: "tag", Value: tag}) + } + if n.Anchor != "" { + result = append(result, MapItem{Key: "anchor", Value: n.Anchor}) + } + if n.HeadComment != "" { + result = append(result, MapItem{Key: "head", Value: n.HeadComment}) + } + if n.LineComment != "" { + result = append(result, MapItem{Key: "line", Value: n.LineComment}) + } + if n.FootComment != "" { + result = append(result, MapItem{Key: "foot", Value: n.FootComment}) + } + + // Convert content (added last) + var content []interface{} + for _, node := range n.Content { + content = append(content, FormatNodeCompact(*node)) + } + result = append(result, MapItem{Key: "sequence", Value: content}) + return result + + case yaml.ScalarNode: + result := MapSlice{} + + // Add optional fields in order: tag, anchor, comments + if tag := formatTag(n.Tag, n.Style, false); tag != "" && tag != "!!str" { + result = append(result, MapItem{Key: "tag", Value: tag}) + } + if n.Anchor != "" { + result = append(result, MapItem{Key: "anchor", Value: n.Anchor}) + } + if n.HeadComment != "" { + result = append(result, MapItem{Key: "head", Value: n.HeadComment}) + } + if n.LineComment != "" { + result = append(result, MapItem{Key: "line", Value: n.LineComment}) + } + if n.FootComment != "" { + result = append(result, MapItem{Key: "foot", Value: n.FootComment}) + } + + // Use style name as the key (added last) + styleName := formatStyleName(n.Style) + result = append(result, MapItem{Key: styleName, Value: n.Value}) + return result + + case yaml.AliasNode: + result := MapSlice{} + result = append(result, MapItem{Key: "alias", Value: n.Value}) + return result + + default: + return nil + } +} diff --git a/cmd/go-yaml/parser.go b/cmd/go-yaml/parser.go index d42d1e26..92b93fb0 100644 --- a/cmd/go-yaml/parser.go +++ b/cmd/go-yaml/parser.go @@ -9,8 +9,10 @@ import ( // Parser provides access to the internal YAML Parser for CLI use type Parser struct { - parser libyaml.Parser - done bool + parser libyaml.Parser + done bool + pendingTokens []*Token + commentsHead int } // NewParser creates a new YAML parser reading from the given reader for CLI use @@ -24,6 +26,13 @@ func NewParser(reader io.Reader) (*Parser, error) { // Next returns the next token in the YAML stream func (p *Parser) Next() (*Token, error) { + // Return pending tokens first + if len(p.pendingTokens) > 0 { + token := p.pendingTokens[0] + p.pendingTokens = p.pendingTokens[1:] + return token, nil + } + if p.done { return nil, nil } @@ -45,28 +54,6 @@ func (p *Parser) Next() (*Token, error) { EndColumn: int(yamlToken.EndMark.Column), } - // Call unfoldComments to process comment information from the parser - // This moves comments from the comments queue to the parser's comment fields - p.parser.UnfoldComments(&yamlToken) - - // Access comment information from the parser - // The parser stores comments in head_comment, line_comment, and foot_comment fields - if len(p.parser.HeadComment) > 0 { - token.HeadComment = string(p.parser.HeadComment) - // Clear the comment after using it to avoid duplication - p.parser.HeadComment = nil - } - if len(p.parser.LineComment) > 0 { - token.LineComment = string(p.parser.LineComment) - // Clear the comment after using it to avoid duplication - p.parser.LineComment = nil - } - if len(p.parser.FootComment) > 0 { - token.FootComment = string(p.parser.FootComment) - // Clear the comment after using it to avoid duplication - p.parser.FootComment = nil - } - switch yamlToken.Type { case libyaml.STREAM_START_TOKEN: token.Type = "STREAM-START" @@ -120,9 +107,61 @@ func (p *Parser) Next() (*Token, error) { token.Type = "UNKNOWN" } + // Process comments that should be emitted before this token + p.processComments(&yamlToken, token) + + // Return first pending token if comments were queued, otherwise return the main token + if len(p.pendingTokens) > 0 { + // Add the main token to the end of pending tokens + p.pendingTokens = append(p.pendingTokens, token) + // Return the first pending token + result := p.pendingTokens[0] + p.pendingTokens = p.pendingTokens[1:] + return result, nil + } + return token, nil } +// processComments extracts comments from the parser and creates COMMENT tokens +func (p *Parser) processComments(yamlToken *libyaml.Token, mainToken *Token) { + comments := p.parser.GetPendingComments() + + for p.commentsHead < len(comments) { + comment := &comments[p.commentsHead] + + // Check if this comment should be emitted before the current token + // Comments are associated with tokens based on their TokenMark + if yamlToken.StartMark.Index < comment.TokenMark.Index { + // This comment is for a future token, stop processing + break + } + + // Create comment tokens for head, line, and foot comments + p.appendCommentTokenIfNotEmpty(comment.Head, "head", comment) + p.appendCommentTokenIfNotEmpty(comment.Line, "line", comment) + p.appendCommentTokenIfNotEmpty(comment.Foot, "foot", comment) + + p.commentsHead++ + } +} + +// appendCommentTokenIfNotEmpty creates and appends a comment token if the value is not empty. +func (p *Parser) appendCommentTokenIfNotEmpty(value []byte, commentType string, comment *libyaml.Comment) { + if len(value) > 0 { + commentToken := &Token{ + Type: "COMMENT", + Value: string(value), + CommentType: commentType, + StartLine: int(comment.StartMark.Line) + 1, + StartColumn: int(comment.StartMark.Column), + EndLine: int(comment.EndMark.Line) + 1, + EndColumn: int(comment.EndMark.Column), + } + p.pendingTokens = append(p.pendingTokens, commentToken) + } +} + // Close releases the parser resources func (p *Parser) Close() { p.parser.Delete() diff --git a/cmd/go-yaml/testdata/basic.yaml b/cmd/go-yaml/testdata/basic.yaml new file mode 100644 index 00000000..be216e84 --- /dev/null +++ b/cmd/go-yaml/testdata/basic.yaml @@ -0,0 +1,94 @@ +# Basic functionality tests for go-yaml CLI + +- name: Simple scalar mapping + text: | + key: value + token: | + - {token: STREAM-START} + - {token: BLOCK-MAPPING-START} + - {token: KEY} + - {token: SCALAR, value: key} + - {token: VALUE} + - {token: SCALAR, value: value} + - {token: BLOCK-END} + - {token: STREAM-END} + event: | + - {event: DOCUMENT-START} + - {event: MAPPING-START} + - {event: SCALAR, value: key} + - {event: SCALAR, value: value} + - {event: MAPPING-END} + - {event: DOCUMENT-END} + node: | + mapping: + - plain: key + - plain: value + +- name: Simple sequence + text: | + - item1 + - item2 + token: | + - {token: STREAM-START} + - {token: BLOCK-SEQUENCE-START} + - {token: BLOCK-ENTRY} + - {token: SCALAR, value: item1} + - {token: BLOCK-ENTRY} + - {token: SCALAR, value: item2} + - {token: BLOCK-END} + - {token: STREAM-END} + event: | + - {event: DOCUMENT-START} + - {event: SEQUENCE-START} + - {event: SCALAR, value: item1} + - {event: SCALAR, value: item2} + - {event: SEQUENCE-END} + - {event: DOCUMENT-END} + +- name: Nested mapping and sequence + text: | + list: + - a + - b + token: | + - {token: STREAM-START} + - {token: BLOCK-MAPPING-START} + - {token: KEY} + - {token: SCALAR, value: list} + - {token: VALUE} + - {token: BLOCK-SEQUENCE-START} + - {token: BLOCK-ENTRY} + - {token: SCALAR, value: a} + - {token: BLOCK-ENTRY} + - {token: SCALAR, value: b} + - {token: BLOCK-END} + - {token: BLOCK-END} + - {token: STREAM-END} + event: | + - {event: DOCUMENT-START} + - {event: MAPPING-START} + - {event: SCALAR, value: list} + - {event: SEQUENCE-START} + - {event: SCALAR, value: a} + - {event: SCALAR, value: b} + - {event: SEQUENCE-END} + - {event: MAPPING-END} + - {event: DOCUMENT-END} + +- name: Flow style mapping + text: | + {a: b, c: d} + token: | + - {token: STREAM-START} + - {token: FLOW-MAPPING-START} + - {token: KEY} + - {token: SCALAR, value: a} + - {token: VALUE} + - {token: SCALAR, value: b} + - {token: FLOW-ENTRY} + - {token: KEY} + - {token: SCALAR, value: c} + - {token: VALUE} + - {token: SCALAR, value: d} + - {token: FLOW-MAPPING-END} + - {token: STREAM-END} diff --git a/cmd/go-yaml/testdata/comments.yaml b/cmd/go-yaml/testdata/comments.yaml new file mode 100644 index 00000000..35fa2b31 --- /dev/null +++ b/cmd/go-yaml/testdata/comments.yaml @@ -0,0 +1,73 @@ +# Comment-related tests for go-yaml CLI + +- name: Simple line comment on a pair + text: | + a: b #c + token: | + - {token: STREAM-START} + - {token: BLOCK-MAPPING-START} + - {token: KEY} + - {token: SCALAR, value: a} + - {token: VALUE} + - {token: COMMENT, line: '#c'} + - {token: SCALAR, value: b} + - {token: BLOCK-END} + - {token: STREAM-END} + TOKEN: | + - {token: STREAM-START, pos: 1/0} + - {token: BLOCK-MAPPING-START, pos: 1/0} + - {token: KEY, pos: 1/0} + - {token: SCALAR, value: a, pos: 1/0-1} + - {token: VALUE, pos: 1/1-2} + - {token: COMMENT, line: '#c', pos: 1/5-7} + - {token: SCALAR, value: b, pos: 1/3-4} + - {token: BLOCK-END, pos: 2/0} + - {token: STREAM-END, pos: 2/0} + event: | + - {event: DOCUMENT-START} + - {event: MAPPING-START} + - {event: SCALAR, value: a} + - {event: SCALAR, value: b, line: '#c'} + - {event: MAPPING-END} + - {event: DOCUMENT-END} + +- name: Head, line, and foot comments + text: | + # head comment + key: value # line comment + # foot comment + token: | + - {token: COMMENT, head: '# head comment'} + - {token: STREAM-START} + - {token: BLOCK-MAPPING-START} + - {token: KEY} + - {token: SCALAR, value: key} + - {token: VALUE} + - {token: COMMENT, line: '# line comment'} + - {token: COMMENT, foot: '# foot comment'} + - {token: SCALAR, value: value} + - {token: BLOCK-END} + - {token: STREAM-END} + event: | + - {event: DOCUMENT-START} + - {event: MAPPING-START} + - {event: SCALAR, value: key, head: '# head comment', foot: '# foot comment'} + - {event: SCALAR, value: value, line: '# line comment'} + - {event: MAPPING-END} + - {event: DOCUMENT-END} + +- name: Multiple head comments + text: | + # comment1 + # comment2 + key: value + token: | + - {token: COMMENT, head: "# comment1\n# comment2"} + - {token: STREAM-START} + - {token: BLOCK-MAPPING-START} + - {token: KEY} + - {token: SCALAR, value: key} + - {token: VALUE} + - {token: SCALAR, value: value} + - {token: BLOCK-END} + - {token: STREAM-END} diff --git a/cmd/go-yaml/testdata/document-implicit.yaml b/cmd/go-yaml/testdata/document-implicit.yaml new file mode 100644 index 00000000..5074b780 --- /dev/null +++ b/cmd/go-yaml/testdata/document-implicit.yaml @@ -0,0 +1,254 @@ +# Tests for implicit/explicit document markers in -e and -E modes + +- name: Single implicit document + text: | + key: value + event: | + - {event: DOCUMENT-START} + - {event: MAPPING-START} + - {event: SCALAR, value: key} + - {event: SCALAR, value: value} + - {event: MAPPING-END} + - {event: DOCUMENT-END} + EVENT: | + - {event: DOCUMENT-START, implicit: true, pos: 1/0} + - {event: MAPPING-START, style: Plain, pos: 1/0} + - {event: SCALAR, value: key, style: Plain, pos: 1/0-3} + - {event: SCALAR, value: value, style: Plain, pos: 1/5-10} + - {event: MAPPING-END, pos: 1/0} + - {event: DOCUMENT-END, implicit: true, pos: 1/0} + +- name: Single explicit document + text: | + --- + key: value + event: | + - {event: DOCUMENT-START, explicit: true} + - {event: MAPPING-START} + - {event: SCALAR, value: key} + - {event: SCALAR, value: value} + - {event: MAPPING-END} + - {event: DOCUMENT-END} + EVENT: | + - {event: DOCUMENT-START, pos: 1/0} + - {event: MAPPING-START, style: Plain, pos: 2/0} + - {event: SCALAR, value: key, style: Plain, pos: 2/0-3} + - {event: SCALAR, value: value, style: Plain, pos: 2/5-10} + - {event: MAPPING-END, pos: 2/0} + - {event: DOCUMENT-END, implicit: true, pos: 1/0} + +- name: Implicit then explicit document + text: | + first + --- + second + event: | + - {event: DOCUMENT-START} + - {event: SCALAR, value: first} + - {event: DOCUMENT-END} + - {event: DOCUMENT-START, explicit: true} + - {event: SCALAR, value: second} + - {event: DOCUMENT-END} + EVENT: | + - {event: DOCUMENT-START, implicit: true, pos: 1/0} + - {event: SCALAR, value: first, style: Plain, pos: 1/0-5} + - {event: DOCUMENT-END, implicit: true, pos: 1/0} + - {event: DOCUMENT-START, pos: 2/0} + - {event: SCALAR, value: second, style: Plain, pos: 3/0-6} + - {event: DOCUMENT-END, implicit: true, pos: 2/0} + +- name: Two explicit documents + text: | + --- + first + --- + second + event: | + - {event: DOCUMENT-START, explicit: true} + - {event: SCALAR, value: first} + - {event: DOCUMENT-END} + - {event: DOCUMENT-START, explicit: true} + - {event: SCALAR, value: second} + - {event: DOCUMENT-END} + EVENT: | + - {event: DOCUMENT-START, pos: 1/0} + - {event: SCALAR, value: first, style: Plain, pos: 2/0-5} + - {event: DOCUMENT-END, implicit: true, pos: 1/0} + - {event: DOCUMENT-START, pos: 3/0} + - {event: SCALAR, value: second, style: Plain, pos: 4/0-6} + - {event: DOCUMENT-END, implicit: true, pos: 3/0} + +- name: Document with explicit end marker + text: | + --- + key: value + ... + event: | + - {event: DOCUMENT-START, explicit: true} + - {event: MAPPING-START} + - {event: SCALAR, value: key} + - {event: SCALAR, value: value} + - {event: MAPPING-END} + - {event: DOCUMENT-END, explicit: true} + EVENT: | + - {event: DOCUMENT-START, pos: 1/0} + - {event: MAPPING-START, style: Plain, pos: 2/0} + - {event: SCALAR, value: key, style: Plain, pos: 2/0-3} + - {event: SCALAR, value: value, style: Plain, pos: 2/5-10} + - {event: MAPPING-END, pos: 2/0} + - {event: DOCUMENT-END, pos: 1/0} + +- name: Implicit doc with explicit end then explicit doc + text: | + first + ... + --- + second + event: | + - {event: DOCUMENT-START} + - {event: SCALAR, value: first} + - {event: DOCUMENT-END, explicit: true} + - {event: DOCUMENT-START, explicit: true} + - {event: SCALAR, value: second} + - {event: DOCUMENT-END} + EVENT: | + - {event: DOCUMENT-START, implicit: true, pos: 1/0} + - {event: SCALAR, value: first, style: Plain, pos: 1/0-5} + - {event: DOCUMENT-END, pos: 1/0} + - {event: DOCUMENT-START, pos: 3/0} + - {event: SCALAR, value: second, style: Plain, pos: 4/0-6} + - {event: DOCUMENT-END, implicit: true, pos: 3/0} + +- name: Three documents with mixed markers + text: | + first + --- + second + --- + third + event: | + - {event: DOCUMENT-START} + - {event: SCALAR, value: first} + - {event: DOCUMENT-END} + - {event: DOCUMENT-START, explicit: true} + - {event: SCALAR, value: second} + - {event: DOCUMENT-END} + - {event: DOCUMENT-START, explicit: true} + - {event: SCALAR, value: third} + - {event: DOCUMENT-END} + EVENT: | + - {event: DOCUMENT-START, implicit: true, pos: 1/0} + - {event: SCALAR, value: first, style: Plain, pos: 1/0-5} + - {event: DOCUMENT-END, implicit: true, pos: 1/0} + - {event: DOCUMENT-START, pos: 2/0} + - {event: SCALAR, value: second, style: Plain, pos: 3/0-6} + - {event: DOCUMENT-END, implicit: true, pos: 2/0} + - {event: DOCUMENT-START, pos: 4/0} + - {event: SCALAR, value: third, style: Plain, pos: 5/0-5} + - {event: DOCUMENT-END, implicit: true, pos: 4/0} + +- name: Explicit document with mapping + text: | + --- + doc1: value1 + doc2: value2 + ... + event: | + - {event: DOCUMENT-START, explicit: true} + - {event: MAPPING-START} + - {event: SCALAR, value: doc1} + - {event: SCALAR, value: value1} + - {event: SCALAR, value: doc2} + - {event: SCALAR, value: value2} + - {event: MAPPING-END} + - {event: DOCUMENT-END, explicit: true} + EVENT: | + - {event: DOCUMENT-START, pos: 1/0} + - {event: MAPPING-START, style: Plain, pos: 2/0} + - {event: SCALAR, value: doc1, style: Plain, pos: 2/0-4} + - {event: SCALAR, value: value1, style: Plain, pos: 2/6-12} + - {event: SCALAR, value: doc2, style: Plain, pos: 3/0-4} + - {event: SCALAR, value: value2, style: Plain, pos: 3/6-12} + - {event: MAPPING-END, pos: 2/0} + - {event: DOCUMENT-END, pos: 1/0} + +- name: Empty explicit documents + text: | + --- + --- + event: | + - {event: DOCUMENT-START, explicit: true} + - {event: SCALAR, tag: '!!null'} + - {event: DOCUMENT-END} + - {event: DOCUMENT-START, explicit: true} + - {event: SCALAR, tag: '!!null'} + - {event: DOCUMENT-END} + EVENT: | + - {event: DOCUMENT-START, pos: 1/0} + - {event: SCALAR, style: Plain, tag: '!!null', pos: 2/0} + - {event: DOCUMENT-END, implicit: true, pos: 1/0} + - {event: DOCUMENT-START, pos: 2/0} + - {event: SCALAR, style: Plain, tag: '!!null', pos: 3/0} + - {event: DOCUMENT-END, implicit: true, pos: 2/0} + +- name: Implicit sequence document + text: | + - item1 + - item2 + event: | + - {event: DOCUMENT-START} + - {event: SEQUENCE-START} + - {event: SCALAR, value: item1} + - {event: SCALAR, value: item2} + - {event: SEQUENCE-END} + - {event: DOCUMENT-END} + EVENT: | + - {event: DOCUMENT-START, implicit: true, pos: 1/0} + - {event: SEQUENCE-START, style: Plain, pos: 1/0} + - {event: SCALAR, value: item1, style: Plain, pos: 1/2-7} + - {event: SCALAR, value: item2, style: Plain, pos: 2/2-7} + - {event: SEQUENCE-END, pos: 1/0} + - {event: DOCUMENT-END, implicit: true, pos: 1/0} + +- name: Explicit sequence document + text: | + --- + - item1 + - item2 + event: | + - {event: DOCUMENT-START, explicit: true} + - {event: SEQUENCE-START} + - {event: SCALAR, value: item1} + - {event: SCALAR, value: item2} + - {event: SEQUENCE-END} + - {event: DOCUMENT-END} + EVENT: | + - {event: DOCUMENT-START, pos: 1/0} + - {event: SEQUENCE-START, style: Plain, pos: 2/0} + - {event: SCALAR, value: item1, style: Plain, pos: 2/2-7} + - {event: SCALAR, value: item2, style: Plain, pos: 3/2-7} + - {event: SEQUENCE-END, pos: 2/0} + - {event: DOCUMENT-END, implicit: true, pos: 1/0} + +- name: All explicit markers + text: | + --- + first + ... + --- + second + ... + event: | + - {event: DOCUMENT-START, explicit: true} + - {event: SCALAR, value: first} + - {event: DOCUMENT-END, explicit: true} + - {event: DOCUMENT-START, explicit: true} + - {event: SCALAR, value: second} + - {event: DOCUMENT-END, explicit: true} + EVENT: | + - {event: DOCUMENT-START, pos: 1/0} + - {event: SCALAR, value: first, style: Plain, pos: 2/0-5} + - {event: DOCUMENT-END, pos: 1/0} + - {event: DOCUMENT-START, pos: 4/0} + - {event: SCALAR, value: second, style: Plain, pos: 5/0-6} + - {event: DOCUMENT-END, pos: 4/0} diff --git a/cmd/go-yaml/testdata/key-ordering.yaml b/cmd/go-yaml/testdata/key-ordering.yaml new file mode 100644 index 00000000..2a167d3a --- /dev/null +++ b/cmd/go-yaml/testdata/key-ordering.yaml @@ -0,0 +1,245 @@ +# Key ordering tests for -n and -N flags +# These tests verify that keys appear in the correct order: +# For -n: tag, anchor, head, line, foot, then content (mapping/sequence/scalar) +# For -N: kind, style, tag, anchor, head, line, foot, then text/content + +- name: Scalar with tag and anchor + text: | + &myanchor !!str value + node: | + anchor: myanchor + plain: value + NODE: | + kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!str' + anchor: myanchor + text: value + +- name: Scalar with anchor and comments + text: | + # head comment + &myanchor value # line comment + # foot comment + node: | + anchor: myanchor + head: '# head comment' + line: '# line comment' + foot: '# foot comment' + plain: value + NODE: | + kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!str' + anchor: myanchor + head: '# head comment' + line: '# line comment' + foot: '# foot comment' + text: value + +- name: Mapping with tag, anchor, and comments + text: | + # head comment + &myanchor !!map + key: value # line comment + # foot comment + node: | + tag: '!!map' + anchor: myanchor + mapping: + - head: '# head comment' + foot: '# foot comment' + plain: key + - line: '# line comment' + plain: value + NODE: | + kind: Document + content: + - kind: Mapping + style: Plain + tag: '!!map' + anchor: myanchor + content: + - kind: Scalar + style: Plain + tag: '!!str' + head: '# head comment' + foot: '# foot comment' + text: key + - kind: Scalar + style: Plain + tag: '!!str' + line: '# line comment' + text: value + +- name: Sequence with tag and anchor + text: | + &myseq !!seq + - item1 + - item2 + node: | + tag: '!!seq' + anchor: myseq + sequence: + - plain: item1 + - plain: item2 + NODE: | + kind: Document + content: + - kind: Sequence + style: Plain + tag: '!!seq' + anchor: myseq + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: item1 + - kind: Scalar + style: Plain + tag: '!!str' + text: item2 + +- name: Sequence with all metadata fields + text: | + # head comment + &myseq !!seq + - item1 + - item2 + # foot comment + node: | + tag: '!!seq' + anchor: myseq + sequence: + - head: '# head comment' + plain: item1 + - foot: '# foot comment' + plain: item2 + NODE: | + kind: Document + content: + - kind: Sequence + style: Plain + tag: '!!seq' + anchor: myseq + content: + - kind: Scalar + style: Plain + tag: '!!str' + head: '# head comment' + text: item1 + - kind: Scalar + style: Plain + tag: '!!str' + foot: '# foot comment' + text: item2 + +- name: Tag comes before anchor + text: | + &anchor !!str tagged + node: | + anchor: anchor + plain: tagged + NODE: | + kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!str' + anchor: anchor + text: tagged + +- name: Comments come before content + text: | + # head + key: value # line + node: | + mapping: + - head: '# head' + plain: key + - line: '# line' + plain: value + NODE: | + kind: Document + content: + - kind: Mapping + style: Plain + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + head: '# head' + text: key + - kind: Scalar + style: Plain + tag: '!!str' + line: '# line' + text: value + +- name: Double quoted scalar with tag and anchor + text: | + &myanchor !!str "quoted" + node: | + anchor: myanchor + double: quoted + NODE: | + kind: Document + content: + - kind: Scalar + style: Double + tag: '!!str' + anchor: myanchor + text: quoted + +- name: Nested mapping with ordered keys + text: | + &outer !!map + # outer head + nested: + &inner !!map + # inner head + key: value + node: | + tag: '!!map' + anchor: outer + mapping: + - head: '# outer head' + plain: nested + - tag: '!!map' + anchor: inner + mapping: + - head: '# inner head' + plain: key + - plain: value + NODE: | + kind: Document + content: + - kind: Mapping + style: Plain + tag: '!!map' + anchor: outer + content: + - kind: Scalar + style: Plain + tag: '!!str' + head: '# outer head' + text: nested + - kind: Mapping + style: Plain + tag: '!!map' + anchor: inner + content: + - kind: Scalar + style: Plain + tag: '!!str' + head: '# inner head' + text: key + - kind: Scalar + style: Plain + tag: '!!str' + text: value diff --git a/cmd/go-yaml/testdata/multi-document.yaml b/cmd/go-yaml/testdata/multi-document.yaml new file mode 100644 index 00000000..252563f6 --- /dev/null +++ b/cmd/go-yaml/testdata/multi-document.yaml @@ -0,0 +1,579 @@ +# Multi-document tests for -n and -N flags +# Multiple documents should be output as a YAML sequence + +- name: Two simple mappings + text: | + doc1: value1 + --- + doc2: value2 + node: | + - mapping: + - plain: doc1 + - plain: value1 + - mapping: + - plain: doc2 + - plain: value2 + NODE: | + - kind: Document + content: + - kind: Mapping + style: Plain + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: doc1 + - kind: Scalar + style: Plain + tag: '!!str' + text: value1 + - kind: Document + content: + - kind: Mapping + style: Plain + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: doc2 + - kind: Scalar + style: Plain + tag: '!!str' + text: value2 + +- name: Three documents with mixed types + text: | + scalar1 + --- + - item1 + - item2 + --- + key: value + node: | + - plain: scalar1 + - sequence: + - plain: item1 + - plain: item2 + - mapping: + - plain: key + - plain: value + NODE: | + - kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: scalar1 + - kind: Document + content: + - kind: Sequence + style: Plain + tag: '!!seq' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: item1 + - kind: Scalar + style: Plain + tag: '!!str' + text: item2 + - kind: Document + content: + - kind: Mapping + style: Plain + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: key + - kind: Scalar + style: Plain + tag: '!!str' + text: value + +- name: Documents with anchors and tags + text: | + &anchor1 !!str first + --- + &anchor2 !!map + key: value + node: | + - anchor: anchor1 + plain: first + - tag: '!!map' + anchor: anchor2 + mapping: + - plain: key + - plain: value + NODE: | + - kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!str' + anchor: anchor1 + text: first + - kind: Document + content: + - kind: Mapping + style: Plain + tag: '!!map' + anchor: anchor2 + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: key + - kind: Scalar + style: Plain + tag: '!!str' + text: value + +- name: Documents with comments + text: | + # comment 1 + doc1: value1 + --- + # comment 2 + doc2: value2 + node: | + - mapping: + - head: '# comment 1' + plain: doc1 + - plain: value1 + - mapping: + - head: '# comment 2' + plain: doc2 + - plain: value2 + NODE: | + - kind: Document + content: + - kind: Mapping + style: Plain + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + head: '# comment 1' + text: doc1 + - kind: Scalar + style: Plain + tag: '!!str' + text: value1 + - kind: Document + content: + - kind: Mapping + style: Plain + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + head: '# comment 2' + text: doc2 + - kind: Scalar + style: Plain + tag: '!!str' + text: value2 + +- name: Five documents + text: | + first + --- + second + --- + third + --- + fourth + --- + fifth + node: | + - plain: first + - plain: second + - plain: third + - plain: fourth + - plain: fifth + NODE: | + - kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: first + - kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: second + - kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: third + - kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: fourth + - kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: fifth + +- name: Multiple sequences + text: | + - a + - b + --- + - c + - d + --- + - e + - f + node: | + - sequence: + - plain: a + - plain: b + - sequence: + - plain: c + - plain: d + - sequence: + - plain: e + - plain: f + NODE: | + - kind: Document + content: + - kind: Sequence + style: Plain + tag: '!!seq' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: a + - kind: Scalar + style: Plain + tag: '!!str' + text: b + - kind: Document + content: + - kind: Sequence + style: Plain + tag: '!!seq' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: c + - kind: Scalar + style: Plain + tag: '!!str' + text: d + - kind: Document + content: + - kind: Sequence + style: Plain + tag: '!!seq' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: e + - kind: Scalar + style: Plain + tag: '!!str' + text: f + +- name: Documents with different scalar styles + text: | + plain + --- + "double" + --- + 'single' + node: | + - plain: plain + - double: double + - single: single + NODE: | + - kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: plain + - kind: Document + content: + - kind: Scalar + style: Double + tag: '!!str' + text: double + - kind: Document + content: + - kind: Scalar + style: Single + tag: '!!str' + text: single + +- name: Nested structures across documents + text: | + outer: + inner: + deep: value1 + --- + list: + - nested: + item: value2 + node: | + - mapping: + - plain: outer + - mapping: + - plain: inner + - mapping: + - plain: deep + - plain: value1 + - mapping: + - plain: list + - sequence: + - mapping: + - plain: nested + - mapping: + - plain: item + - plain: value2 + NODE: | + - kind: Document + content: + - kind: Mapping + style: Plain + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: outer + - kind: Mapping + style: Plain + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: inner + - kind: Mapping + style: Plain + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: deep + - kind: Scalar + style: Plain + tag: '!!str' + text: value1 + - kind: Document + content: + - kind: Mapping + style: Plain + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: list + - kind: Sequence + style: Plain + tag: '!!seq' + content: + - kind: Mapping + style: Plain + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: nested + - kind: Mapping + style: Plain + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: item + - kind: Scalar + style: Plain + tag: '!!str' + text: value2 + +- name: Documents with full metadata + text: | + # head1 + &anchor1 !!str value1 # line1 + # foot1 + --- + # head2 + &anchor2 !!seq + - item # line2 + # foot2 + --- + # head3 + &anchor3 !!map + key: val # line3 + # foot3 + node: | + - foot: '# foot1' + anchor: anchor1 + head: '# head1' + line: '# line1' + plain: value1 + - foot: '# foot2' + tag: '!!seq' + anchor: anchor2 + sequence: + - head: '# head2' + line: '# line2' + plain: item + - tag: '!!map' + anchor: anchor3 + mapping: + - head: '# head3' + foot: '# foot3' + plain: key + - line: '# line3' + plain: val + NODE: | + - kind: Document + foot: '# foot1' + content: + - kind: Scalar + style: Plain + tag: '!!str' + anchor: anchor1 + head: '# head1' + line: '# line1' + text: value1 + - kind: Document + foot: '# foot2' + content: + - kind: Sequence + style: Plain + tag: '!!seq' + anchor: anchor2 + content: + - kind: Scalar + style: Plain + tag: '!!str' + head: '# head2' + line: '# line2' + text: item + - kind: Document + content: + - kind: Mapping + style: Plain + tag: '!!map' + anchor: anchor3 + content: + - kind: Scalar + style: Plain + tag: '!!str' + head: '# head3' + foot: '# foot3' + text: key + - kind: Scalar + style: Plain + tag: '!!str' + line: '# line3' + text: val + +- name: Documents with integers and special values + text: | + 42 + --- + 3.14 + --- + true + --- + null + node: | + - plain: "42" + - plain: "3.14" + - plain: "true" + - plain: "null" + NODE: | + - kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!int' + text: "42" + - kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!float' + text: "3.14" + - kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!bool' + text: "true" + - kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!null' + text: "null" + +- name: Documents with flow style + text: | + {key1: value1} + --- + [item1, item2] + node: | + - mapping: + - plain: key1 + - plain: value1 + - sequence: + - plain: item1 + - plain: item2 + NODE: | + - kind: Document + content: + - kind: Mapping + style: Flow + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: key1 + - kind: Scalar + style: Plain + tag: '!!str' + text: value1 + - kind: Document + content: + - kind: Sequence + style: Flow + tag: '!!seq' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: item1 + - kind: Scalar + style: Plain + tag: '!!str' + text: item2 diff --git a/cmd/go-yaml/testdata/node-profuse.yaml b/cmd/go-yaml/testdata/node-profuse.yaml new file mode 100644 index 00000000..1ff40a5d --- /dev/null +++ b/cmd/go-yaml/testdata/node-profuse.yaml @@ -0,0 +1,76 @@ +# Node profuse mode tests - comparing -n and -N flags + +- name: Plain scalar + text: | + hello + node: | + plain: hello + NODE: | + kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: hello + +- name: Double quoted scalar + text: | + "hello" + node: | + double: hello + NODE: | + kind: Document + content: + - kind: Scalar + style: Double + tag: '!!str' + text: hello + +- name: Integer scalar + text: | + 42 + node: | + plain: "42" + NODE: | + kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!int' + text: "42" + +- name: Explicit tag + text: | + !!str 42 + node: | + plain: "42" + NODE: | + kind: Document + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: "42" + +- name: Mapping with plain scalars + text: | + key: value + node: | + mapping: + - plain: key + - plain: value + NODE: | + kind: Document + content: + - kind: Mapping + style: Plain + tag: '!!map' + content: + - kind: Scalar + style: Plain + tag: '!!str' + text: key + - kind: Scalar + style: Plain + tag: '!!str' + text: value diff --git a/cmd/go-yaml/testdata/positions.yaml b/cmd/go-yaml/testdata/positions.yaml new file mode 100644 index 00000000..6e2b0134 --- /dev/null +++ b/cmd/go-yaml/testdata/positions.yaml @@ -0,0 +1,54 @@ +# Position format tests for go-yaml CLI + +- name: Single character position + text: | + a + TOKEN: | + - {token: STREAM-START, pos: 1/0} + - {token: SCALAR, value: a, pos: 1/0-1} + - {token: STREAM-END, pos: 2/0} + +- name: | + Same line range + text: | + key: value + TOKEN: | + - {token: STREAM-START, pos: 1/0} + - {token: BLOCK-MAPPING-START, pos: 1/0} + - {token: KEY, pos: 1/0} + - {token: SCALAR, value: key, pos: 1/0-3} + - {token: VALUE, pos: 1/3-4} + - {token: SCALAR, value: value, pos: 1/5-10} + - {token: BLOCK-END, pos: 2/0} + - {token: STREAM-END, pos: 2/0} + +- name: | + Multi-line literal scalar + text: | + key: | + line1 + line2 + TOKEN: | + - {token: STREAM-START, pos: 1/0} + - {token: BLOCK-MAPPING-START, pos: 1/0} + - {token: KEY, pos: 1/0} + - {token: SCALAR, value: key, pos: 1/0-3} + - {token: VALUE, pos: 1/3-4} + - {token: SCALAR, value: "line1\nline2\n", style: Literal, pos: 1/5-4/0} + - {token: BLOCK-END, pos: 4/0} + - {token: STREAM-END, pos: 4/0} + +- name: | + Comment position with line comment + text: | + a: b #c + TOKEN: | + - {token: STREAM-START, pos: 1/0} + - {token: BLOCK-MAPPING-START, pos: 1/0} + - {token: KEY, pos: 1/0} + - {token: SCALAR, value: a, pos: 1/0-1} + - {token: VALUE, pos: 1/1-2} + - {token: COMMENT, line: '#c', pos: 1/5-7} + - {token: SCALAR, value: b, pos: 1/3-4} + - {token: BLOCK-END, pos: 2/0} + - {token: STREAM-END, pos: 2/0} diff --git a/cmd/go-yaml/token.go b/cmd/go-yaml/token.go index fae3dd68..2059b380 100644 --- a/cmd/go-yaml/token.go +++ b/cmd/go-yaml/token.go @@ -17,6 +17,7 @@ type Token struct { Type string Value string Style string + CommentType string // For COMMENT tokens: "head", "line", or "foot" StartLine int StartColumn int EndLine int @@ -28,13 +29,13 @@ type Token struct { // TokenInfo represents the information about a YAML token for YAML encoding type TokenInfo struct { - Token string `yaml:"Token"` - Value string `yaml:"Value,omitempty"` - Style string `yaml:"Style,omitempty"` - Head string `yaml:"Head,omitempty"` - Line string `yaml:"Line,omitempty"` - Foot string `yaml:"Foot,omitempty"` - Pos string `yaml:"Pos,omitempty"` + Token string `yaml:"token"` + Value string `yaml:"value,omitempty"` + Style string `yaml:"style,omitempty"` + Head string `yaml:"head,omitempty"` + Line string `yaml:"line,omitempty"` + Foot string `yaml:"foot,omitempty"` + Pos string `yaml:"pos,omitempty"` } // ProcessTokens reads YAML from stdin and outputs token information using the internal scanner @@ -81,38 +82,38 @@ func processTokensDecode(profuse, compact bool) error { // Add the Token field compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Token"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "token"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Token}) // Add other fields if they exist if info.Value != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Value"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "value"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Value}) } if info.Style != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Style"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "style"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Style}) } if info.Head != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Head"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "head"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Head}) } if info.Line != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Line"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "line"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Line}) } if info.Foot != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Foot"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "foot"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Foot}) } if info.Pos != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Pos"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "pos"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Pos}) } @@ -175,38 +176,38 @@ func processTokensWithParser(profuse, compact bool) error { // Add the Token field compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Token"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "token"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Token}) // Add other fields if they exist if info.Value != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Value"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "value"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Value}) } if info.Style != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Style"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "style"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Style}) } if info.Head != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Head"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "head"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Head}) } if info.Line != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Line"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "line"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Line}) } if info.Foot != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Foot"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "foot"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Foot}) } if info.Pos != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Pos"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "pos"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Pos}) } @@ -287,38 +288,38 @@ func processTokensUnmarshal(profuse, compact bool) error { // Add the Token field compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Token"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "token"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Token}) // Add other fields if they exist if info.Value != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Value"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "value"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Value}) } if info.Style != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Style"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "style"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Style}) } if info.Head != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Head"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "head"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Head}) } if info.Line != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Line"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "line"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Line}) } if info.Foot != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Foot"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "foot"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Foot}) } if info.Pos != "" { compactNode.Content = append(compactNode.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "Pos"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: "pos"}, &yaml.Node{Kind: yaml.ScalarNode, Value: info.Pos}) } @@ -359,26 +360,47 @@ func formatTokenInfo(token *Token, profuse bool) *TokenInfo { Token: token.Type, } - if token.Value != "" { - info.Value = token.Value - } - if token.Style != "" && token.Style != "Plain" { - info.Style = token.Style - } - if token.HeadComment != "" { - info.Head = token.HeadComment - } - if token.LineComment != "" { - info.Line = token.LineComment - } - if token.FootComment != "" { - info.Foot = token.FootComment + // For COMMENT tokens, use the CommentType to determine which field to populate + if token.Type == "COMMENT" && token.CommentType != "" && token.Value != "" { + switch token.CommentType { + case "head": + info.Head = token.Value + case "line": + info.Line = token.Value + case "foot": + info.Foot = token.Value + } + } else { + // For non-COMMENT tokens + if token.Value != "" { + info.Value = token.Value + } + + if token.Style != "" && token.Style != "Plain" { + info.Style = token.Style + } + + if token.HeadComment != "" { + info.Head = token.HeadComment + } + if token.LineComment != "" { + info.Line = token.LineComment + } + if token.FootComment != "" { + info.Foot = token.FootComment + } } + if profuse { if token.StartLine == token.EndLine && token.StartColumn == token.EndColumn { - info.Pos = fmt.Sprintf("%d;%d", token.StartLine, token.StartColumn) + // Single position + info.Pos = fmt.Sprintf("%d/%d", token.StartLine, token.StartColumn) + } else if token.StartLine == token.EndLine { + // Range on same line + info.Pos = fmt.Sprintf("%d/%d-%d", token.StartLine, token.StartColumn, token.EndColumn) } else { - info.Pos = fmt.Sprintf("%d;%d-%d;%d", token.StartLine, token.StartColumn, token.EndLine, token.EndColumn) + // Range across different lines + info.Pos = fmt.Sprintf("%d/%d-%d/%d", token.StartLine, token.StartColumn, token.EndLine, token.EndColumn) } } @@ -546,7 +568,7 @@ func processNodeToTokensRecursive(node *yaml.Node, profuse bool) []*Token { StartColumn: node.Column, EndLine: endLine, EndColumn: endColumn, - Style: formatStyle(node.Style), + Style: formatStyle(node.Style, false), HeadComment: node.HeadComment, LineComment: node.LineComment, FootComment: node.FootComment, diff --git a/internal/libyaml/api.go b/internal/libyaml/api.go index dbd94647..ab0c534b 100644 --- a/internal/libyaml/api.go +++ b/internal/libyaml/api.go @@ -100,6 +100,16 @@ func (parser *Parser) SetEncoding(encoding Encoding) { parser.encoding = encoding } +// GetPendingComments returns the parser's comment queue for CLI access. +func (parser *Parser) GetPendingComments() []Comment { + return parser.comments +} + +// GetCommentsHead returns the current position in the comment queue. +func (parser *Parser) GetCommentsHead() int { + return parser.comments_head +} + // NewEmitter creates a new emitter object. func NewEmitter() Emitter { return Emitter{ diff --git a/internal/libyaml/parser.go b/internal/libyaml/parser.go index 765ce333..8295d4b4 100644 --- a/internal/libyaml/parser.go +++ b/internal/libyaml/parser.go @@ -78,29 +78,29 @@ func (parser *Parser) peekToken() *Token { // comments behind the position of the provided token into the respective // top-level comment slices in the parser. func (parser *Parser) UnfoldComments(token *Token) { - for parser.comments_head < len(parser.comments) && token.StartMark.Index >= parser.comments[parser.comments_head].token_mark.Index { + for parser.comments_head < len(parser.comments) && token.StartMark.Index >= parser.comments[parser.comments_head].TokenMark.Index { comment := &parser.comments[parser.comments_head] - if len(comment.head) > 0 { + if len(comment.Head) > 0 { if token.Type == BLOCK_END_TOKEN { - // No heads on ends, so keep comment.head for a follow up token. + // No heads on ends, so keep comment.Head for a follow up token. break } if len(parser.HeadComment) > 0 { parser.HeadComment = append(parser.HeadComment, '\n') } - parser.HeadComment = append(parser.HeadComment, comment.head...) + parser.HeadComment = append(parser.HeadComment, comment.Head...) } - if len(comment.foot) > 0 { + if len(comment.Foot) > 0 { if len(parser.FootComment) > 0 { parser.FootComment = append(parser.FootComment, '\n') } - parser.FootComment = append(parser.FootComment, comment.foot...) + parser.FootComment = append(parser.FootComment, comment.Foot...) } - if len(comment.line) > 0 { + if len(comment.Line) > 0 { if len(parser.LineComment) > 0 { parser.LineComment = append(parser.LineComment, '\n') } - parser.LineComment = append(parser.LineComment, comment.line...) + parser.LineComment = append(parser.LineComment, comment.Line...) } *comment = Comment{} parser.comments_head++ diff --git a/internal/libyaml/scanner.go b/internal/libyaml/scanner.go index fbb3b5c8..b5b1e1c9 100644 --- a/internal/libyaml/scanner.go +++ b/internal/libyaml/scanner.go @@ -1045,22 +1045,22 @@ func (parser *Parser) unrollIndent(column int, scan_mark Mark) bool { for i := len(parser.comments) - 1; i >= 0; i-- { comment := &parser.comments[i] - if comment.end_mark.Index < stop_index { + if comment.EndMark.Index < stop_index { // Don't go back beyond the start of the comment/whitespace scan, unless column < 0. // If requested indent column is < 0, then the document is over and everything else // is a foot anyway. break } - if comment.start_mark.Column == parser.indent+1 { + if comment.StartMark.Column == parser.indent+1 { // This is a good match. But maybe there's a former comment // at that same indent level, so keep searching. - block_mark = comment.start_mark + block_mark = comment.StartMark } // While the end of the former comment matches with // the start of the following one, we know there's // nothing in between and scanning is still safe. - stop_index = comment.scan_mark.Index + stop_index = comment.ScanMark.Index } // Create a token and append it to the queue. @@ -1566,14 +1566,14 @@ func (parser *Parser) scanToNextToken() bool { tokenA := parser.tokens[len(parser.tokens)-2] tokenB := parser.tokens[len(parser.tokens)-1] comment := &parser.comments[len(parser.comments)-1] - if tokenA.Type == BLOCK_SEQUENCE_START_TOKEN && tokenB.Type == BLOCK_ENTRY_TOKEN && len(comment.line) > 0 && !isLineBreak(parser.buffer, parser.buffer_pos) { + if tokenA.Type == BLOCK_SEQUENCE_START_TOKEN && tokenB.Type == BLOCK_ENTRY_TOKEN && len(comment.Line) > 0 && !isLineBreak(parser.buffer, parser.buffer_pos) { // If it was in the prior line, reposition so it becomes a // header of the follow up token. Otherwise, keep it in place // so it becomes a header of the former. - comment.head = comment.line - comment.line = nil - if comment.start_mark.Line == parser.mark.Line-1 { - comment.token_mark = parser.mark + comment.Head = comment.Line + comment.Line = nil + if comment.StartMark.Line == parser.mark.Line-1 { + comment.TokenMark = parser.mark } } } @@ -2866,9 +2866,11 @@ func (parser *Parser) scanLineComment(token_mark Mark) bool { } if len(text) > 0 { parser.comments = append(parser.comments, Comment{ - token_mark: token_mark, - start_mark: start_mark, - line: text, + ScanMark: token_mark, + TokenMark: token_mark, + StartMark: start_mark, + EndMark: parser.mark, + Line: text, }) } return true @@ -2934,11 +2936,11 @@ func (parser *Parser) scanComments(scan_mark Mark) bool { token_mark = start_mark } parser.comments = append(parser.comments, Comment{ - scan_mark: scan_mark, - token_mark: token_mark, - start_mark: start_mark, - end_mark: Mark{parser.mark.Index + peek, line, column}, - foot: text, + ScanMark: scan_mark, + TokenMark: token_mark, + StartMark: start_mark, + EndMark: Mark{parser.mark.Index + peek, line, column}, + Foot: text, }) scan_mark = Mark{parser.mark.Index + peek, line, column} token_mark = scan_mark @@ -2964,11 +2966,11 @@ func (parser *Parser) scanComments(scan_mark Mark) bool { // The comment at the different indentation is a foot of the // preceding data rather than a head of the upcoming one. parser.comments = append(parser.comments, Comment{ - scan_mark: scan_mark, - token_mark: token_mark, - start_mark: start_mark, - end_mark: Mark{parser.mark.Index + peek, line, column}, - foot: text, + ScanMark: scan_mark, + TokenMark: token_mark, + StartMark: start_mark, + EndMark: Mark{parser.mark.Index + peek, line, column}, + Foot: text, }) scan_mark = Mark{parser.mark.Index + peek, line, column} token_mark = scan_mark @@ -3019,11 +3021,11 @@ func (parser *Parser) scanComments(scan_mark Mark) bool { if len(text) > 0 { parser.comments = append(parser.comments, Comment{ - scan_mark: scan_mark, - token_mark: start_mark, - start_mark: start_mark, - end_mark: Mark{parser.mark.Index + peek - 1, line, column}, - head: text, + ScanMark: scan_mark, + TokenMark: start_mark, + StartMark: start_mark, + EndMark: Mark{parser.mark.Index + peek - 1, line, column}, + Head: text, }) } return true diff --git a/internal/libyaml/yaml.go b/internal/libyaml/yaml.go index c745b3c5..1c92f690 100644 --- a/internal/libyaml/yaml.go +++ b/internal/libyaml/yaml.go @@ -175,10 +175,11 @@ const ( KEY_TOKEN // A KEY token. VALUE_TOKEN // A VALUE token. - ALIAS_TOKEN // An ALIAS token. - ANCHOR_TOKEN // An ANCHOR token. - TAG_TOKEN // A TAG token. - SCALAR_TOKEN // A SCALAR token. + ALIAS_TOKEN // An ALIAS token. + ANCHOR_TOKEN // An ANCHOR token. + TAG_TOKEN // A TAG token. + SCALAR_TOKEN // A SCALAR token. + COMMENT_TOKEN // A COMMENT token. ) func (tt TokenType) String() string { @@ -227,6 +228,8 @@ func (tt TokenType) String() string { return "TAG_TOKEN" case SCALAR_TOKEN: return "SCALAR_TOKEN" + case COMMENT_TOKEN: + return "COMMENT_TOKEN" } return "" } @@ -645,14 +648,14 @@ type Parser struct { } type Comment struct { - scan_mark Mark // Position where scanning for comments started - token_mark Mark // Position after which tokens will be associated with this comment - start_mark Mark // Position of '#' comment mark - end_mark Mark // Position where comment terminated - - head []byte - line []byte - foot []byte + ScanMark Mark // Position where scanning for comments started + TokenMark Mark // Position after which tokens will be associated with this comment + StartMark Mark // Position of '#' comment mark + EndMark Mark // Position where comment terminated + + Head []byte + Line []byte + Foot []byte } // Emitter Definitions