Prototype validate taking directories as input (#546)

Signed-off-by: Juan Cruz Viotti <[email protected]>

Author: jviotti

docs/validate.markdown

@@ -5,7 +5,7 @@ Validating
 > JSON Schema Draft 3 and older are not supported at this point in time.
 
 ```sh
-jsonschema validate <schema.json|.yaml> <instance.json|.jsonl|.yaml...>
+jsonschema validate <schema.json|.yaml> <instance.json|.jsonl|.yaml|directory...>
   [--http/-h] [--verbose/-v] [--resolve/-r <schemas-or-directories> ...]
   [--benchmark/-b] [--loop <iterations>] [--extension/-e <extension>]
   [--ignore/-i <schemas-or-directories>] [--trace/-t] [--fast/-f]
@@ -14,8 +14,8 @@ jsonschema validate <schema.json|.yaml> <instance.json|.jsonl|.yaml...>
 
 The most popular use case of JSON Schema is to validate JSON documents. The
 JSON Schema CLI offers a `validate` command to evaluate one or many JSON
-instances or JSONL datasets against a JSON Schema, presenting human-friendly
-information on unsuccessful validation.
+instances, directories of instances, or JSONL datasets against a JSON Schema,
+presenting human-friendly information on unsuccessful validation.
 
 The `--json`/`-j` option outputs the evaluation result using the JSON Schema
 [`Flag`](https://json-schema.org/draft/2020-12/json-schema-core#section-12.4.1) or
@@ -163,3 +163,22 @@ jsonschema validate path/to/my/schema.json path/to/my/instance.json --benchmark
 ```sh
 jsonschema validate path/to/my/schema.json path/to/my/instance.json --trace
 ```
+
+### Validate a directory of instances against a schema
+
+```sh
+jsonschema validate path/to/my/schema.json path/to/instances/
+```
+
+### Validate a directory of instances with a specific extension
+
+```sh
+jsonschema validate path/to/my/schema.json path/to/instances/ --extension .data.json
+```
+
+### Validate a directory of instances while ignoring certain paths
+
+```sh
+jsonschema validate path/to/my/schema.json path/to/instances/ \
+  --ignore path/to/instances/drafts
+```

src/command_validate.cc

@@ -140,14 +140,14 @@ auto sourcemeta::jsonschema::validate(const sourcemeta::core::Options &options)
         "jsonschema validate path/to/schema.json path/to/instance.json"};
   }
 
-  if (options.positional().size() < 2) {
-    throw PositionalArgumentError{
-        "In addition to the schema, you must also pass an argument\n"
-        "that represents the instance to validate against",
-        "jsonschema validate path/to/schema.json path/to/instance.json"};
+  const auto &schema_path{options.positional().at(0)};
+
+  if (std::filesystem::is_directory(schema_path)) {
+    throw std::filesystem::filesystem_error{
+        "The input was supposed to be a file but it is a directory",
+        schema_path, std::make_error_code(std::errc::is_a_directory)};
   }
 
-  const auto &schema_path{options.positional().at(0)};
   const auto configuration_path{find_configuration(schema_path)};
   const auto &configuration{read_configuration(options, configuration_path)};
   const auto dialect{default_dialect(options, configuration)};
@@ -188,10 +188,40 @@ auto sourcemeta::jsonschema::validate(const sourcemeta::core::Options &options)
 
   bool result{true};
 
-  auto iterator{options.positional().cbegin()};
-  std::advance(iterator, 1);
-  for (; iterator != options.positional().cend(); ++iterator) {
-    const std::filesystem::path instance_path{*iterator};
+  std::vector<std::string_view> instance_arguments;
+  if (options.positional().size() > 1) {
+    instance_arguments.assign(options.positional().cbegin() + 1,
+                              options.positional().cend());
+  } else {
+    instance_arguments.push_back(".");
+  }
+
+  if (trace && instance_arguments.size() > 1) {
+    throw std::runtime_error{
+        "The `--trace/-t` option is only allowed given a single instance"};
+  }
+
+  if (benchmark && instance_arguments.size() > 1) {
+    throw std::runtime_error{
+        "The `--benchmark/-b` option is only allowed given a single instance"};
+  }
+
+  for (const auto &instance_path_view : instance_arguments) {
+    const std::filesystem::path instance_path{instance_path_view};
+    if (trace && instance_path.extension() == ".jsonl") {
+      throw std::runtime_error{
+          "The `--trace/-t` option is only allowed given a single instance"};
+    }
+
+    if (trace && std::filesystem::is_directory(instance_path)) {
+      throw std::runtime_error{
+          "The `--trace/-t` option is only allowed given a single instance"};
+    }
+
+    if (benchmark && std::filesystem::is_directory(instance_path)) {
+      throw std::runtime_error{"The `--benchmark/-b` option is only allowed "
+                               "given a single instance"};
+    }
     if (instance_path.extension() == ".jsonl") {
       LOG_VERBOSE(options)
           << "Interpreting input as JSONL: "
@@ -288,6 +318,51 @@ auto sourcemeta::jsonschema::validate(const sourcemeta::core::Options &options)
       if (index == 0) {
         sourcemeta::jsonschema::LOG_WARNING() << "The JSONL file is empty\n";
       }
+    } else if (std::filesystem::is_directory(instance_path)) {
+      for (const auto &entry : for_each_json({instance_path_view}, options)) {
+        std::ostringstream error;
+        sourcemeta::blaze::SimpleOutput output{entry.second};
+        bool subresult{true};
+        if (fast_mode) {
+          subresult = evaluator.validate(schema_template, entry.second);
+        } else if (!json_output) {
+          subresult = evaluator.validate(schema_template, entry.second,
+                                         std::ref(output));
+        }
+
+        if (json_output) {
+          std::cerr << entry.first.string() << "\n";
+          const auto suboutput{sourcemeta::blaze::standard(
+              evaluator, schema_template, entry.second,
+              fast_mode ? sourcemeta::blaze::StandardOutput::Flag
+                        : sourcemeta::blaze::StandardOutput::Basic,
+              entry.positions)};
+          assert(suboutput.is_object());
+          assert(suboutput.defines("valid"));
+          assert(suboutput.at("valid").is_boolean());
+          if (!suboutput.at("valid").to_boolean()) {
+            result = false;
+          }
+
+          sourcemeta::core::prettify(suboutput, std::cout);
+          std::cout << "\n";
+        } else if (subresult) {
+          LOG_VERBOSE(options)
+              << "ok: "
+              << sourcemeta::core::weakly_canonical(entry.first).string()
+              << "\n  matches "
+              << sourcemeta::core::weakly_canonical(schema_path).string()
+              << "\n";
+          print_annotations(output, options, entry.positions, std::cerr);
+        } else {
+          std::cerr << "fail: "
+                    << sourcemeta::core::weakly_canonical(entry.first).string()
+                    << "\n";
+          std::cerr << error.str();
+          print(output, entry.positions, std::cerr);
+          result = false;
+        }
+      }
     } else {
       sourcemeta::core::PointerPositionTracker tracker;
       const auto instance{sourcemeta::core::read_yaml_or_json(

src/main.cc

@@ -32,7 +32,7 @@ Global Options:
 
        Print this command reference help.
 
-   validate <schema.json|.yaml> <instance.json|.jsonl|.yaml...> [--http/-h]
+   validate <schema.json|.yaml> <instance.json|.jsonl|.yaml|directory...> [--http/-h]
             [--benchmark/-b] [--loop <iterations>]
             [--extension/-e <extension>]
             [--ignore/-i <schemas-or-directories>] [--trace/-t] [--fast/-f]
@@ -148,6 +148,7 @@ auto jsonschema_main(const std::string &program, const std::string &command,
     app.flag("trace", {"t"});
     app.flag("fast", {"f"});
     app.option("extension", {"e"});
+    app.option("ignore", {"i"});
     app.option("template", {"m"});
     app.option("loop", {"l"});
     app.parse(argc, argv, {.skip = 1});

test/CMakeLists.txt

@@ -55,25 +55,43 @@ add_jsonschema_test_unix(format/pass_default_dialect_config_relative)
 add_jsonschema_test_unix(format/pass_bignum_real)
 
 # Validate
-add_jsonschema_test_unix(validate/fail_instance_directory)
 add_jsonschema_test_unix(validate/fail_instance_enoent)
 add_jsonschema_test_unix(validate/fail_instance_invalid_json)
 add_jsonschema_test_unix(validate/fail_invalid_ref)
-add_jsonschema_test_unix(validate/fail_no_instance)
 add_jsonschema_test_unix(validate/fail_no_schema)
 add_jsonschema_test_unix(validate/fail_relative_external_ref_missing)
 add_jsonschema_test_unix(validate/fail_resolve_enoent)
 add_jsonschema_test_unix(validate/fail_resolve_directory_with_invalid_json)
 add_jsonschema_test_unix(validate/fail_resolve_invalid_json)
 add_jsonschema_test_unix(validate/fail_resolve_missing_open_brace)
 add_jsonschema_test_unix(validate/fail_schema_directory)
+add_jsonschema_test_unix(validate/pass_directory)
+add_jsonschema_test_unix(validate/pass_directory_verbose)
+add_jsonschema_test_unix(validate/fail_directory)
+add_jsonschema_test_unix(validate/fail_directory_verbose)
+add_jsonschema_test_unix(validate/fail_directory_json)
+add_jsonschema_test_unix(validate/pass_directory_extension)
+add_jsonschema_test_unix(validate/pass_directory_extension_verbose)
+add_jsonschema_test_unix(validate/pass_directory_ignore)
+add_jsonschema_test_unix(validate/pass_directory_ignore_verbose)
+add_jsonschema_test_unix(validate/pass_directory_multiple)
+add_jsonschema_test_unix(validate/pass_directory_and_file)
+add_jsonschema_test_unix(validate/pass_directory_ignore_jsonl)
+add_jsonschema_test_unix(validate/pass_cwd)
+add_jsonschema_test_unix(validate/pass_directory_fast)
+add_jsonschema_test_unix(validate/fail_directory_fast)
+add_jsonschema_test_unix(validate/pass_directory_fast_json)
+add_jsonschema_test_unix(validate/fail_directory_fast_json)
 add_jsonschema_test_unix(validate/fail_schema_enoent)
 add_jsonschema_test_unix(validate/fail_schema_invalid_json)
 add_jsonschema_test_unix(validate/fail_schema_non_schema)
 add_jsonschema_test_unix(validate/fail_schema_unknown_dialect)
 add_jsonschema_test_unix(validate/pass_jsonl_bigint)
 add_jsonschema_test_unix(validate/fail_trace)
 add_jsonschema_test_unix(validate/fail_trace_fast)
+add_jsonschema_test_unix(validate/fail_trace_jsonl)
+add_jsonschema_test_unix(validate/fail_trace_multiple)
+add_jsonschema_test_unix(validate/fail_trace_directory)
 add_jsonschema_test_unix(validate/pass_trace)
 add_jsonschema_test_unix(validate/pass_trace_fast)
 add_jsonschema_test_unix(validate/pass_trace_unknown_keyword)
@@ -136,6 +154,8 @@ add_jsonschema_test_unix(validate/fail_jsonl_one_json)
 add_jsonschema_test_unix(validate/fail_jsonl_one_json_verbose)
 add_jsonschema_test_unix(validate/pass_benchmark)
 add_jsonschema_test_unix(validate/fail_benchmark)
+add_jsonschema_test_unix(validate/fail_benchmark_multiple)
+add_jsonschema_test_unix(validate/fail_benchmark_directory)
 add_jsonschema_test_unix(validate/pass_benchmark_loop)
 add_jsonschema_test_unix(validate/pass_benchmark_loop_jsonl)
 add_jsonschema_test_unix(validate/fail_benchmark_zero)

test/validate/fail_2019_09.sh

@@ -24,8 +24,8 @@ cat << 'EOF' > "$TMP/instance.json"
 EOF
 
 "$1" validate "$TMP/schema.json" "$TMP/instance.json" 2> "$TMP/stderr.txt" \
-  && CODE="$?" || CODE="$?"
-test "$CODE" = "2" || exit 1
+  && EXIT_CODE="$?" || EXIT_CODE="$?"
+test "$EXIT_CODE" = "2" || exit 1
 
 cat << EOF > "$TMP/expected.txt"
 fail: $(realpath "$TMP")/instance.json

test/validate/fail_2020_12.sh

@@ -24,8 +24,8 @@ cat << 'EOF' > "$TMP/instance.json"
 EOF
 
 "$1" validate "$TMP/schema.json" "$TMP/instance.json" 2> "$TMP/stderr.txt" \
-  && CODE="$?" || CODE="$?"
-test "$CODE" = "2" || exit 1
+  && EXIT_CODE="$?" || EXIT_CODE="$?"
+test "$EXIT_CODE" = "2" || exit 1
 
 cat << EOF > "$TMP/expected.txt"
 fail: $(realpath "$TMP")/instance.json

test/validate/fail_2020_12_fast.sh

@@ -24,8 +24,8 @@ cat << 'EOF' > "$TMP/instance.json"
 EOF
 
 "$1" validate "$TMP/schema.json" "$TMP/instance.json" --fast 2> "$TMP/stderr.txt" \
-  && CODE="$?" || CODE="$?"
-test "$CODE" = "2" || exit 1
+  && EXIT_CODE="$?" || EXIT_CODE="$?"
+test "$EXIT_CODE" = "2" || exit 1
 
 cat << EOF > "$TMP/expected.txt"
 fail: $(realpath "$TMP")/instance.json

test/validate/fail_benchmark.sh

@@ -24,8 +24,8 @@ cat << 'EOF' > "$TMP/instance.json"
 EOF
 
 "$1" validate "$TMP/schema.json" "$TMP/instance.json" --benchmark > "$TMP/output.txt" 2>&1 \
-  && CODE="$?" || CODE="$?"
-test "$CODE" = "2" || exit 1
+  && EXIT_CODE="$?" || EXIT_CODE="$?"
+test "$EXIT_CODE" = "2" || exit 1
 
 if ! grep -E "/instance\.json: FAIL [0-9]+\.[0-9]+ \+- [0-9]+\.[0-9]+ us \([0-9]+\.[0-9]+\)$" "$TMP/output.txt" > /dev/null
 then

test/validate/fail_benchmark_directory.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "string"
+}
+EOF
+
+mkdir "$TMP/instances"
+
+cat << 'EOF' > "$TMP/instances/instance_1.json"
+"foo"
+EOF
+
+cat << 'EOF' > "$TMP/instances/instance_2.json"
+"bar"
+EOF
+
+"$1" validate "$TMP/schema.json" "$TMP/instances" --benchmark 2> "$TMP/stderr.txt" && EXIT_CODE="$?" || EXIT_CODE="$?"
+test "$EXIT_CODE" = "1" || exit 1
+
+cat << 'EOF' > "$TMP/expected.txt"
+error: The `--benchmark/-b` option is only allowed given a single instance
+EOF
+
+diff "$TMP/stderr.txt" "$TMP/expected.txt"
+
+# JSON error
+"$1" validate "$TMP/schema.json" "$TMP/instances" --benchmark --json > "$TMP/stdout.txt" 2>&1 && EXIT_CODE="$?" || EXIT_CODE="$?"
+test "$EXIT_CODE" = "1" || exit 1
+
+cat << 'EOF' > "$TMP/expected.txt"
+{
+  "error": "The `--benchmark/-b` option is only allowed given a single instance"
+}
+EOF
+
+diff "$TMP/stdout.txt" "$TMP/expected.txt"

test/validate/fail_benchmark_multiple.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "string"
+}
+EOF
+
+cat << 'EOF' > "$TMP/instance_1.json"
+"foo"
+EOF
+
+cat << 'EOF' > "$TMP/instance_2.json"
+"bar"
+EOF
+
+"$1" validate "$TMP/schema.json" "$TMP/instance_1.json" "$TMP/instance_2.json" --benchmark 2> "$TMP/stderr.txt" && EXIT_CODE="$?" || EXIT_CODE="$?"
+test "$EXIT_CODE" = "1" || exit 1
+
+cat << 'EOF' > "$TMP/expected.txt"
+error: The `--benchmark/-b` option is only allowed given a single instance
+EOF
+
+diff "$TMP/stderr.txt" "$TMP/expected.txt"
+
+# JSON error
+"$1" validate "$TMP/schema.json" "$TMP/instance_1.json" "$TMP/instance_2.json" --benchmark --json > "$TMP/stdout.txt" 2>&1 && EXIT_CODE="$?" || EXIT_CODE="$?"
+test "$EXIT_CODE" = "1" || exit 1
+
+cat << 'EOF' > "$TMP/expected.txt"
+{
+  "error": "The `--benchmark/-b` option is only allowed given a single instance"
+}
+EOF
+
+diff "$TMP/stdout.txt" "$TMP/expected.txt"