Read files from stdin

Avoids argument length limitations with cmd line arguments

Author: franzpoeschel

include/openPMD/cli/convert-toml-json.hpp

@@ -1,7 +1,9 @@
 #pragma once
 
+#include <cstdint>
 #include <cstdlib>
 #include <cstring>
+#include <iterator>
 #include <openPMD/auxiliary/JSON_internal.hpp>
 
 #include <iostream>
@@ -91,8 +93,22 @@ class convert_json_toml
         }
     }
 
-    static auto merge(char const **begin, char const **end)
-        -> openPMD::json::ParsedConfig
+    struct ByLine : std::string
+    {
+        friend auto operator>>(std::istream &i, ByLine &l) -> std::istream &
+        {
+            decltype(auto) res = std::getline(i, l);
+            if (res)
+            {
+                l.insert(0, 1, '@');
+            }
+            return res;
+        }
+    };
+    using ByLineIterator = std::istream_iterator<ByLine>;
+
+    template <typename It>
+    static auto merge(It begin, It end) -> openPMD::json::ParsedConfig
     {
         namespace json = openPMD::json;
         if (begin == end)
@@ -116,21 +132,40 @@ class convert_json_toml
     }
 
 public:
+    enum class UseStdinAs : std::uint8_t
+    {
+        InlineJson,
+        ListOfJson
+    };
+
     static void run_application(
-        int argc, char const **argv, void (*print_help_message)(char const *))
+        int argc,
+        char const **argv,
+        UseStdinAs stdinconfig,
+        void (*print_help_message)(char const *))
     {
         std::string jsonOrToml;
         switch (argc)
         {
         case 0:
         case 1:
-            // Just read the whole stream into memory
-            // Not very elegant, but we'll hold the entire JSON/TOML dataset
-            // in memory at some point anyway, so it doesn't really matter
+            switch (stdinconfig)
             {
+            case UseStdinAs::InlineJson: {
+                // Just read the whole stream into memory
+                // Not very elegant, but we'll hold the entire JSON/TOML dataset
+                // in memory at some point anyway, so it doesn't really matter
                 std::stringbuf readEverything;
                 std::cin >> &readEverything;
                 jsonOrToml = readEverything.str();
+                break;
+            }
+            case UseStdinAs::ListOfJson: {
+                auto parsed_config =
+                    merge(ByLineIterator(std::cin), ByLineIterator{});
+                with_parsed_cmdline_args(std::move(parsed_config));
+                break;
+            }
             }
             break;
         default:

src/IO/JSON/JSONIOHandlerImpl.cpp

@@ -2615,18 +2615,18 @@ else
 fi
 echo "Will merge files to '$serial_dir'." >&2
 if [[ -e "$serial_dir" ]]; then
-    echo "Target dir already exists, aborting." >&2
+    echo "Target file already exists, aborting." >&2
     exit 1
 fi
-if ! which openpmd-merge-json 2>/dev/null; then
+if ! which openpmd-merge-json >/dev/null 2>&1; then
     echo "Did not find 'openpmd-merge-json' on PATH, aborting." >&2
     exit 1
 fi
 for file in "$parallel_dir"/mpi_rank_*.json; do
-    echo "@$file"
+    echo "$file"
 done |
-    xargs openpmd-merge-json >"$serial_dir"
-# TODO: xargs will only work up to a certain number of files)END";
+    openpmd-merge-json >"$serial_dir"
+)END";
             std::string const merge_script_path = dirpath + "/merge.sh";
             std::fstream merge_file;
             merge_file.open(

src/cli/convert-toml-json.cpp

@@ -17,6 +17,7 @@ equivalently from TOML to JSON.
 
 int main(int argc, char const **argv)
 {
-    convert_json_toml<from_format_to_format::switch_>::run_application(
-        argc, argv, print_help_message);
+    using convert = convert_json_toml<from_format_to_format::switch_>;
+    convert::run_application(
+        argc, argv, convert::UseStdinAs::InlineJson, print_help_message);
 }

src/cli/merge-json.cpp

@@ -17,6 +17,7 @@ determined by the type of the first file.
 
 int main(int argc, char const **argv)
 {
-    convert_json_toml<from_format_to_format::ID>::run_application(
-        argc, argv, print_help_message);
+    using convert = convert_json_toml<from_format_to_format::ID>;
+    convert::run_application(
+        argc, argv, convert::UseStdinAs::ListOfJson, print_help_message);
 }