feat(c++): support read and write multi-property (#719)

Author: yangxk1

cpp/CMakeLists.txt

@@ -566,6 +566,7 @@ if (BUILD_TESTS)
     add_test(test_arrow_chunk_reader SRCS test/test_arrow_chunk_reader.cc)
     add_test(test_graph SRCS test/test_graph.cc)
     add_test(test_multi_label SRCS test/test_multi_label.cc)
+    add_test(test_multi_property SRCS test/test_multi_property.cc)
 
     # enable_testing()
 endif()

cpp/src/graphar/arrow/chunk_reader.cc

@@ -49,8 +49,11 @@ Result<std::shared_ptr<arrow::Schema>> PropertyGroupToSchema(
         GeneralParams::kVertexIndexCol, arrow::int64()));
   }
   for (const auto& prop : pg->GetProperties()) {
-    fields.push_back(std::make_shared<arrow::Field>(
-        prop.name, DataType::DataTypeToArrowDataType(prop.type)));
+    auto dataType = DataType::DataTypeToArrowDataType(prop.type);
+    if (prop.cardinality != Cardinality::SINGLE) {
+      dataType = arrow::list(dataType);
+    }
+    fields.push_back(std::make_shared<arrow::Field>(prop.name, dataType));
   }
   return arrow::schema(fields);
 }

cpp/src/graphar/arrow/chunk_writer.cc

@@ -321,17 +321,6 @@ Status VertexPropertyWriter::WriteTable(
   return Status::OK();
 }
 
-// Helper function to split a string by a delimiter
-std::vector<std::string> SplitString(const std::string& str, char delimiter) {
-  std::vector<std::string> tokens;
-  std::string token;
-  std::istringstream tokenStream(str);
-  while (std::getline(tokenStream, token, delimiter)) {
-    tokens.push_back(token);
-  }
-  return tokens;
-}
-
 Status VertexPropertyWriter::WriteLabelTable(
     const std::shared_ptr<arrow::Table>& input_table, IdType start_chunk_index,
     FileType file_type, ValidateLevel validate_level) const {
@@ -379,6 +368,8 @@ Result<std::shared_ptr<arrow::Table>> VertexPropertyWriter::GetLabelTable(
     auto label_column = std::static_pointer_cast<arrow::StringArray>(chunk);
 
     // Populate the matrix based on :LABEL column values
+    // TODO(@yangxk):  store array in the label_column, split the string when
+    // reading file
     for (int64_t row = 0; row < label_column->length(); ++row) {
       if (label_column->IsValid(row)) {
         std::string labels_string = label_column->GetString(row);

cpp/src/graphar/fwd.h

@@ -72,6 +72,8 @@ class FileSystem;
 using IdType = int64_t;
 enum class Type;
 class DataType;
+/** Defines how multiple values are handled for a given property key */
+enum Cardinality { SINGLE, LIST, SET };
 /** Type of file format */
 enum FileType { CSV = 0, PARQUET = 1, ORC = 2, JSON = 3 };
 enum SelectType { PROPERTIES = 0, LABELS = 1 };

cpp/src/graphar/graph_info.cc

@@ -20,6 +20,7 @@
 #include <unordered_set>
 #include <utility>
 
+#include "graphar/status.h"
 #include "mini-yaml/yaml/Yaml.hpp"
 
 #include "graphar/filesystem.h"
@@ -86,7 +87,8 @@ std::string BuildPath(const std::vector<std::string>& paths) {
 bool operator==(const Property& lhs, const Property& rhs) {
   return (lhs.name == rhs.name) && (lhs.type == rhs.type) &&
          (lhs.is_primary == rhs.is_primary) &&
-         (lhs.is_nullable == rhs.is_nullable);
+         (lhs.is_nullable == rhs.is_nullable) &&
+         (lhs.cardinality == rhs.cardinality);
 }
 
 PropertyGroup::PropertyGroup(const std::vector<Property>& properties,
@@ -138,6 +140,11 @@ bool PropertyGroup::IsValidated() const {
       // list type is not supported in csv file
       return false;
     }
+    // TODO(@yangxk): support cardinality in csv file
+    if (p.cardinality != Cardinality::SINGLE && file_type_ == FileType::CSV) {
+      // list cardinality is not supported in csv file
+      return false;
+    }
   }
   return true;
 }
@@ -212,6 +219,7 @@ class VertexInfo::Impl {
         property_name_to_primary_.emplace(p.name, p.is_primary);
         property_name_to_nullable_.emplace(p.name, p.is_nullable);
         property_name_to_type_.emplace(p.name, p.type);
+        property_name_to_cardinality_.emplace(p.name, p.cardinality);
       }
     }
   }
@@ -251,6 +259,7 @@ class VertexInfo::Impl {
   std::unordered_map<std::string, bool> property_name_to_nullable_;
   std::unordered_map<std::string, std::shared_ptr<DataType>>
       property_name_to_type_;
+  std::unordered_map<std::string, Cardinality> property_name_to_cardinality_;
 };
 
 VertexInfo::VertexInfo(const std::string& type, IdType chunk_size,
@@ -363,6 +372,15 @@ Result<std::shared_ptr<DataType>> VertexInfo::GetPropertyType(
   return it->second;
 }
 
+Result<Cardinality> VertexInfo::GetPropertyCardinality(
+    const std::string& property_name) const {
+  auto it = impl_->property_name_to_cardinality_.find(property_name);
+  if (it == impl_->property_name_to_cardinality_.end()) {
+    return Status::Invalid("property name not found: ", property_name);
+  }
+  return it->second;
+}
+
 Result<std::shared_ptr<VertexInfo>> VertexInfo::AddPropertyGroup(
     std::shared_ptr<PropertyGroup> property_group) const {
   if (property_group == nullptr) {
@@ -440,8 +458,13 @@ Result<std::shared_ptr<VertexInfo>> VertexInfo::Load(
         bool is_primary = p_node["is_primary"].As<bool>();
         bool is_nullable =
             p_node["is_nullable"].IsNone() || p_node["is_nullable"].As<bool>();
+        Cardinality cardinality = Cardinality::SINGLE;
+        if (!p_node["cardinality"].IsNone()) {
+          cardinality =
+              StringToCardinality(p_node["cardinality"].As<std::string>());
+        }
         property_vec.emplace_back(property_name, property_type, is_primary,
-                                  is_nullable);
+                                  is_nullable, cardinality);
       }
       property_groups.push_back(
           std::make_shared<PropertyGroup>(property_vec, file_type, pg_prefix));
@@ -485,6 +508,9 @@ Result<std::string> VertexInfo::Dump() const noexcept {
         p_node["data_type"] = p.type->ToTypeName();
         p_node["is_primary"] = p.is_primary ? "true" : "false";
         p_node["is_nullable"] = p.is_nullable ? "true" : "false";
+        if (p.cardinality != Cardinality::SINGLE) {
+          p_node["cardinality"] = CardinalityToString(p.cardinality);
+        }
         pg_node["properties"].PushBack();
         pg_node["properties"][pg_node["properties"].Size() - 1] = p_node;
       }
@@ -574,6 +600,13 @@ class EdgeInfo::Impl {
       }
       // check if property name is unique in all property groups
       for (const auto& p : pg->GetProperties()) {
+        if (p.cardinality != Cardinality::SINGLE) {
+          // edge property only supports single cardinality
+          std::cout
+              << "Edge property only supports single cardinality, but got: "
+              << CardinalityToString(p.cardinality) << std::endl;
+          return false;
+        }
         if (check_property_unique_set.find(p.name) !=
             check_property_unique_set.end()) {
           return false;
@@ -910,6 +943,12 @@ Result<std::shared_ptr<EdgeInfo>> EdgeInfo::Load(std::shared_ptr<Yaml> yaml) {
         auto property_name = p_node["name"].As<std::string>();
         auto property_type =
             DataType::TypeNameToDataType(p_node["data_type"].As<std::string>());
+        if (!p_node["cardinality"].IsNone() &&
+            StringToCardinality(p_node["cardinality"].As<std::string>()) !=
+                Cardinality::SINGLE) {
+          return Status::YamlError(
+              "Unsupported set cardinality for edge property.");
+        }
         bool is_primary = p_node["is_primary"].As<bool>();
         bool is_nullable =
             p_node["is_nullable"].IsNone() || p_node["is_nullable"].As<bool>();

cpp/src/graphar/graph_info.h

@@ -37,16 +37,20 @@ class Property {
   std::shared_ptr<DataType> type;  // property data type
   bool is_primary;                 // primary key tag
   bool is_nullable;                // nullable tag for non-primary key
+  Cardinality
+      cardinality;  // cardinality of the property, only use in vertex info
 
   Property() = default;
 
   explicit Property(const std::string& name,
                     const std::shared_ptr<DataType>& type = nullptr,
-                    bool is_primary = false, bool is_nullable = true)
+                    bool is_primary = false, bool is_nullable = true,
+                    Cardinality cardinality = Cardinality::SINGLE)
       : name(name),
         type(type),
         is_primary(is_primary),
-        is_nullable(!is_primary && is_nullable) {}
+        is_nullable(!is_primary && is_nullable),
+        cardinality(cardinality) {}
 };
 
 bool operator==(const Property& lhs, const Property& rhs);
@@ -276,6 +280,8 @@ class VertexInfo {
   Result<std::shared_ptr<DataType>> GetPropertyType(
       const std::string& property_name) const;
 
+  Result<Cardinality> GetPropertyCardinality(
+      const std::string& property_name) const;
   /**
    * Get whether the vertex info contains the specified property.
    *

cpp/src/graphar/types.h

@@ -23,6 +23,7 @@
 #include <memory>
 #include <string>
 #include <utility>
+#include <vector>
 
 #include "graphar/fwd.h"
 #include "graphar/macros.h"
@@ -242,4 +243,42 @@ static inline const char* FileTypeToString(FileType file_type) {
   return file_type2string.at(file_type);
 }
 
+static inline Cardinality StringToCardinality(const std::string& str) {
+  static const std::map<std::string, Cardinality> str2cardinality{
+      {"single", Cardinality::SINGLE},
+      {"list", Cardinality::LIST},
+      {"set", Cardinality::SET},
+  };
+  try {
+    return str2cardinality.at(str.c_str());
+  } catch (const std::exception& e) {
+    throw std::runtime_error("KeyError: " + str);
+  }
+}
+
+static inline const char* CardinalityToString(Cardinality cardinality) {
+  static const std::map<Cardinality, const char*> cardinality2string{
+      {Cardinality::SINGLE, "single"},
+      {Cardinality::LIST, "list"},
+      {Cardinality::SET, "set"},
+  };
+  try {
+    return cardinality2string.at(cardinality);
+  } catch (const std::exception& e) {
+    throw std::runtime_error("KeyError: " +
+                             std::to_string(static_cast<int>(cardinality)));
+  }
+}
+
+// Helper function to split a string by a delimiter
+inline std::vector<std::string> SplitString(const std::string& str,
+                                            char delimiter) {
+  std::vector<std::string> tokens;
+  std::string token;
+  std::istringstream tokenStream(str);
+  while (std::getline(tokenStream, token, delimiter)) {
+    tokens.push_back(token);
+  }
+  return tokens;
+}
 }  // namespace graphar