feat(core): External representation enum (#5912)

Author: dranikpg

src/core/compact_object.cc

@@ -875,9 +875,18 @@ uint64_t CompactObj::HashCode(string_view str) {
 }
 
 CompactObjType CompactObj::ObjType() const {
-  if (IsInline() || taglen_ == INT_TAG || taglen_ == SMALL_TAG || taglen_ == EXTERNAL_TAG)
+  if (IsInline() || taglen_ == INT_TAG || taglen_ == SMALL_TAG)
     return OBJ_STRING;
 
+  if (taglen_ == EXTERNAL_TAG) {
+    switch (static_cast<ExternalRep>(u_.ext_ptr.representation)) {
+      case ExternalRep::STRING:
+        return OBJ_STRING;
+      case ExternalRep::SERIALIZED_MAP:
+        return OBJ_HASH;
+    };
+  }
+
   if (taglen_ == ROBJ_TAG)
     return u_.r_obj.type();
 
@@ -1192,15 +1201,21 @@ void CompactObj::GetString(char* dest) const {
   LOG(FATAL) << "Bad tag " << int(taglen_);
 }
 
-void CompactObj::SetExternal(size_t offset, uint32_t sz) {
+void CompactObj::SetExternal(size_t offset, uint32_t sz, ExternalRep rep) {
   SetMeta(EXTERNAL_TAG, mask_);
 
   u_.ext_ptr.is_cool = 0;
+  u_.ext_ptr.representation = static_cast<uint8_t>(rep);
   u_.ext_ptr.page_offset = offset % 4096;
   u_.ext_ptr.serialized_size = sz;
   u_.ext_ptr.offload.page_index = offset / 4096;
 }
 
+CompactObj::ExternalRep CompactObj::GetExternalRep() const {
+  DCHECK(IsExternal());
+  return static_cast<CompactObj::ExternalRep>(u_.ext_ptr.representation);
+}
+
 void CompactObj::SetCool(size_t offset, uint32_t sz, detail::TieredColdRecord* record) {
   // We copy the mask of the "cooled" referenced object because it contains the encoding info.
   SetMeta(EXTERNAL_TAG, record->value.mask_);
@@ -1221,24 +1236,17 @@ auto CompactObj::GetCool() const -> CoolItem {
   return res;
 }
 
-void CompactObj::ImportExternal(const CompactObj& src) {
-  DCHECK(src.IsExternal());
-  uint8_t encoding = src.mask_bits_.encoding;
-  SetMeta(EXTERNAL_TAG, 0);
-  mask_bits_.encoding = encoding;
-  u_.ext_ptr = src.u_.ext_ptr;
-}
-
 std::pair<size_t, size_t> CompactObj::GetExternalSlice() const {
   DCHECK_EQ(EXTERNAL_TAG, taglen_);
   auto& ext = u_.ext_ptr;
   size_t offset = ext.page_offset;
   offset += size_t(ext.is_cool ? ext.cool_record->page_index : ext.offload.page_index) * 4096;
-  return pair<size_t, size_t>(offset, size_t(u_.ext_ptr.serialized_size));
+  return {offset, size_t(u_.ext_ptr.serialized_size)};
 }
 
 void CompactObj::Materialize(std::string_view blob, bool is_raw) {
   CHECK(IsExternal()) << int(taglen_);
+  DCHECK_EQ(u_.ext_ptr.representation, static_cast<uint8_t>(ExternalRep::STRING));
 
   DCHECK_GT(blob.size(), kInlineLen);
 

src/core/compact_object.h

@@ -160,6 +160,12 @@ class CompactObj {
   using PrefixArray = std::vector<std::string_view>;
   using MemoryResource = detail::RobjWrapper::MemoryResource;
 
+  // Different representations of external values
+  enum class ExternalRep : uint8_t {
+    STRING,         // OBJ_STRING, Basic representation with various string encodings
+    SERIALIZED_MAP  // OBJ_HASH, Serialized map
+  };
+
   CompactObj() {  // By default - empty string.
   }
 
@@ -348,7 +354,8 @@ class CompactObj {
     return u_.ext_ptr.is_cool;
   }
 
-  void SetExternal(size_t offset, uint32_t sz);
+  void SetExternal(size_t offset, uint32_t sz, ExternalRep rep);
+  ExternalRep GetExternalRep() const;
 
   // Switches to empty, non-external string.
   // Preserves all the attributes.
@@ -369,8 +376,6 @@ class CompactObj {
   // Returns the external data of the object incuding its ColdRecord.
   CoolItem GetCool() const;
 
-  void ImportExternal(const CompactObj& src);
-
   std::pair<size_t, size_t> GetExternalSlice() const;
 
   // Injects either the the raw string (extracted with GetRawString()) or the usual string
@@ -466,8 +471,9 @@ class CompactObj {
   struct ExternalPtr {
     uint32_t serialized_size;
     uint16_t page_offset;  // 0 for multi-page blobs. != 0 for small blobs.
-    uint16_t is_cool : 1;
-    uint16_t is_reserved : 15;
+    uint8_t is_cool : 1;
+    uint8_t representation : 2;  // See ExternalRep
+    uint16_t is_reserved : 13;
 
     // We do not have enough space in the common area to store page_index together with
     // cool_record pointer. Therefore, we moved this field into TieredColdRecord itself.

src/core/compact_object_test.cc

@@ -17,6 +17,7 @@
 #include "core/huff_coder.h"
 #include "core/mi_memory_resource.h"
 #include "core/page_usage_stats.h"
+#include "core/string_map.h"
 #include "core/string_or_view.h"
 #include "core/string_set.h"
 
@@ -616,13 +617,29 @@ TEST_F(CompactObjectTest, StrEncodingAndMaterialize) {
       EXPECT_EQ(test_str, enc.Decode(raw_str).Take());
 
       // Test Materialize
-      obj.SetExternal(0, 0);  // dummy values
+      obj.SetExternal(0, 0, CompactObj::ExternalRep::STRING);  // dummy values
       obj.Materialize(raw_str, true);
       EXPECT_EQ(test_str, obj.ToString());
     }
   }
 }
 
+TEST_F(CompactObjectTest, ExternalRepresentation) {
+  {
+    CompactObj obj;
+    obj.SetString("test", false);
+    obj.SetExternal(0, 4, CompactObj::ExternalRep::STRING);
+    EXPECT_EQ(obj.ObjType(), OBJ_STRING);
+  }
+  {
+    StringMap sm{};
+    CompactObj obj;
+    obj.SetRObjPtr(&sm);
+    obj.SetExternal(0, 4, CompactObj::ExternalRep::SERIALIZED_MAP);
+    EXPECT_EQ(obj.ObjType(), OBJ_HASH);
+  }
+}
+
 TEST_F(CompactObjectTest, AsanTriggerReadOverflow) {
   cobj_.SetString(string(32, 'a'), false);
   auto dest = make_unique<char[]>(32);

src/server/tiered_storage.cc

@@ -134,6 +134,7 @@ class TieredStorage::ShardOpManager : public tiering::OpManager {
   // Set value to be an in-memory type again. Update memory stats.
   void Upload(DbIndex dbid, string_view value, bool is_raw, size_t serialized_len, PrimeValue* pv) {
     DCHECK(!value.empty());
+    DCHECK_EQ(uint8_t(pv->GetExternalRep()), uint8_t(CompactObj::ExternalRep::STRING));
 
     pv->Materialize(value, is_raw);
     RecordDeleted(*pv, serialized_len, GetDbTableStats(dbid));
@@ -155,7 +156,7 @@ class TieredStorage::ShardOpManager : public tiering::OpManager {
         ts_->CoolDown(key.first, key.second, segment, pv);
       } else {
         stats->AddTypeMemoryUsage(pv->ObjType(), -pv->MallocUsed());
-        pv->SetExternal(segment.offset, segment.length);
+        pv->SetExternal(segment.offset, segment.length, CompactObj::ExternalRep::STRING);
       }
     } else {
       LOG(DFATAL) << "Should not reach here";
@@ -542,7 +543,7 @@ size_t TieredStorage::ReclaimMemory(size_t goal) {
     tiering::DiskSegment segment = FromCoolItem(pv.GetCool());
 
     // Now the item is only in storage.
-    pv.SetExternal(segment.offset, segment.length);
+    pv.SetExternal(segment.offset, segment.length, CompactObj::ExternalRep::STRING);
 
     auto* stats = op_manager_->GetDbTableStats(record->db_index);
     stats->AddTypeMemoryUsage(record->value.ObjType(), -record->value.MallocUsed());