Make HashTable entries use Tag instead of a full hash

`VacantEntry` now stores a `Tag` instead of a full `hash: u64`. This
means that `OccupiedEntry` doesn't need to store anything, because it
can get that tag from the control byte for `remove -> VacantEntry`.
The `get_bucket_entry` method doesn't need a hash argument either.

Also, since `OccupiedEntry` is now smaller, `enum Entry` will be the
same size as `VacantEntry` by using a niche for the discriminant.
(Although this is not _guaranteed_ by the compiler.)

Author: cuviper

src/raw/mod.rs

@@ -1,5 +1,5 @@
 use crate::alloc::alloc::{handle_alloc_error, Layout};
-use crate::control::{BitMaskIter, Group, Tag, TagSliceExt};
+use crate::control::{BitMaskIter, Group, TagSliceExt};
 use crate::scopeguard::{guard, ScopeGuard};
 use crate::util::{invalid_mut, likely, unlikely};
 use crate::TryReserveError;
@@ -11,6 +11,8 @@ use core::ptr::NonNull;
 use core::slice;
 use core::{hint, ptr};
 
+pub(crate) use crate::control::Tag;
+
 mod alloc;
 #[cfg(test)]
 pub(crate) use self::alloc::AllocError;
@@ -848,6 +850,19 @@ impl<T, A: Allocator> RawTable<T, A> {
         )
     }
 
+    /// Removes an element from the table, returning it.
+    ///
+    /// This also returns an `InsertSlot` pointing to the newly free bucket
+    /// and the former `Tag` for that bucket.
+    #[cfg_attr(feature = "inline-more", inline)]
+    #[allow(clippy::needless_pass_by_value)]
+    pub unsafe fn remove_tagged(&mut self, item: Bucket<T>) -> (T, InsertSlot, Tag) {
+        let index = self.bucket_index(&item);
+        let tag = *self.table.ctrl(index);
+        self.table.erase(index);
+        (item.read(), InsertSlot { index }, tag)
+    }
+
     /// Finds and removes an element from the table, returning it.
     #[cfg_attr(feature = "inline-more", inline)]
     pub fn remove_entry(&mut self, hash: u64, eq: impl FnMut(&T) -> bool) -> Option<T> {
@@ -1183,8 +1198,8 @@ impl<T, A: Allocator> RawTable<T, A> {
         }
     }
 
-    /// Inserts a new element into the table in the given slot, and returns its
-    /// raw bucket.
+    /// Inserts a new element into the table in the given slot with the given hash,
+    /// and returns its raw bucket.
     ///
     /// # Safety
     ///
@@ -1193,8 +1208,26 @@ impl<T, A: Allocator> RawTable<T, A> {
     /// occurred since that call.
     #[inline]
     pub unsafe fn insert_in_slot(&mut self, hash: u64, slot: InsertSlot, value: T) -> Bucket<T> {
+        self.insert_tagged_in_slot(Tag::full(hash), slot, value)
+    }
+
+    /// Inserts a new element into the table in the given slot with the given tag,
+    /// and returns its raw bucket.
+    ///
+    /// # Safety
+    ///
+    /// `slot` must point to a slot previously returned by
+    /// `find_or_find_insert_slot`, and no mutation of the table must have
+    /// occurred since that call.
+    #[inline]
+    pub unsafe fn insert_tagged_in_slot(
+        &mut self,
+        tag: Tag,
+        slot: InsertSlot,
+        value: T,
+    ) -> Bucket<T> {
         let old_ctrl = *self.table.ctrl(slot.index);
-        self.table.record_item_insert_at(slot.index, old_ctrl, hash);
+        self.table.record_item_insert_at(slot.index, old_ctrl, tag);
 
         let bucket = self.bucket(slot.index);
         bucket.write(value);
@@ -1269,11 +1302,11 @@ impl<T, A: Allocator> RawTable<T, A> {
     }
 
     /// Returns a pointer to an element in the table, but only after verifying that
-    /// the index is in-bounds and that its control byte matches the given hash.
+    /// the index is in-bounds and the bucket is occupied.
     #[inline]
-    pub fn checked_bucket(&self, hash: u64, index: usize) -> Option<Bucket<T>> {
+    pub fn checked_bucket(&self, index: usize) -> Option<Bucket<T>> {
         unsafe {
-            if index < self.buckets() && *self.table.ctrl(index) == Tag::full(hash) {
+            if index < self.buckets() && self.is_bucket_full(index) {
                 Some(self.bucket(index))
             } else {
                 None
@@ -2456,9 +2489,9 @@ impl RawTableInner {
     }
 
     #[inline]
-    unsafe fn record_item_insert_at(&mut self, index: usize, old_ctrl: Tag, hash: u64) {
+    unsafe fn record_item_insert_at(&mut self, index: usize, old_ctrl: Tag, new_ctrl: Tag) {
         self.growth_left -= usize::from(old_ctrl.special_is_empty());
-        self.set_ctrl_hash(index, hash);
+        self.set_ctrl(index, new_ctrl);
         self.items += 1;
     }
 

src/table.rs

@@ -3,7 +3,7 @@ use core::{fmt, iter::FusedIterator, marker::PhantomData};
 use crate::{
     raw::{
         Allocator, Bucket, Global, InsertSlot, RawDrain, RawExtractIf, RawIntoIter, RawIter,
-        RawIterHash, RawTable,
+        RawIterHash, RawTable, Tag,
     },
     TryReserveError,
 };
@@ -303,7 +303,6 @@ where
     ) -> Result<OccupiedEntry<'_, T, A>, AbsentEntry<'_, T, A>> {
         match self.raw.find(hash, eq) {
             Some(bucket) => Ok(OccupiedEntry {
-                hash,
                 bucket,
                 table: self,
             }),
@@ -413,24 +412,19 @@ where
     ) -> Entry<'_, T, A> {
         match self.raw.find_or_find_insert_slot(hash, eq, hasher) {
             Ok(bucket) => Entry::Occupied(OccupiedEntry {
-                hash,
                 bucket,
                 table: self,
             }),
             Err(insert_slot) => Entry::Vacant(VacantEntry {
-                hash,
+                tag: Tag::full(hash),
                 insert_slot,
                 table: self,
             }),
         }
     }
 
-    /// Returns an `OccupiedEntry` for a bucket index in the table with the given hash,
-    /// or `None` if the index is out of bounds or if its hash doesn't match.
-    ///
-    /// However, note that the hash is only compared for the few bits that are directly stored in
-    /// the table, and even in full this could not guarantee equality. Use [`OccupiedEntry::get`]
-    /// if you need to further validate a match.
+    /// Returns an `OccupiedEntry` for the given bucket index in the table,
+    /// or `None` if it is unoccupied or out of bounds.
     ///
     /// # Examples
     ///
@@ -447,29 +441,25 @@ where
     /// table.insert_unique(hasher(&2), (2, 'b'), |val| hasher(&val.0));
     /// table.insert_unique(hasher(&3), (3, 'c'), |val| hasher(&val.0));
     ///
-    /// let hash = hasher(&2);
-    /// let index = table.find_bucket_index(hash, |val| val.0 == 2).unwrap();
+    /// let index = table.find_bucket_index(hasher(&2), |val| val.0 == 2).unwrap();
     ///
-    /// let bad_hash = !hash;
-    /// assert!(table.get_bucket_entry(bad_hash, index).is_none());
-    /// assert!(table.get_bucket_entry(hash, usize::MAX).is_none());
+    /// assert!(table.get_bucket_entry(usize::MAX).is_none());
     ///
-    /// let occupied_entry = table.get_bucket_entry(hash, index).unwrap();
+    /// let occupied_entry = table.get_bucket_entry(index).unwrap();
     /// assert_eq!(occupied_entry.get(), &(2, 'b'));
     /// assert_eq!(occupied_entry.remove().0, (2, 'b'));
     ///
-    /// assert!(table.find(hash, |val| val.0 == 2).is_none());
+    /// assert!(table.find(hasher(&2), |val| val.0 == 2).is_none());
     /// # }
     /// # fn main() {
     /// #     #[cfg(feature = "nightly")]
     /// #     test()
     /// # }
     /// ```
     #[inline]
-    pub fn get_bucket_entry(&mut self, hash: u64, index: usize) -> Option<OccupiedEntry<'_, T, A>> {
+    pub fn get_bucket_entry(&mut self, index: usize) -> Option<OccupiedEntry<'_, T, A>> {
         Some(OccupiedEntry {
-            hash,
-            bucket: self.raw.checked_bucket(hash, index)?,
+            bucket: self.raw.checked_bucket(index)?,
             table: self,
         })
     }
@@ -573,7 +563,6 @@ where
     ) -> OccupiedEntry<'_, T, A> {
         let bucket = self.raw.insert(hash, value, hasher);
         OccupiedEntry {
-            hash,
             bucket,
             table: self,
         }
@@ -1771,7 +1760,6 @@ pub struct OccupiedEntry<'a, T, A = Global>
 where
     A: Allocator,
 {
-    hash: u64,
     bucket: Bucket<T>,
     table: &'a mut HashTable<T, A>,
 }
@@ -1840,12 +1828,12 @@ where
     /// ```
     #[cfg_attr(feature = "inline-more", inline)]
     pub fn remove(self) -> (T, VacantEntry<'a, T, A>) {
-        let (val, slot) = unsafe { self.table.raw.remove(self.bucket) };
+        let (val, insert_slot, tag) = unsafe { self.table.raw.remove_tagged(self.bucket) };
         (
             val,
             VacantEntry {
-                hash: self.hash,
-                insert_slot: slot,
+                tag,
+                insert_slot,
                 table: self.table,
             },
         )
@@ -2083,7 +2071,7 @@ pub struct VacantEntry<'a, T, A = Global>
 where
     A: Allocator,
 {
-    hash: u64,
+    tag: Tag,
     insert_slot: InsertSlot,
     table: &'a mut HashTable<T, A>,
 }
@@ -2134,10 +2122,9 @@ where
         let bucket = unsafe {
             self.table
                 .raw
-                .insert_in_slot(self.hash, self.insert_slot, value)
+                .insert_tagged_in_slot(self.tag, self.insert_slot, value)
         };
         OccupiedEntry {
-            hash: self.hash,
             bucket,
             table: self.table,
         }