Skip to content

Commit b972983

Browse files
emmaling27Convex, Inc.
emmaling27
authored and
Convex, Inc.
committed
Add stagedIndex, stagedSearchIndex, and stagedVectorIndex to TableDefinition (#39673)
Adds `stagedIndex`, `stagedSearchIndex`, and `stagedVectorIndex` to `TableDefinition` on the client and passes through to the server. Doesn't make any changes to backfilling logic - these stage indexes are ignored. Preserves checks for index uniqueness across staged and not-staged indexes GitOrigin-RevId: 86650162b65ffa4673038282efafb98ccb31e989
1 parent 4211cae commit b972983

File tree

11 files changed

+267
-30
lines changed

11 files changed

+267
-30
lines changed

‎crates/application/src/schema_worker/mod.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,8 +294,11 @@ mod tests {
294294
let table_definition = TableDefinition {
295295
table_name: table_name.clone(),
296296
indexes: btreemap! {},
297+
staged_db_indexes: btreemap! {},
297298
search_indexes: btreemap! {},
299+
staged_search_indexes: btreemap! {},
298300
vector_indexes: btreemap! {},
301+
staged_vector_indexes: btreemap! {},
299302
document_type: Some(DocumentSchema::Any),
300303
};
301304
let db_schema = DatabaseSchema {

‎crates/common/src/bootstrap_model/index/index_validation_error.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,13 @@ use crate::{
1010
},
1111
};
1212

13-
pub fn empty_index(table_name: &TableName, index: &IndexSchema) -> ErrorMetadata {
13+
pub fn empty_index(table_name: &TableName, index: &IndexSchema, is_staged: bool) -> ErrorMetadata {
1414
ErrorMetadata::bad_request(
1515
"EmptyIndex",
16-
format!("In table \"{table_name}\" index \"{index}\" must have at least one field."),
16+
format!(
17+
"In table \"{table_name}\" {}index \"{index}\" must have at least one field.",
18+
if is_staged { "staged " } else { "" }
19+
),
1720
)
1821
}
1922
pub fn fields_not_unique_within_index(field: &FieldPath) -> ErrorMetadata {

‎crates/common/src/schemas/json.rs‎

Lines changed: 73 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,11 @@ impl TryFrom<DatabaseSchema> for DatabaseSchemaJson {
116116
pub struct TableDefinitionJson {
117117
table_name: String,
118118
indexes: Vec<IndexSchemaJson>,
119+
staged_db_indexes: Option<Vec<IndexSchemaJson>>,
119120
search_indexes: Option<Vec<SearchIndexSchemaJson>>,
121+
staged_search_indexes: Option<Vec<SearchIndexSchemaJson>>,
120122
vector_indexes: Option<Vec<VectorIndexSchemaJson>>,
123+
staged_vector_indexes: Option<Vec<VectorIndexSchemaJson>>,
121124
document_type: Option<ValidatorJson>,
122125
}
123126

@@ -142,13 +145,12 @@ fn parse_names_and_indexes<T: TryFrom<U, Error = anyhow::Error>, U>(
142145
.map_err(|e: anyhow::Error| e.wrap_error_message(|s| format!("In table \"{table_name}\": {s}")))
143146
}
144147

145-
fn validate_unique_index_fields<T, Y: Clone + Eq + std::hash::Hash>(
146-
indexes: &BTreeMap<IndexDescriptor, T>,
148+
fn validate_unique_index_fields<'a, T: 'a, Y: Clone + Eq + std::hash::Hash>(
149+
indexes: impl Iterator<Item = (&'a IndexDescriptor, &'a T)>,
147150
unique_index_field: impl Fn(&T) -> Y,
148151
non_unique_error: impl Fn(&IndexDescriptor, &IndexDescriptor) -> ErrorMetadata,
149152
) -> anyhow::Result<()> {
150153
let index_fields: BTreeMap<_, _> = indexes
151-
.iter()
152154
.map(|(name, fields)| (name, unique_index_field(fields)))
153155
.collect();
154156

@@ -165,8 +167,11 @@ impl TryFrom<TableDefinitionJson> for TableDefinition {
165167
type Error = anyhow::Error;
166168

167169
fn try_from(j: TableDefinitionJson) -> Result<Self, Self::Error> {
170+
let staged_db_indexes = j.staged_db_indexes.unwrap_or_default();
168171
let search_indexes = j.search_indexes.unwrap_or_default();
172+
let staged_search_indexes = j.staged_search_indexes.unwrap_or_default();
169173
let vector_indexes = j.vector_indexes.unwrap_or_default();
174+
let staged_vector_indexes = j.staged_vector_indexes.unwrap_or_default();
170175

171176
let document_type = j.document_type.map(|t| t.try_into()).transpose()?;
172177

@@ -192,38 +197,69 @@ impl TryFrom<TableDefinitionJson> for TableDefinition {
192197
})?;
193198
for schema in indexes.values() {
194199
if schema.fields.is_empty() {
195-
anyhow::bail!(index_validation_error::empty_index(&table_name, schema));
200+
anyhow::bail!(index_validation_error::empty_index(
201+
&table_name,
202+
schema,
203+
false
204+
));
205+
}
206+
}
207+
let (staged_db_index_names, staged_db_indexes) =
208+
parse_names_and_indexes(&table_name, staged_db_indexes, |idx: &IndexSchema| {
209+
&idx.index_descriptor
210+
})?;
211+
for schema in staged_db_indexes.values() {
212+
if schema.fields.is_empty() {
213+
anyhow::bail!(index_validation_error::empty_index(
214+
&table_name,
215+
schema,
216+
true
217+
));
196218
}
197219
}
198220
validate_unique_index_fields(
199-
&indexes,
200-
|idx| Vec::<FieldPath>::from(idx.fields.clone()),
221+
indexes.iter().chain(staged_db_indexes.iter()),
222+
|idx: &IndexSchema| Vec::<FieldPath>::from(idx.fields.clone()),
201223
|index1, index2| index_not_unique(&table_name, index1, index2),
202224
)?;
203225

204226
let (search_index_names, search_indexes) =
205227
parse_names_and_indexes(&table_name, search_indexes, |idx: &SearchIndexSchema| {
206228
&idx.index_descriptor
207229
})?;
230+
let (staged_search_index_names, staged_search_indexes) = parse_names_and_indexes(
231+
&table_name,
232+
staged_search_indexes,
233+
|idx: &SearchIndexSchema| &idx.index_descriptor,
234+
)?;
208235
validate_unique_index_fields(
209-
&search_indexes,
210-
|idx| idx.search_field.clone(),
236+
search_indexes.iter().chain(staged_search_indexes.iter()),
237+
|idx: &SearchIndexSchema| idx.search_field.clone(),
211238
|index1, index2| search_field_not_unique(&table_name, index1, index2),
212239
)?;
213240

214241
let (vector_index_names, vector_indexes): (Vec<_>, BTreeMap<_, _>) =
215242
parse_names_and_indexes(&table_name, vector_indexes, |idx: &VectorIndexSchema| {
216243
&idx.index_descriptor
217244
})?;
245+
let (staged_vector_index_names, staged_vector_indexes): (Vec<_>, BTreeMap<_, _>) =
246+
parse_names_and_indexes(
247+
&table_name,
248+
staged_vector_indexes,
249+
|idx: &VectorIndexSchema| &idx.index_descriptor,
250+
)?;
218251
validate_unique_index_fields(
219-
&vector_indexes,
220-
|idx| (idx.vector_field.clone(), idx.dimension),
252+
vector_indexes.iter().chain(staged_vector_indexes.iter()),
253+
|idx: &VectorIndexSchema| (idx.vector_field.clone(), idx.dimension),
221254
|index1, index2| vector_field_not_unique(&table_name, index1, index2),
222255
)?;
223256

224257
let all_index_names: Vec<_> = index_names
225258
.into_iter()
259+
.chain(staged_db_index_names)
226260
.chain(search_index_names)
261+
.chain(staged_search_index_names)
262+
.chain(staged_vector_index_names)
227263
.chain(vector_index_names)
228264
.collect();
229265

@@ -248,8 +284,11 @@ impl TryFrom<TableDefinitionJson> for TableDefinition {
248284
Ok(Self {
249285
table_name,
250286
indexes,
287+
staged_db_indexes,
251288
search_indexes,
289+
staged_search_indexes,
252290
vector_indexes,
291+
staged_vector_indexes,
253292
document_type,
254293
})
255294
}
@@ -262,8 +301,11 @@ impl TryFrom<TableDefinition> for TableDefinitionJson {
262301
TableDefinition {
263302
table_name,
264303
indexes,
304+
staged_db_indexes,
265305
search_indexes,
306+
staged_search_indexes,
266307
vector_indexes,
308+
staged_vector_indexes,
267309
document_type,
268310
}: TableDefinition,
269311
) -> anyhow::Result<Self> {
@@ -272,24 +314,45 @@ impl TryFrom<TableDefinition> for TableDefinitionJson {
272314
.into_values()
273315
.map(IndexSchemaJson::try_from)
274316
.collect::<anyhow::Result<Vec<_>>>()?;
317+
let staged_db_indexes = Some(
318+
staged_db_indexes
319+
.into_values()
320+
.map(IndexSchemaJson::try_from)
321+
.collect::<anyhow::Result<Vec<_>>>()?,
322+
);
275323
let search_indexes = Some(
276324
search_indexes
277325
.into_values()
278326
.map(SearchIndexSchemaJson::try_from)
279327
.collect::<anyhow::Result<Vec<_>>>()?,
280328
);
329+
let staged_search_indexes = Some(
330+
staged_search_indexes
331+
.into_values()
332+
.map(SearchIndexSchemaJson::try_from)
333+
.collect::<anyhow::Result<Vec<_>>>()?,
334+
);
281335
let document_type = document_type.map(ValidatorJson::try_from).transpose()?;
282336
let vector_indexes = Some(
283337
vector_indexes
284338
.into_values()
285339
.map(VectorIndexSchemaJson::try_from)
286340
.collect::<anyhow::Result<Vec<_>>>()?,
287341
);
342+
let staged_vector_indexes = Some(
343+
staged_vector_indexes
344+
.into_values()
345+
.map(VectorIndexSchemaJson::try_from)
346+
.collect::<anyhow::Result<Vec<_>>>()?,
347+
);
288348
Ok(TableDefinitionJson {
289349
table_name,
290350
indexes,
351+
staged_db_indexes,
291352
search_indexes,
353+
staged_search_indexes,
292354
vector_indexes,
355+
staged_vector_indexes,
293356
document_type,
294357
})
295358
}

‎crates/common/src/schemas/mod.rs‎

Lines changed: 55 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -169,8 +169,11 @@ macro_rules! db_schema {
169169
let table_def = $crate::schemas::TableDefinition {
170170
table_name: table_name.clone(),
171171
indexes: Default::default(),
172+
staged_db_indexes: Default::default(),
172173
search_indexes: Default::default(),
174+
staged_search_indexes: Default::default(),
173175
vector_indexes: Default::default(),
176+
staged_vector_indexes: Default::default(),
174177
document_type: Some($document_schema),
175178
};
176179
tables.insert(table_name, table_def);
@@ -200,8 +203,11 @@ macro_rules! db_schema_not_validated {
200203
let table_def = $crate::schemas::TableDefinition {
201204
table_name: table_name.clone(),
202205
indexes: Default::default(),
206+
staged_db_indexes: Default::default(),
203207
search_indexes: Default::default(),
208+
staged_search_indexes: Default::default(),
204209
vector_indexes: Default::default(),
210+
staged_vector_indexes: Default::default(),
205211
document_type: Some($document_schema),
206212
};
207213
tables.insert(table_name, table_def);
@@ -253,8 +259,11 @@ macro_rules! db_schema_with_vector_indexes {
253259
let table_def = $crate::schemas::TableDefinition {
254260
table_name: table_name.clone(),
255261
indexes: Default::default(),
262+
staged_db_indexes: Default::default(),
256263
search_indexes: Default::default(),
264+
staged_search_indexes: Default::default(),
257265
vector_indexes,
266+
staged_vector_indexes: Default::default(),
258267
document_type: Some($document_schema),
259268
};
260269
tables.insert(table_name, table_def);
@@ -543,8 +552,11 @@ impl proptest::arbitrary::Arbitrary for DatabaseSchema {
543552
pub struct TableDefinition {
544553
pub table_name: TableName,
545554
pub indexes: BTreeMap<IndexDescriptor, IndexSchema>,
555+
pub staged_db_indexes: BTreeMap<IndexDescriptor, IndexSchema>,
546556
pub search_indexes: BTreeMap<IndexDescriptor, SearchIndexSchema>,
557+
pub staged_search_indexes: BTreeMap<IndexDescriptor, SearchIndexSchema>,
547558
pub vector_indexes: BTreeMap<IndexDescriptor, VectorIndexSchema>,
559+
pub staged_vector_indexes: BTreeMap<IndexDescriptor, VectorIndexSchema>,
548560
pub document_type: Option<DocumentSchema>, /* FIXME: `Option` could be removed here, since
549561
* `None` is handled the same way as
550562
* `Some(DocumentSchema::Any)`. */
@@ -557,29 +569,32 @@ impl TableDefinition {
557569
let index_fields = self
558570
.indexes
559571
.iter()
572+
.chain(self.staged_db_indexes.iter())
560573
.flat_map(|(index_descriptor, index_schema)| {
561574
index_schema
562575
.fields
563576
.iter()
564577
.map(move |field_path| (index_descriptor, field_path))
565578
});
566579

567-
let search_index_fields =
568-
self.search_indexes
569-
.iter()
570-
.map(|(index_descriptor, search_index_schema)| {
571-
(index_descriptor, (&search_index_schema.search_field))
572-
});
580+
let search_index_fields = self
581+
.search_indexes
582+
.iter()
583+
.chain(self.staged_search_indexes.iter())
584+
.map(|(index_descriptor, search_index_schema)| {
585+
(index_descriptor, (&search_index_schema.search_field))
586+
});
573587

574-
let search_index_filter_fields =
575-
self.search_indexes
576-
.iter()
577-
.flat_map(|(index_descriptor, search_index_schema)| {
578-
search_index_schema
579-
.filter_fields
580-
.iter()
581-
.map(move |field_path| (index_descriptor, field_path))
582-
});
588+
let search_index_filter_fields = self
589+
.search_indexes
590+
.iter()
591+
.chain(self.staged_search_indexes.iter())
592+
.flat_map(|(index_descriptor, search_index_schema)| {
593+
search_index_schema
594+
.filter_fields
595+
.iter()
596+
.map(move |field_path| (index_descriptor, field_path))
597+
});
583598

584599
let vector_index_fields = self.vector_fields();
585600

@@ -592,6 +607,7 @@ impl TableDefinition {
592607
pub fn vector_fields(&self) -> impl Iterator<Item = (&IndexDescriptor, &FieldPath)> {
593608
self.vector_indexes
594609
.iter()
610+
.chain(self.staged_vector_indexes.iter())
595611
.map(|(index_descriptor, vector_index_schema)| {
596612
(index_descriptor, (&vector_index_schema.vector_field))
597613
})
@@ -609,16 +625,27 @@ impl proptest::arbitrary::Arbitrary for TableDefinition {
609625

610626
(
611627
prop::collection::vec(any::<IndexSchema>(), 0..6),
628+
prop::collection::vec(any::<IndexSchema>(), 0..6),
629+
prop::collection::vec(any::<SearchIndexSchema>(), 0..3),
612630
prop::collection::vec(any::<SearchIndexSchema>(), 0..3),
613631
prop::collection::vec(any::<VectorIndexSchema>(), 0..3),
632+
prop::collection::vec(any::<VectorIndexSchema>(), 0..3),
614633
any_with::<Option<DocumentSchema>>((
615634
prop::option::Probability::default(),
616635
all_table_names,
617636
)),
618637
)
619638
.prop_filter_map(
620639
"index names must be unique",
621-
move |(indexes, search_indexes, vector_indexes, document_type)| {
640+
move |(
641+
indexes,
642+
staged_db_indexes,
643+
search_indexes,
644+
staged_search_indexes,
645+
vector_indexes,
646+
staged_vector_indexes,
647+
document_type,
648+
)| {
622649
let index_descriptors: BTreeSet<_> = indexes
623650
.iter()
624651
.map(|i| &i.index_descriptor)
@@ -634,14 +661,26 @@ impl proptest::arbitrary::Arbitrary for TableDefinition {
634661
.into_iter()
635662
.map(|i| (i.index_descriptor.clone(), i))
636663
.collect(),
664+
staged_db_indexes: staged_db_indexes
665+
.into_iter()
666+
.map(|i| (i.index_descriptor.clone(), i))
667+
.collect(),
637668
search_indexes: search_indexes
638669
.into_iter()
639670
.map(|i| (i.index_descriptor.clone(), i))
640671
.collect(),
672+
staged_search_indexes: staged_search_indexes
673+
.into_iter()
674+
.map(|i| (i.index_descriptor.clone(), i))
675+
.collect(),
641676
vector_indexes: vector_indexes
642677
.into_iter()
643678
.map(|i| (i.index_descriptor.clone(), i))
644679
.collect(),
680+
staged_vector_indexes: staged_vector_indexes
681+
.into_iter()
682+
.map(|i| (i.index_descriptor.clone(), i))
683+
.collect(),
645684
document_type,
646685
})
647686
} else {

‎crates/database/src/tests/mod.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,8 +281,11 @@ async fn test_build_indexes(rt: TestRuntime) -> anyhow::Result<()> {
281281
TableDefinition {
282282
table_name: table_name.clone(),
283283
indexes,
284+
staged_db_indexes: BTreeMap::new(),
284285
search_indexes: BTreeMap::new(),
286+
staged_search_indexes: BTreeMap::new(),
285287
vector_indexes: BTreeMap::new(),
288+
staged_vector_indexes: BTreeMap::new(),
286289
document_type: None,
287290
},
288291
);
@@ -337,8 +340,11 @@ async fn test_build_indexes(rt: TestRuntime) -> anyhow::Result<()> {
337340
TableDefinition {
338341
table_name,
339342
indexes,
343+
staged_db_indexes: BTreeMap::new(),
340344
search_indexes: BTreeMap::new(),
345+
staged_search_indexes: BTreeMap::new(),
341346
vector_indexes: BTreeMap::new(),
347+
staged_vector_indexes: BTreeMap::new(),
342348
document_type: None,
343349
},
344350
);

0 commit comments

Comments
 (0)